Update to mautrix-python 0.8.0rc1

* Cross-server double puppeting is now possible
* Python 3.9 is now supported
* End-to-bridge encryption no longer requires login_shared_secret,
  but the homeserver must support MSC2778 (Synapse 1.21+)

alembic/versions/33078cd14618_add_double_puppet_base_url_to_puppet_.py

+"""Add double puppet base URL to puppet table
+
+Revision ID: 33078cd14618
+Revises: 7d9a9a4d50d7
+Create Date: 2020-10-26 13:44:49.576035
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '33078cd14618'
+down_revision = '7d9a9a4d50d7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('base_url', sa.Text(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.drop_column('base_url')
+
+    # ### end Alembic commands ###

mautrix_hangouts/__main__.py

@@ -25,6 +25,7 @@ from .matrix import MatrixHandler
 from .context import Context
 from .web import HangoutsAuthServer
 from .version import version, linkified_version
+from . import commands as _
 
 
 class HangoutsBridge(Bridge):

mautrix_hangouts/commands/__init__.py

-from .handler import (CommandProcessor, CommandHandler, CommandEvent, command_handler,
-                      SECTION_AUTH)
 from .auth import login

mautrix_hangouts/commands/auth.py

@@ -15,17 +15,20 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from mautrix.client import Client
 from mautrix.bridge import custom_puppet as cpu
+from mautrix.bridge.commands import HelpSection, command_handler
 
 from hangups import hangouts_pb2 as hangouts
 
 from .. import puppet as pu
-from . import command_handler, CommandEvent, SECTION_AUTH
+from .typehint import CommandEvent
+
+SECTION_AUTH = HelpSection("Authentication", 10, "")
 
 
 @command_handler(needs_auth=False, management_only=True,
                  help_section=SECTION_AUTH, help_text="Log in to Hangouts")
 async def login(evt: CommandEvent) -> None:
-    token = evt.processor.context.auth_server.make_token(evt.sender.mxid)
+    token = evt.bridge.auth_server.make_token(evt.sender.mxid)
     public_prefix = evt.config["bridge.web.auth.public"]
     url = f"{public_prefix}#{token}"
     await evt.reply(f"Please visit the [login portal]({url}) to log in.")

mautrix_hangouts/commands/handler.py

-# mautrix-hangouts - A Matrix-Hangouts puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable, Dict, Optional, NamedTuple
-
-from mautrix.bridge.commands import (HelpSection, CommandEvent as BaseCommandEvent,
-                                     command_handler as base_command_handler,
-                                     CommandHandler as BaseCommandHandler,
-                                     CommandProcessor as BaseCommandProcessor)
-from .. import user as u, context as c, web as w
-
-HelpCacheKey = NamedTuple('HelpCacheKey', is_management=bool, is_admin=bool)
-
-SECTION_AUTH = HelpSection("Authentication", 10, "")
-SECTION_CREATING_PORTALS = HelpSection("Creating portals", 20, "")
-SECTION_PORTAL_MANAGEMENT = HelpSection("Portal management", 30, "")
-SECTION_MISC = HelpSection("Miscellaneous", 40, "")
-SECTION_ADMIN = HelpSection("Administration", 50, "")
-
-
-class CommandEvent(BaseCommandEvent):
-    sender: 'u.User'
-    processor: 'CommandProcessor'
-
-    @property
-    def print_error_traceback(self) -> bool:
-        return self.sender.is_admin
-
-    async def help_key(self) -> HelpCacheKey:
-        return HelpCacheKey(is_management=self.is_management,
-                            is_admin=self.sender.is_admin,)
-                            #is_logged_in=await self.sender.is_logged_in())
-
-
-class CommandHandler(BaseCommandHandler):
-    needs_auth: bool
-    needs_admin: bool
-
-    async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
-        err = await super().get_permission_error(evt)
-        if err:
-            return err
-        elif self.needs_admin and not evt.sender.is_admin:
-            return "This command requires administrator privileges."
-        # elif self.needs_auth and not await evt.sender.is_logged_in():
-        #     return "This command requires you to be logged in."
-        return None
-
-    def has_permission(self, key: HelpCacheKey) -> bool:
-        return ((not self.management_only or key.is_management) and
-                (not self.needs_admin or key.is_admin))# and
-                #(not self.needs_auth or key.is_logged_in))
-
-
-def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] = None, *,
-                    needs_auth: bool = True, needs_admin: bool = False,
-                    management_only: bool = False, name: Optional[str] = None,
-                    help_text: str = "", help_args: str = "", help_section: HelpSection = None
-                    ) -> Callable[[Callable[[CommandEvent], Awaitable[Optional[Dict]]]],
-                                  CommandHandler]:
-    return base_command_handler(_func, management_only=management_only, name=name,
-                                help_text=help_text, help_args=help_args, help_section=help_section,
-                                needs_auth=needs_auth, needs_admin=needs_admin,
-                                _handler_class=CommandHandler)
-
-
-class CommandProcessor(BaseCommandProcessor):
-    context: c.Context
-
-    def __init__(self, context: c.Context) -> None:
-        super().__init__(event_class=CommandEvent, bridge=context.bridge)
-        self.context = context

mautrix_hangouts/commands/typehint.py

+from typing import TYPE_CHECKING
+
+from mautrix.bridge.commands import CommandEvent as BaseCommandEvent
+
+if TYPE_CHECKING:
+    from ..__main__ import HangoutsBridge
+    from ..user import User
+
+
+class CommandEvent(BaseCommandEvent):
+    bridge: 'HangoutsBridge'
+    sender: 'User'

mautrix_hangouts/context.py

@@ -20,6 +20,7 @@ if TYPE_CHECKING:
 
     from mautrix.appservice import AppService
 
+    from .__main__ import HangoutsBridge
     from .config import Config
     from .matrix import MatrixHandler
     from .web import HangoutsAuthServer

mautrix_hangouts/db/puppet.py

@@ -33,6 +33,7 @@ class Puppet(Base):
     custom_mxid: UserID = Column(String(255), nullable=True)
     access_token: str = Column(Text, nullable=True)
     next_batch: SyncToken = Column(String(255), nullable=True)
+    base_url: str = Column(Text, nullable=True)
 
     @classmethod
     def get_by_gid(cls, gid: str) -> Optional['Puppet']:

mautrix_hangouts/example-config.yaml

 # Homeserver details
 homeserver:
     # The address that this appservice can use to connect to the homeserver.
-    address: https://matrix.org
+    address: https://example.com
     # The domain of the homeserver (for MXIDs, etc).
-    domain: matrix.org
+    domain: example.com
     # Whether or not to verify the SSL certificate of the homeserver.
     # Only applies if address starts with https://
     verify_ssl: true
@@ -88,12 +88,20 @@ bridge:
     # Note that updating the m.direct event is not atomic (except with mautrix-asmux)
     # and is therefore prone to race conditions.
     sync_direct_chat_list: false
+    # Servers to always allow double puppeting from
+    double_puppet_server_map:
+        example.com: https://example.com
+    # Allow using double puppeting from any server with a valid client .well-known file.
+    double_puppet_allow_discovery: false
     # Shared secret for https://github.com/devture/matrix-synapse-shared-secret-auth
     #
     # If set, custom puppets will be enabled automatically for local users
     # instead of users having to find an access token and run `login-matrix`
     # manually.
-    login_shared_secret: null
+    # If using this for other servers than the bridge's server,
+    # you must also set the URL in the double_puppet_server_map.
+    login_shared_secret_map:
+        example.com: foobar
     # Whether or not to update avatars when syncing all contacts at startup.
     update_avatar_initial_sync: true
     # End-to-bridge encryption support options. These require matrix-nio to be installed with pip

mautrix_hangouts/matrix.py

@@ -19,7 +19,7 @@ from mautrix.types import (EventID, RoomID, UserID, Event, EventType, MessageEve
                            EncryptedEvent, PresenceEventContent, ReceiptEvent, PresenceState)
 from mautrix.bridge import BaseMatrixHandler
 
-from . import user as u, puppet as pu, portal as po, commands as c
+from . import user as u, puppet as pu, portal as po
 
 if TYPE_CHECKING:
     from .context import Context
@@ -32,7 +32,7 @@ class MatrixHandler(BaseMatrixHandler):
         self.user_id_prefix = f"@{prefix}"
         self.user_id_suffix = f"{suffix}:{homeserver}"
 
-        super().__init__(command_processor=c.CommandProcessor(context), bridge=context.bridge)
+        super().__init__(bridge=context.bridge)
 
     # async def handle_puppet_invite(self, room_id: RoomID, puppet: 'pu.Puppet', invited_by: 'u.User'
     #                                ) -> None:

mautrix_hangouts/puppet.py

@@ -57,6 +57,7 @@ class Puppet(BasePuppet):
     custom_mxid: UserID
     access_token: str
     _next_batch: SyncToken
+    base_url: Optional[URL]
 
     _db_instance: Optional[DBPuppet]
 
@@ -64,7 +65,7 @@ class Puppet(BasePuppet):
 
     def __init__(self, gid: str, name: str = "", photo_url: str = "", is_registered: bool = False,
                  custom_mxid: UserID = "", access_token: str = "", next_batch: SyncToken = "",
-                 db_instance: Optional[DBPuppet] = None) -> None:
+                 base_url: Optional[str] = None, db_instance: Optional[DBPuppet] = None) -> None:
         self.gid = gid
         self.name = name
         self.photo_url = photo_url
@@ -74,6 +75,7 @@ class Puppet(BasePuppet):
         self.custom_mxid = custom_mxid
         self.access_token = access_token
         self._next_batch = next_batch
+        self.base_url = URL(base_url) if base_url else None
 
         self._db_instance = db_instance
 
@@ -104,7 +106,8 @@ class Puppet(BasePuppet):
             self._db_instance = DBPuppet(gid=self.gid, name=self.name, photo_url=self.photo_url,
                                          matrix_registered=self.is_registered,
                                          custom_mxid=self.custom_mxid, next_batch=self.next_batch,
-                                         access_token=self.access_token)
+                                         access_token=self.access_token,
+                                         base_url=str(self.base_url) if self.base_url else None)
         return self._db_instance
 
     @classmethod
@@ -112,12 +115,13 @@ class Puppet(BasePuppet):
         return Puppet(gid=db_puppet.gid, name=db_puppet.name, photo_url=db_puppet.photo_url,
                       is_registered=db_puppet.matrix_registered, custom_mxid=db_puppet.custom_mxid,
                       access_token=db_puppet.access_token, next_batch=db_puppet.next_batch,
-                      db_instance=db_puppet)
+                      base_url=db_puppet.base_url, db_instance=db_puppet)
 
     async def save(self) -> None:
         self.db_instance.edit(name=self.name, photo_url=self.photo_url,
                               matrix_registered=self.is_registered, custom_mxid=self.custom_mxid,
-                              access_token=self.access_token)
+                              access_token=self.access_token,
+                              base_url=str(self.base_url) if self.base_url else None)
 
     # endregion
 
@@ -265,15 +269,18 @@ def init(context: 'Context') -> Iterable[Awaitable[None]]:
     Puppet.az, config, Puppet.loop = context.core
     Puppet.mx = context.mx
     username_template = config["bridge.username_template"].lower()
-    Puppet.sync_with_custom_puppets = config["bridge.sync_with_custom_puppets"]
     index = username_template.index("{userid}")
     length = len("{userid}")
     Puppet.hs_domain = config["homeserver"]["domain"]
     Puppet._mxid_prefix = f"@{username_template[:index]}"
     Puppet._mxid_suffix = f"{username_template[index + length:]}:{Puppet.hs_domain}"
 
-    secret = config["bridge.login_shared_secret"]
-    Puppet.login_shared_secret = secret.encode("utf-8") if secret else None
+    Puppet.sync_with_custom_puppets = config["bridge.sync_with_custom_puppets"]
+    Puppet.homeserver_url_map = {server: URL(url) for server, url
+                                 in config["bridge.double_puppet_server_map"].items()}
+    Puppet.allow_discover_url = config["bridge.double_puppet_allow_discovery"]
+    Puppet.login_shared_secret_map = {server: secret.encode("utf-8") for server, secret
+                                      in config["bridge.login_shared_secret_map"].items()}
     Puppet.login_device_name = "Hangouts Bridge"
 
     return (puppet.start() for puppet in Puppet.get_all_with_custom_mxid())

mautrix_hangouts/user.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import (Any, Dict, Iterator, Optional, List, Awaitable, Union, Callable,
                     TYPE_CHECKING)
+from collections import defaultdict
 from concurrent import futures
 import datetime
 import asyncio
@@ -31,7 +32,7 @@ from mautrix.types import UserID, RoomID
 from mautrix.client import Client as MxClient
 from mautrix.bridge import BaseUser
 from mautrix.bridge._community import CommunityHelper, CommunityID
-from mautrix.util.opt_prometheus import Enum, Summary, async_time
+from mautrix.util.opt_prometheus import Gauge, Summary, async_time
 
 from .config import Config
 from .db import User as DBUser, UserPortal, Contact, Message as DBMessage, Portal as DBPortal
@@ -48,10 +49,8 @@ METRIC_SYNC_USERS = Summary('bridge_sync_users', 'calls to sync_users')
 METRIC_TYPING = Summary('bridge_on_typing', 'calls to on_typing')
 METRIC_EVENT = Summary('bridge_on_event', 'calls to on_event')
 METRIC_RECEIPT = Summary('bridge_on_receipt', 'calls to on_receipt')
-METRIC_LOGGED_IN = Enum('bridge_logged_in', 'Bridge Logged in', states=["true", "false"],
-                        labelnames=("gid",))
-METRIC_CONNECTED = Enum('bridge_connected', 'Bridge Connected', states=["true", "false"],
-                        labelnames=("gid",))
+METRIC_LOGGED_IN = Gauge('bridge_logged_in', 'Number of users logged into the bridge')
+METRIC_CONNECTED = Gauge('bridge_connected', 'Number of users connected to Hangouts')
 
 
 class User(BaseUser):
@@ -99,6 +98,7 @@ class User(BaseUser):
         self.users = None
         self._intentional_disconnect = False
         self.dm_update_lock = asyncio.Lock()
+        self._metric_value = defaultdict(lambda: False)
 
         self.log = self.log.getChild(self.mxid)
 
@@ -202,7 +202,7 @@ class User(BaseUser):
         try:
             self._intentional_disconnect = False
             await self.client.connect()
-            METRIC_CONNECTED.labels(gid=self.gid).state("false")
+            self._track_metric(METRIC_CONNECTED, False)
             if self._intentional_disconnect:
                 self.log.info("Client connection finished")
             else:
@@ -218,7 +218,7 @@ class User(BaseUser):
             await self.client.disconnect()
 
     async def logout(self) -> None:
-        METRIC_LOGGED_IN.labels(gid=self.gid).state("false")
+        self._track_metric(METRIC_LOGGED_IN, False)
         await self.stop()
         self.client = None
         self.gid = None
@@ -250,8 +250,8 @@ class User(BaseUser):
             self.log.exception("Failed to get_self_info")
             return
         self.gid = info.self_entity.id.gaia_id
-        METRIC_CONNECTED.labels(gid=self.gid).state("true")
-        METRIC_LOGGED_IN.labels(gid=self.gid).state("true")
+        self._track_metric(METRIC_CONNECTED, True)
+        self._track_metric(METRIC_LOGGED_IN, True)
         self.name = info.self_entity.properties.display_name
         self.name_future.set_result(self.name)
         self.save()

requirements.txt

-aiohttp>=3,<4
+aiohttp>=3,<3.7
+yarl<1.6
 SQLAlchemy>=1.2,<2
 alembic>=1,<2
 ruamel.yaml>=0.15.94,<0.17
 commonmark>=0.8,<0.10
 python-magic>=0.4,<0.5
 hangups>=0.4.12,<0.5
-mautrix>=0.7.13,<0.8
+mautrix==0.8.0rc1
+setuptools

setup.py

@@ -59,6 +59,7 @@ setuptools.setup(
         "Programming Language :: Python :: 3",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
     ],
     package_data={"mautrix_hangouts": [
         "web/static/*.png", "web/static/*.css", "web/static/*.html", "web/static/*.js",