Merge branch 'unpuppet'

.gitlab-ci.yml

@@ -20,6 +20,8 @@ build:
 build puppeteer:
   stage: build
   image: docker:latest
+  variables:
+    GIT_SUBMODULE_STRATEGY: recursive
   before_script:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
   script:

.gitmodules

+[submodule "puppet/dh-messages-api"]
+	path = puppet/dh-messages-api
+	url = https://github.com/tulir/dh-messages-api.git

Dockerfile

-FROM alpine:3.12
+FROM alpine:3.13
 
 ARG TARGETARCH=amd64
 
-RUN echo $'\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/main\n\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/testing\n\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
-
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
       py3-virtualenv \
@@ -14,22 +9,20 @@ RUN apk add --no-cache \
       py3-aiohttp \
       py3-magic \
       py3-ruamel.yaml \
-      py3-commonmark@edge \
+      py3-commonmark \
       # Other dependencies
       ca-certificates \
       su-exec \
       # encryption
       olm-dev \
       py3-cffi \
-	  py3-pycryptodome \
+      py3-pycryptodome \
       py3-unpaddedbase64 \
       py3-future \
       bash \
       curl \
-      jq && \
-  curl -sLo yq https://github.com/mikefarah/yq/releases/download/3.3.2/yq_linux_${TARGETARCH} && \
-  chmod +x yq && mv yq /usr/bin/yq
-
+      jq \
+      yq
 
 COPY requirements.txt /opt/mautrix-amp/requirements.txt
 COPY optional-requirements.txt /opt/mautrix-amp/optional-requirements.txt

mautrix_amp/__main__.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -33,8 +33,7 @@ class MessagesBridge(Bridge):
     module = "mautrix_amp"
     name = "mautrix-amp"
     command = "python -m mautrix-amp"
-    description = ("A very hacky Matrix-SMS bridge based on using "
-                   "Android Messages for Web in Puppeteer.")
+    description = "A hacky Matrix-SMS bridge using the JS from Android Messages for Web."
     repo_url = "https://github.com/tulir/mautrix-amp"
     real_user_content_key = "net.maunium.amp.puppet"
     version = version

mautrix_amp/commands/auth.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -31,9 +31,9 @@ SECTION_AUTH = HelpSection("Authentication", 10, "")
                  help_text="Log into Android Messages")
 async def login(evt: CommandEvent) -> None:
     status = await evt.sender.client.start()
-    if status.is_logged_in:
-        await evt.reply("You're already logged in")
-        return
+    # if status.is_logged_in:
+    #     await evt.reply("You're already logged in")
+    #     return
     qr_event_id: Optional[EventID] = None
     async for url in evt.sender.client.login():
         buffer = io.BytesIO()
@@ -51,5 +51,6 @@ async def login(evt: CommandEvent) -> None:
         else:
             content.set_reply(evt.event_id)
             qr_event_id = await evt.az.intent.send_message(evt.room_id, content)
-    await evt.reply("Successfully logged in, now syncing")
-    await evt.sender.sync()
+    await evt.reply("Successfully logged in")
+    await evt.sender.client.start(session_data=evt.sender.session_data)
+    # await evt.sender.sync()

mautrix_amp/commands/conn.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -28,20 +28,20 @@ async def set_notice_room(evt: CommandEvent) -> None:
     await evt.reply("This room has been marked as your bridge notice room")
 
 
-@command_handler(needs_auth=False, management_only=True, help_section=SECTION_CONNECTION,
-                 help_text="Check if you're logged into Android Messages")
-async def ping(evt: CommandEvent) -> None:
-    status = await evt.sender.client.start()
-    if status.is_logged_in:
-        await evt.reply("You're logged in")
-    elif status.is_permanently_disconnected or not status.is_connected:
-        await evt.reply("You're not connected")
-    else:
-        await evt.reply("You're not logged in")
-
-
-@command_handler(needs_auth=True, management_only=False, help_section=SECTION_CONNECTION,
-                 help_text="Synchronize portals")
-async def sync(evt: CommandEvent) -> None:
-    await evt.sender.sync()
-    await evt.reply("Synchronization complete")
+# @command_handler(needs_auth=False, management_only=True, help_section=SECTION_CONNECTION,
+#                  help_text="Check if you're logged into Android Messages")
+# async def ping(evt: CommandEvent) -> None:
+#     status = await evt.sender.client.start()
+#     if status.is_logged_in:
+#         await evt.reply("You're logged in")
+#     elif status.is_permanently_disconnected or not status.is_connected:
+#         await evt.reply("You're not connected")
+#     else:
+#         await evt.reply("You're not logged in")
+#
+#
+# @command_handler(needs_auth=True, management_only=False, help_section=SECTION_CONNECTION,
+#                  help_text="Synchronize portals")
+# async def sync(evt: CommandEvent) -> None:
+#     await evt.sender.sync()
+#     await evt.reply("Synchronization complete")

mautrix_amp/config.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -68,10 +68,10 @@ class Config(BaseBridgeConfig):
         copy("bridge.encryption.key_sharing.allow")
         copy("bridge.encryption.key_sharing.require_cross_signing")
         copy("bridge.encryption.key_sharing.require_verification")
-        copy("bridge.private_chat_portal_meta")
         copy("bridge.delivery_receipts")
         copy("bridge.delivery_error_reports")
         copy("bridge.resend_bridge_info")
+        copy("bridge.sync_direct_chat_list")
         copy("bridge.command_prefix")
         copy("bridge.user")
 

mautrix_amp/db/message.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_amp/db/portal.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,7 +17,7 @@ from typing import Optional, ClassVar, List, TYPE_CHECKING
 
 from attr import dataclass
 
-from mautrix.types import RoomID
+from mautrix.types import RoomID, ContentURI
 from mautrix.util.async_db import Database
 
 fake_db = Database("") if TYPE_CHECKING else None
@@ -28,26 +28,36 @@ class Portal:
     db: ClassVar[Database] = fake_db
 
     chat_id: int
-    other_user: str
+    sender_id: Optional[int]
+    other_user: Optional[int]
     mxid: Optional[RoomID]
     name: Optional[str]
+    name_set: bool
+    avatar_hash: Optional[bytes]
+    avatar_mxc: Optional[ContentURI]
+    avatar_set: bool
     encrypted: bool
 
     async def insert(self) -> None:
-        q = ("INSERT INTO portal (chat_id, other_user, mxid, name, encrypted) "
-             "VALUES ($1, $2, $3, $4, $5)")
-        await self.db.execute(q, self.chat_id, self.other_user, self.mxid, self.name,
-                              self.encrypted)
+        q = ("INSERT INTO portal (chat_id, sender_id, other_user, mxid, name, name_set,"
+             "                    avatar_hash, avatar_mxc, avatar_set, encrypted) "
+             "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)")
+        await self.db.execute(q, self.chat_id, self.sender_id, self.other_user, self.mxid,
+                              self.name, self.name_set, self.avatar_hash, self.avatar_mxc,
+                              self.avatar_set, self.encrypted)
 
     async def update(self) -> None:
-        q = ("UPDATE portal SET other_user=$2, mxid=$3, name=$4, encrypted=$5 "
+        q = ("UPDATE portal SET sender_id=$2, other_user=$3, mxid=$4, name=$5, name_set=$6,"
+             "                  avatar_hash=$7, avatar_mxc=$8, avatar_set=$9, encrypted=$10 "
              "WHERE chat_id=$1")
-        await self.db.execute(q, self.chat_id, self.other_user,
-                              self.mxid, self.name, self.encrypted)
+        await self.db.execute(q, self.chat_id, self.sender_id, self.other_user, self.mxid,
+                              self.name, self.name_set, self.avatar_hash, self.avatar_mxc,
+                              self.avatar_set, self.encrypted)
 
     @classmethod
     async def get_by_mxid(cls, mxid: RoomID) -> Optional['Portal']:
-        q = ("SELECT chat_id, other_user, mxid, name, encrypted "
+        q = ("SELECT chat_id, sender_id, other_user, mxid, name, name_set, avatar_hash, "
+             "       avatar_mxc, avatar_set, encrypted "
              "FROM portal WHERE mxid=$1")
         row = await cls.db.fetchrow(q, mxid)
         if not row:
@@ -56,21 +66,36 @@ class Portal:
 
     @classmethod
     async def get_by_chat_id(cls, chat_id: int) -> Optional['Portal']:
-        q = ("SELECT chat_id, other_user, mxid, name, encrypted "
+        q = ("SELECT chat_id, sender_id, other_user, mxid, name, name_set, avatar_hash, "
+             "       avatar_mxc, avatar_set, encrypted "
              "FROM portal WHERE chat_id=$1")
         row = await cls.db.fetchrow(q, chat_id)
         if not row:
             return None
         return cls(**row)
 
+    @classmethod
+    async def get_by_other_user(cls, other_user: int) -> Optional['Portal']:
+        q = ("SELECT chat_id, sender_id, other_user, mxid, name, name_set, avatar_hash, "
+             "       avatar_mxc, avatar_set, encrypted "
+             "FROM portal WHERE other_user=$1")
+        row = await cls.db.fetchrow(q, other_user)
+        if not row:
+            return None
+        return cls(**row)
+
     @classmethod
     async def find_private_chats(cls) -> List['Portal']:
-        rows = await cls.db.fetch("SELECT chat_id, other_user, mxid, name, encrypted "
-                                  "FROM portal WHERE other_user IS NOT NULL")
+        q = ("SELECT chat_id, sender_id, other_user, mxid, name, name_set, avatar_hash, "
+             "       avatar_mxc, avatar_set, encrypted "
+             "FROM portal WHERE other_user IS NOT NULL")
+        rows = await cls.db.fetch(q)
         return [cls(**row) for row in rows]
 
     @classmethod
     async def all_with_room(cls) -> List['Portal']:
-        rows = await cls.db.fetch("SELECT chat_id, other_user, mxid, name, encrypted "
-                                  "FROM portal WHERE mxid IS NOT NULL")
+        q = ("SELECT chat_id, sender_id, other_user, mxid, name, name_set, avatar_hash, "
+             "       avatar_mxc, avatar_set, encrypted "
+             "FROM portal WHERE mxid IS NOT NULL")
+        rows = await cls.db.fetch(q)
         return [cls(**row) for row in rows]

mautrix_amp/db/puppet.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -26,22 +26,23 @@ fake_db = Database("") if TYPE_CHECKING else None
 class Puppet:
     db: ClassVar[Database] = fake_db
 
-    mid: str
+    user_id: int
+    number: Optional[str]
     name: Optional[str]
     is_registered: bool
 
     async def insert(self) -> None:
-        q = "INSERT INTO puppet (mid, name, is_registered) VALUES ($1, $2, $3)"
-        await self.db.execute(q, self.mid, self.name, self.is_registered)
+        q = "INSERT INTO puppet (user_id, number, name, is_registered) VALUES ($1, $2, $3, $4)"
+        await self.db.execute(q, self.user_id, self.number, self.name, self.is_registered)
 
     async def update(self) -> None:
-        q = "UPDATE puppet SET name=$2, is_registered=$3 WHERE mid=$1"
-        await self.db.execute(q, self.mid, self.name, self.is_registered)
+        q = "UPDATE puppet SET number=$2, name=$3, is_registered=$4 WHERE user_id=$1"
+        await self.db.execute(q, self.user_id, self.number, self.name, self.is_registered)
 
     @classmethod
-    async def get_by_mid(cls, mid: str) -> Optional['Puppet']:
-        row = await cls.db.fetchrow("SELECT mid, name, is_registered FROM puppet WHERE mid=$1",
-                                    mid)
+    async def get_by_user_id(cls, user_id: int) -> Optional['Puppet']:
+        q = "SELECT user_id, number, name, is_registered FROM puppet WHERE user_id=$1"
+        row = await cls.db.fetchrow(q, user_id)
         if not row:
             return None
         return cls(**row)

mautrix_amp/db/upgrade.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -46,3 +46,22 @@ async def upgrade_v1(conn: Connection) -> None:
 
         UNIQUE (mxid, mx_room)
     )""")
+
+
+@upgrade_table.register(description="Store Android Messages auth data in database")
+async def upgrade_v2(conn: Connection) -> None:
+    await conn.execute('ALTER TABLE "user" ADD COLUMN session_data jsonb')
+    await conn.execute("ALTER TABLE portal DROP COLUMN other_user")
+    await conn.execute("ALTER TABLE portal ADD COLUMN other_user INTEGER")
+    await conn.execute("ALTER TABLE portal ADD COLUMN sender_id INTEGER")
+    await conn.execute("ALTER TABLE portal ADD COLUMN avatar_hash bytea")
+    await conn.execute("ALTER TABLE portal ADD COLUMN avatar_mxc TEXT")
+    await conn.execute("ALTER TABLE portal ADD COLUMN name_set BOOLEAN DEFAULT false")
+    await conn.execute("ALTER TABLE portal ADD COLUMN avatar_set BOOLEAN DEFAULT false")
+    await conn.execute("DROP TABLE puppet")
+    await conn.execute("""CREATE TABLE puppet (
+        user_id       INTEGER PRIMARY KEY,
+        number        TEXT,
+        name          TEXT,
+        is_registered BOOLEAN NOT NULL DEFAULT false
+    )""")

mautrix_amp/db/user.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,8 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, ClassVar, TYPE_CHECKING
+from typing import Optional, Dict, Any, ClassVar, TYPE_CHECKING
+import json
 
 from attr import dataclass
 
@@ -29,21 +30,28 @@ class User:
 
     mxid: UserID
     notice_room: Optional[RoomID]
+    session_data: Optional[Dict[str, Any]]
+
+    @property
+    def session_data_str(self) -> Optional[str]:
+        return json.dumps(self.session_data) if self.session_data else None
 
     async def insert(self) -> None:
-        q = ('INSERT INTO "user" (mxid, notice_room) '
-             'VALUES ($1, $2)')
-        await self.db.execute(q, self.mxid, self.notice_room)
+        q = ('INSERT INTO "user" (mxid, notice_room, session_data) '
+             'VALUES ($1, $2, $3)')
+        await self.db.execute(q, self.mxid, self.notice_room, self.session_data_str)
 
     async def update(self) -> None:
-        await self.db.execute('UPDATE "user" SET notice_room=$2 WHERE mxid=$1',
-                              self.mxid, self.notice_room)
+        await self.db.execute('UPDATE "user" SET notice_room=$2, session_data=$3 WHERE mxid=$1',
+                              self.mxid, self.notice_room, self.session_data_str)
 
     @classmethod
     async def get_by_mxid(cls, mxid: UserID) -> Optional['User']:
-        q = ("SELECT mxid, notice_room "
+        q = ("SELECT mxid, notice_room, session_data "
              'FROM "user" WHERE mxid=$1')
         row = await cls.db.fetchrow(q, mxid)
         if not row:
             return None
-        return cls(**row)
+        data = {**row}
+        session_data = data.pop("session_data", None)
+        return cls(**data, session_data=json.loads(session_data) if session_data else None)

mautrix_amp/example-config.yaml

@@ -7,6 +7,7 @@ homeserver:
     # Whether or not to verify the SSL certificate of the homeserver.
     # Only applies if address starts with https://
     verify_ssl: true
+    asmux: false
 
 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
@@ -65,7 +66,7 @@ metrics:
 # Bridge config
 bridge:
     # Localpart template of MXIDs for remote users.
-    # {userid} is replaced with the user ID (phone or name converted into a mxid-friendly format).
+    # {userid} is replaced with an internal user ID.
     username_template: "amp_{userid}"
     # Displayname template for remote users.
     # {displayname} is replaced with the display name of the user.
@@ -115,9 +116,6 @@ bridge:
             # Require devices to be verified by the bridge?
             # Verification by the bridge is not yet implemented.
             require_verification: true
-    # Whether or not to explicitly set the avatar and room name for private
-    # chat portal rooms. This will be implicitly enabled if encryption.default is true.
-    private_chat_portal_meta: false
     # Whether or not the bridge should send a read receipt from the bridge bot when a message has
     # been sent.
     delivery_receipts: false
@@ -127,6 +125,10 @@ bridge:
     # This field will automatically be changed back to false after it,
     # except if the config file is not writable.
     resend_bridge_info: false
+    # Whether or not to update the m.direct account data event when double puppeting is enabled.
+    # Note that updating the m.direct event is not atomic (except with mautrix-asmux)
+    # and is therefore prone to race conditions.
+    sync_direct_chat_list: false
 
     # The prefix for commands. Only required in non-management rooms.
     command_prefix: "!am"
@@ -152,7 +154,7 @@ logging:
     version: 1
     formatters:
         colored:
-            (): mautrix_amp.util.ColorFormatter
+            (): mautrix.util.logging.ColorFormatter
             format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
         normal:
             format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"

mautrix_amp/matrix.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_amp/portal.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,9 +13,14 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, Optional, List, Set, Any, AsyncGenerator, NamedTuple, TYPE_CHECKING, cast
+from typing import (Dict, Optional, List, Set, Any, AsyncGenerator, NamedTuple, Deque,
+                    TYPE_CHECKING, cast)
+from collections import deque
 import mimetypes
+import hashlib
 import asyncio
+import base64
+import random
 
 import magic
 
@@ -24,9 +29,7 @@ from mautrix.bridge import BasePortal, NotificationDisabler
 from mautrix.types import (EventID, MessageEventContent, RoomID, EventType, MessageType,
                            TextMessageEventContent, MediaMessageEventContent,
                            ContentURI, EncryptedFile)
-from mautrix.errors import MatrixError
 from mautrix.util.simple_lock import SimpleLock
-from mautrix.util.network_retry import call_with_net_retry
 
 from .db import Portal as DBPortal, Message as DBMessage
 from .config import Config
@@ -51,29 +54,31 @@ ReuploadedMediaInfo = NamedTuple('ReuploadedMediaInfo', mxc=Optional[ContentURI]
 class Portal(DBPortal, BasePortal):
     by_mxid: Dict[RoomID, 'Portal'] = {}
     by_chat_id: Dict[int, 'Portal'] = {}
+    by_other_user: Dict[int, 'Portal'] = {}
     config: Config
     matrix: 'm.MatrixHandler'
     az: AppService
 
-    _main_intent: Optional[IntentAPI]
     _create_room_lock: asyncio.Lock
-    backfill_lock: SimpleLock
-    _last_participant_update: Set[str]
-
-    _main_intent: IntentAPI
-
-    def __init__(self, chat_id: int, other_user: Optional[str] = None,
-                 mxid: Optional[RoomID] = None, name: Optional[str] = None, encrypted: bool = False
-                 ) -> None:
-        super().__init__(chat_id, other_user, mxid, name, encrypted)
+    _last_participant_update: Set[int]
+    _tempid_dedup: Dict[str, EventID]
+    _msgid_dedup: Deque[int]
+    _backfill_requested: Optional[int]
+
+    def __init__(self, chat_id: int, sender_id: Optional[int] = None,
+                 other_user: Optional[int] = None, mxid: Optional[RoomID] = None,
+                 name: Optional[str] = None, name_set: bool = False,
+                 avatar_hash: Optional[bytes] = None, avatar_mxc: Optional[ContentURI] = None,
+                 avatar_set: bool = False, encrypted: bool = False) -> None:
+        super().__init__(chat_id, sender_id, other_user, mxid, name, name_set, avatar_hash,
+                         avatar_mxc, avatar_set, encrypted)
         self._create_room_lock = asyncio.Lock()
         self.log = self.log.getChild(str(chat_id))
 
-        self.backfill_lock = SimpleLock("Waiting for backfilling to finish before handling %s",
-                                        log=self.log)
-        self._main_intent = None
-        self._reaction_lock = asyncio.Lock()
         self._last_participant_update = set()
+        self._tempid_dedup = {}
+        self._msgid_dedup = deque(maxlen=16)
+        self._backfill_requested = None
 
     @property
     def is_direct(self) -> bool:
@@ -81,9 +86,7 @@ class Portal(DBPortal, BasePortal):
 
     @property
     def main_intent(self) -> IntentAPI:
-        if not self._main_intent:
-            raise ValueError("Portal must be postinit()ed before main_intent can be used")
-        return self._main_intent
+        return self.az.intent
 
     @classmethod
     def init_cls(cls, bridge: 'MessagesBridge') -> None:
@@ -91,9 +94,7 @@ class Portal(DBPortal, BasePortal):
         cls.matrix = bridge.matrix
         cls.az = bridge.az
         cls.loop = bridge.loop
-        cls.bridge = bridge
-        NotificationDisabler.puppet_cls = p.Puppet
-        NotificationDisabler.config_enabled = cls.config["bridge.backfill.disable_notifications"]
+        BasePortal.bridge = bridge
 
     async def _send_delivery_receipt(self, event_id: EventID) -> None:
         if event_id and self.config["bridge.delivery_receipts"]:
@@ -126,36 +127,41 @@ class Portal(DBPortal, BasePortal):
             # mime_type = message.info.mimetype or magic.from_buffer(data, mime=True)
             # TODO media
             return
-        message_id = await sender.client.send(self.chat_id, text)
-        msg = DBMessage(mxid=event_id, mx_room=self.mxid, mid=message_id, chat_id=self.chat_id)
-        await msg.insert()
-        await self._send_delivery_receipt(event_id)
-        self.log.debug(f"Handled Matrix message {event_id} -> {message_id}")
+        temp_id = f"tmp_{random.randint(1, 999999999999)}"
+        self._tempid_dedup[temp_id] = event_id
+        await sender.client.send(self.chat_id, self.sender_id, temp_id, text)
+        self.log.debug(f"Handled Matrix message {event_id} -> {temp_id}")
 
     async def handle_matrix_leave(self, user: 'u.User') -> None:
-        if self.is_direct:
-            self.log.info(f"{user.mxid} left private chat portal with {self.other_user}, "
-                          f"cleaning up and deleting...")
-            await self.cleanup_and_delete()
-        else:
-            self.log.debug(f"{user.mxid} left portal to {self.chat_id}")
-            # TODO cleanup if empty
+        self.log.debug(f"{user.mxid} left portal to {self.chat_id}, cleaning up and deleting...")
+        await self.cleanup_and_delete()
 
     async def handle_remote_message(self, source: 'u.User', evt: Message) -> None:
+        if evt.id in self._msgid_dedup:
+            self.log.trace(f"Ignoring duplicate message {evt.id} (found in msgid dedup queue)")
+            return
+        self._msgid_dedup.appendleft(evt.id)
         if evt.is_outgoing:
-            if not source.intent:
+            if evt.temp_id and evt.temp_id in self._tempid_dedup:
+                event_id = self._tempid_dedup.pop(evt.temp_id)
+                self.log.debug(f"Got message ID {evt.id} for Matrix {event_id}/{evt.temp_id}")
+                msg = DBMessage(mxid=event_id, mx_room=self.mxid, mid=evt.id, chat_id=self.chat_id)
+                await msg.insert()
+                await self._send_delivery_receipt(event_id)
+                return
+            elif evt.is_echo:
+                self.log.debug(f"Dropping echo message {evt.id}/{evt.temp_id} with no entry in"
+                               " temp_id dedup map")
+                return
+            elif not source.intent:
                 self.log.warning(f"Ignoring message {evt.id}: double puppeting isn't enabled")
                 return
             intent = source.intent
-        elif self.other_user:
-            intent = (await p.Puppet.get_by_mid(self.other_user)).intent
         else:
-            # TODO group chats
-            self.log.warning(f"Ignoring message {evt.id}: group chats aren't supported yet")
-            return
+            intent = (await p.Puppet.get_by_user_id(evt.user_id)).intent
 
         if await DBMessage.get_by_mid(evt.id):
-            self.log.debug(f"Ignoring duplicate message {evt.id}")
+            self.log.debug(f"Ignoring duplicate message {evt.id} (found in database)")
             return
 
         event_id = None
@@ -186,8 +192,8 @@ class Portal(DBPortal, BasePortal):
             upload_mime_type = "application/octet-stream"
             upload_file_name = None
 
-        mxc = await call_with_net_retry(intent.upload_media, data, mime_type=upload_mime_type,
-                                        filename=upload_file_name, _action="upload media")
+        mxc = await intent.upload_media(data, mime_type=upload_mime_type,
+                                        filename=upload_file_name)
 
         if decryption_info:
             decryption_info.url = mxc
@@ -196,24 +202,49 @@ class Portal(DBPortal, BasePortal):
         return ReuploadedMediaInfo(mxc, decryption_info, mime_type, file_name, len(data))
 
     async def update_info(self, conv: ChatInfo) -> None:
-        if len(conv.participants) == 1:
+        changed = False
+        if self.sender_id != conv.sender_id:
+            self.sender_id = conv.sender_id
+            changed = True
+        if len(conv.participants) == 1 and not self.other_user:
             self.other_user = conv.participants[0].id
-            if self._main_intent is self.az.intent:
-                self._main_intent = (await p.Puppet.get_by_mid(self.other_user)).intent
-        for participant in conv.participants:
-            puppet = await p.Puppet.get_by_mid(participant.id)
-            await puppet.update_info(participant)
-        changed = await self._update_name(conv.name)
+            self.by_other_user[self.other_user] = self
+            changed = True
+        changed = await self._update_name(conv.name) or changed
+        changed = await self._update_avatar(conv.avatar) or changed
         if changed:
             await self.update_bridge_info()
             await self.update()
         await self._update_participants(conv.participants)
 
     async def _update_name(self, name: str) -> bool:
-        if self.name != name:
+        if self.name != name or not self.name_set:
             self.name = name
             if self.mxid:
-                await self.main_intent.set_room_name(self.mxid, name)
+                try:
+                    await self.main_intent.set_room_name(self.mxid, name)
+                    self.name_set = True
+                except Exception:
+                    self.log.exception("Failed to set room name")
+                    self.name_set = False
+            return True
+        return False
+
+    async def _update_avatar(self, avatar_base64: str) -> bool:
+        if not avatar_base64:
+            return False
+        avatar = base64.b64decode(avatar_base64)
+        avatar_hash = hashlib.sha256(avatar).digest()
+        if self.avatar_hash != avatar_hash or not self.avatar_set:
+            self.avatar_hash = avatar_hash
+            try:
+                self.avatar_mxc = await self.az.intent.upload_media(data=avatar)
+                if self.mxid:
+                    await self.main_intent.set_room_avatar(self.mxid, self.avatar_mxc)
+                    self.avatar_set = True
+            except Exception:
+                self.log.exception("Failed to set room avatar")
+                self.avatar_set = False
             return True
         return False
 
@@ -230,42 +261,19 @@ class Portal(DBPortal, BasePortal):
 
         # Make sure puppets who should be here are here
         for participant in participants:
-            puppet = await p.Puppet.get_by_mid(participant.id)
+            puppet = await p.Puppet.get_by_user_id(participant.id)
+            await puppet.update_info(participant)
             await puppet.intent.ensure_joined(self.mxid)
 
-        print(current_members)
-
         # Kick puppets who shouldn't be here
         for user_id in await self.main_intent.get_room_members(self.mxid):
             if user_id == self.az.bot_mxid:
                 continue
             mid = p.Puppet.get_id_from_mxid(user_id)
             if mid and mid not in current_members:
-                print(mid)
                 await self.main_intent.kick_user(self.mxid, user_id,
                                                  reason="User had left this chat")
 
-    async def backfill(self, source: 'u.User') -> None:
-        with self.backfill_lock:
-            await self._backfill(source)
-
-    async def _backfill(self, source: 'u.User') -> None:
-        self.log.debug("Backfilling history through %s", source.mxid)
-
-        max_mid = await DBMessage.get_max_mid(self.mxid) or 0
-        messages = [msg for msg in await source.client.get_messages(self.chat_id)
-                    if msg.id > max_mid]
-
-        if not messages:
-            self.log.debug("Didn't get any entries from server")
-            return
-
-        self.log.debug("Got %d messages from server", len(messages))
-        async with NotificationDisabler(self.mxid, source):
-            for evt in messages:
-                await self.handle_remote_message(source, evt)
-        self.log.info("Backfilled %d messages through %s", len(messages), source.mxid)
-
     @property
     def bridge_info_state_key(self) -> str:
         return f"net.maunium.amp://androidmessages/{self.chat_id}"
@@ -300,7 +308,7 @@ class Portal(DBPortal, BasePortal):
         except Exception:
             self.log.warning("Failed to update bridge info", exc_info=True)
 
-    async def update_matrix_room(self, source: 'u.User', info: ChatInfo) -> Optional[RoomID]:
+    async def update_matrix_room(self, source: 'u.User', info: ChatInfo) -> None:
         try:
             await self._update_matrix_room(source, info)
         except Exception:
@@ -313,21 +321,40 @@ class Portal(DBPortal, BasePortal):
         async with self._create_room_lock:
             return await self._create_matrix_room(source, info)
 
-    async def _update_matrix_room(self, source: 'u.User', info: ChatInfo) -> None:
-        await self.main_intent.invite_user(self.mxid, source.mxid, check_cache=True)
-        puppet = await p.Puppet.get_by_custom_mxid(source.mxid)
-        if puppet:
-            await puppet.intent.ensure_joined(self.mxid)
+    async def _invite_source(self, source: 'u.User') -> None:
+        try:
+            await self.main_intent.invite_user(self.mxid, source.mxid, check_cache=True)
+            puppet = await p.Puppet.get_by_custom_mxid(source.mxid)
+            if puppet:
+                await puppet.intent.ensure_joined(self.mxid)
+        except Exception:
+            self.log.exception("Error inviting user to portal room")
 
+    async def _update_direct_chats(self, source: 'u.User') -> None:
+        if self.is_direct:
+            await source.update_direct_chats({
+                p.Puppet.get_mxid_from_id(self.other_user): [self.mxid],
+            })
+
+    async def _update_matrix_room(self, source: 'u.User', info: ChatInfo) -> None:
+        await self._update_direct_chats(source)
+        await self._invite_source(source)
         await self.update_info(info)
 
+        if not self._backfill_requested or self._backfill_requested < info.last_msg_id:
+            last_msg_id = await DBMessage.get_max_mid(self.mxid)
+            if not last_msg_id or last_msg_id < info.last_msg_id:
+                self.log.debug("Asking for more messages as chat info says last ID is "
+                               f"{info.last_msg_id}, but we've only bridged up to {last_msg_id}")
+                self._backfill_requested = info.last_msg_id
+                await source.client.request_messages(self.chat_id)
+
     async def _create_matrix_room(self, source: 'u.User', info: ChatInfo) -> Optional[RoomID]:
         if self.mxid:
             await self._update_matrix_room(source, info)
             return self.mxid
         await self.update_info(info)
         self.log.debug("Creating Matrix room")
-        name: Optional[str] = None
         initial_state = [{
             "type": str(StateBridge),
             "state_key": self.bridge_info_state_key,
@@ -338,17 +365,12 @@ class Portal(DBPortal, BasePortal):
             "state_key": self.bridge_info_state_key,
             "content": self.bridge_info,
         }]
-        invites = [source.mxid]
         if self.config["bridge.encryption.default"] and self.matrix.e2ee:
             self.encrypted = True
             initial_state.append({
                 "type": str(EventType.ROOM_ENCRYPTION),
                 "content": {"algorithm": "m.megolm.v1.aes-sha2"},
             })
-            if self.is_direct:
-                invites.append(self.az.bot_mxid)
-        if self.encrypted or not self.is_direct:
-            name = self.name
         if self.config["appservice.community_id"]:
             initial_state.append({
                 "type": "m.room.related_groups",
@@ -358,62 +380,49 @@ class Portal(DBPortal, BasePortal):
             "type": str(EventType.ROOM_POWER_LEVELS),
             "content": {
                 "users": {
-                    self.az.bot_mxid: 100,
-                    self.main_intent.mxid: 9001,
+                    self.az.bot_mxid: 9001,
+                    source.mxid: -1 if info.readonly else 0,
                 },
-                "events": {},
-                "events_default": 100 if info.readonly else 0,
+                "events": {
+                    str(EventType.ROOM_NAME): -1,
+                    str(EventType.ROOM_TOPIC): -1,
+                    str(EventType.ROOM_AVATAR): -1,
+                },
+                "events_default": 0,
                 "state_default": 50,
                 "invite": 50,
                 "redact": 0
             }
         })
+        if self.avatar_mxc:
+            initial_state.append({
+                "type": str(EventType.ROOM_AVATAR),
+                "content": {"url": self.avatar_mxc},
+            })
+            self.avatar_set = True
 
-        # We lock backfill lock here so any messages that come between the room being created
-        # and the initial backfill finishing wouldn't be bridged before the backfill messages.
-        with self.backfill_lock:
-            self.mxid = await self.main_intent.create_room(name=name, is_direct=self.is_direct,
-                                                           initial_state=initial_state,
-                                                           invitees=invites)
-            if not self.mxid:
-                raise Exception("Failed to create room: no mxid returned")
-
-            if self.encrypted and self.matrix.e2ee and self.is_direct:
-                try:
-                    await self.az.intent.ensure_joined(self.mxid)
-                except Exception:
-                    self.log.warning("Failed to add bridge bot "
-                                     f"to new private chat {self.mxid}")
-
-            await self.update()
-            self.log.debug(f"Matrix room created: {self.mxid}")
-            self.by_mxid[self.mxid] = self
-            if not self.is_direct:
-                await self._update_participants(info.participants)
-            else:
-                puppet = await p.Puppet.get_by_custom_mxid(source.mxid)
-                if puppet:
-                    try:
-                        await puppet.intent.join_room_by_id(self.mxid)
-                    except MatrixError:
-                        self.log.debug("Failed to join custom puppet into newly created portal",
-                                       exc_info=True)
+        self.mxid = await self.az.intent.create_room(name=self.name,
+                                                     is_direct=self.is_direct,
+                                                     initial_state=initial_state)
+        if not self.mxid:
+            raise Exception("Failed to create room: no mxid returned")
 
-            try:
-                await self.backfill(source)
-            except Exception:
-                self.log.exception("Failed to backfill new portal")
+        await self.update()
+        self.log.debug(f"Matrix room created: {self.mxid}")
+        self.by_mxid[self.mxid] = self
+        await self._update_participants(info.participants)
+        await source.client.request_messages(self.chat_id)
+        await self._update_direct_chats(source)
+        await self._invite_source(source)
 
         return self.mxid
 
-    async def postinit(self) -> None:
+    def _add_to_cache(self) -> None:
         self.by_chat_id[self.chat_id] = self
         if self.mxid:
             self.by_mxid[self.mxid] = self
         if self.other_user:
-            self._main_intent = (await p.Puppet.get_by_mid(self.other_user)).intent
-        else:
-            self._main_intent = self.az.intent
+            self.by_other_user[self.other_user] = self
 
     async def delete(self) -> None:
         await DBMessage.delete_all(self.mxid)
@@ -433,7 +442,7 @@ class Portal(DBPortal, BasePortal):
             try:
                 yield cls.by_chat_id[portal.chat_id]
             except KeyError:
-                await portal.postinit()
+                portal._add_to_cache()
                 yield portal
 
     @classmethod
@@ -445,7 +454,7 @@ class Portal(DBPortal, BasePortal):
 
         portal = cast(cls, await super().get_by_mxid(mxid))
         if portal is not None:
-            await portal.postinit()
+            portal._add_to_cache()
             return portal
 
         return None
@@ -459,13 +468,27 @@ class Portal(DBPortal, BasePortal):
 
         portal = cast(cls, await super().get_by_chat_id(chat_id))
         if portal is not None:
-            await portal.postinit()
+            portal._add_to_cache()
             return portal
 
         if create:
             portal = cls(chat_id)
             await portal.insert()
-            await portal.postinit()
+            portal._add_to_cache()
+            return portal
+
+        return None
+
+    @classmethod
+    async def get_by_other_user(cls, other_user: int) -> Optional['Portal']:
+        try:
+            return cls.by_other_user[other_user]
+        except KeyError:
+            pass
+
+        portal = cast(cls, await super().get_by_other_user(other_user))
+        if portal is not None:
+            portal._add_to_cache()
             return portal
 
         return None

mautrix_amp/puppet.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -22,27 +22,28 @@ from mautrix.util.simple_template import SimpleTemplate
 from .db import Puppet as DBPuppet
 from .config import Config
 from .rpc import Participant
-from . import user as u
+from . import user as u, portal as p
 
 if TYPE_CHECKING:
     from .__main__ import MessagesBridge
 
 
 class Puppet(DBPuppet, BasePuppet):
-    by_mid: Dict[str, 'Puppet'] = {}
+    by_user_id: Dict[int, 'Puppet'] = {}
     hs_domain: str
-    mxid_template: SimpleTemplate[str]
+    mxid_template: SimpleTemplate[int]
 
     bridge: 'MessagesBridge'
     config: Config
 
     default_mxid: UserID
 
-    def __init__(self, mid: str, name: Optional[str] = None, is_registered: bool = False) -> None:
-        super().__init__(mid=mid, name=name, is_registered=is_registered)
-        self.log = self.log.getChild(mid)
+    def __init__(self, user_id: int, number: Optional[str] = None, name: Optional[str] = None,
+                 is_registered: bool = False) -> None:
+        super().__init__(user_id=user_id, number=number, name=name, is_registered=is_registered)
+        self.log = self.log.getChild(str(user_id))
 
-        self.default_mxid = self.get_mxid_from_id(mid)
+        self.default_mxid = self.get_mxid_from_id(user_id)
         self.intent = self.az.intent.user(self.default_mxid)
 
     @classmethod
@@ -54,19 +55,23 @@ class Puppet(DBPuppet, BasePuppet):
         cls.bridge = bridge
         cls.hs_domain = cls.config["homeserver.domain"]
         cls.mxid_template = SimpleTemplate(cls.config["bridge.username_template"], "userid",
-                                           prefix="@", suffix=f":{cls.hs_domain}", type=str)
+                                           prefix="@", suffix=f":{cls.hs_domain}", type=int)
         secret = cls.config["bridge.login_shared_secret"]
         cls.login_shared_secret_map[cls.hs_domain] = secret.encode("utf-8") if secret else None
         cls.login_device_name = "Android Messages Bridge"
 
     async def update_info(self, info: Participant) -> None:
-        update = False
-        update = await self._update_name(info.name) or update
+        update = self.number != info.number
+        self.number = info.number
+        update = await self._update_name() or update
         if update:
             await self.update()
 
-    async def _update_name(self, name: str) -> bool:
-        name = self.config["bridge.displayname_template"].format(displayname=name)
+    async def _update_name(self) -> bool:
+        portal = await p.Portal.get_by_other_user(self.user_id)
+        displayname = (portal.name if portal else None) or self.number
+        name = self.config["bridge.displayname_template"].format(displayname=displayname,
+                                                                 phone=self.number)
         if name != self.name:
             self.name = name
             await self.intent.set_displayname(self.name)
@@ -74,40 +79,45 @@ class Puppet(DBPuppet, BasePuppet):
         return False
 
     def _add_to_cache(self) -> None:
-        self.by_mid[self.mid] = self
+        self.by_user_id[self.user_id] = self
 
     async def save(self) -> None:
         await self.update()
 
     @classmethod
     async def get_by_mxid(cls, mxid: UserID, create: bool = True) -> Optional['Puppet']:
-        mid = cls.get_id_from_mxid(mxid)
-        if mid:
-            return await cls.get_by_mid(mid, create)
+        user_id = cls.get_id_from_mxid(mxid)
+        if user_id:
+            return await cls.get_by_user_id(user_id, create)
         return None
 
     @classmethod
-    def get_id_from_mxid(cls, mxid: UserID) -> Optional[str]:
-        return cls.mxid_template.parse(mxid)
+    def get_id_from_mxid(cls, mxid: UserID) -> Optional[int]:
+        user_id = cls.mxid_template.parse(mxid)
+        # Legacy user IDs had phone numbers, which are too big for the new database,
+        # so filter those out here.
+        if isinstance(user_id, int) and user_id < 2**31:
+            return user_id
+        return None
 
     @classmethod
-    def get_mxid_from_id(cls, mid: str) -> UserID:
+    def get_mxid_from_id(cls, mid: int) -> UserID:
         return UserID(cls.mxid_template.format_full(mid))
 
     @classmethod
-    async def get_by_mid(cls, mid: str, create: bool = True) -> Optional['Puppet']:
+    async def get_by_user_id(cls, user_id: int, create: bool = True) -> Optional['Puppet']:
         try:
-            return cls.by_mid[mid]
+            return cls.by_user_id[user_id]
         except KeyError:
             pass
 
-        puppet = cast(cls, await super().get_by_mid(mid))
+        puppet = cast(cls, await super().get_by_user_id(user_id))
         if puppet is not None:
             puppet._add_to_cache()
             return puppet
 
         if create:
-            puppet = cls(mid)
+            puppet = cls(user_id)
             await puppet.insert()
             puppet._add_to_cache()
             return puppet

mautrix_amp/rpc/__init__.py

 from .client import Client
-from .types import RPCError, ChatListInfo, ChatInfo, Participant, Message, StartStatus
+from .types import RPCError, ChatInfo, Participant, Message, StartStatus

mautrix_amp/rpc/client.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,7 +18,7 @@ from collections import deque
 import asyncio
 
 from .rpc import RPCClient
-from .types import ChatListInfo, ChatInfo, Message, StartStatus
+from .types import ChatInfo, Message, StartStatus
 
 
 class QRCommand(TypedDict):
@@ -30,41 +30,41 @@ class LoginComplete(Exception):
 
 
 class Client(RPCClient):
-    async def start(self) -> StartStatus:
+    async def start(self, session_data: Any = None) -> StartStatus:
         await self.connect()
-        return StartStatus.deserialize(await self.request("start"))
+        return StartStatus.deserialize(await self.request("start", session_data=session_data))
 
     async def stop(self) -> None:
         await self.request("stop")
         await self.disconnect()
 
-    async def get_chats(self) -> List[ChatListInfo]:
-        resp = await self.request("get_chats")
-        return [ChatListInfo.deserialize(data) for data in resp]
-
-    async def get_chat(self, chat_id: int) -> ChatInfo:
-        return ChatInfo.deserialize(await self.request("get_chat", chat_id=chat_id))
-
-    async def get_messages(self, chat_id: int) -> List[Message]:
-        resp = await self.request("get_messages", chat_id=chat_id)
-        return [Message.deserialize(data) for data in resp]
+    async def request_messages(self, chat_id: int) -> None:
+        await self.request("get_messages", chat_id=chat_id)
 
     async def is_connected(self) -> bool:
         resp = await self.request("is_connected")
         return resp["is_connected"]
 
-    async def send(self, chat_id: int, text: str) -> int:
-        resp = await self.request("send", chat_id=chat_id, text=text)
-        return resp["id"]
+    async def send(self, chat_id: int, sender_id: int, temp_id: str, text: str) -> None:
+        await self.request("send", chat_id=chat_id, sender_id=sender_id, temp_id=temp_id,
+                           text=text)
 
-    async def set_last_message_ids(self, msg_ids: Dict[int, int]) -> None:
-        await self.request("set_last_message_ids", msg_ids=msg_ids)
-
-    async def on_message(self, func: Callable[[Message], Awaitable[None]]) -> None:
+    def on_message(self, func: Callable[[Message], Awaitable[None]]) -> None:
         async def wrapper(data: Dict[str, Any]) -> None:
             await func(Message.deserialize(data["message"]))
 
+        async def list_wrapper(data: Dict[str, Any]) -> None:
+            for item in data["messages"]:
+                await func(Message.deserialize(item))
+
         self.add_event_handler("message", wrapper)
+        self.add_event_handler("message_list", list_wrapper)
+
+    def on_chat_update(self, func: Callable[[ChatInfo], Awaitable[None]]) -> None:
+        async def wrapper(data: Dict[str, Any]) -> None:
+            await func(ChatInfo.deserialize(data["chat"]))
+
+        self.add_event_handler("chat", wrapper)
 
     async def login(self) -> AsyncGenerator[str, None]:
         data = deque()

mautrix_amp/rpc/rpc.py

-# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer
-# Copyright (C) 2020 Tulir Asokan
+# mautrix-amp - A hacky Matrix-SMS bridge using the JS from Android Messages for Web
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_amp/rpc/types.py

@@ -25,22 +25,20 @@ class RPCError(Exception):
 
 
 @dataclass
-class ChatListInfo(SerializableAttrs['ChatListInfo']):
+class Participant(SerializableAttrs['Participant']):
     id: int
-    name: str
-    lastMsg: str
-    lastMsgDate: str
+    number: str
 
 
 @dataclass
-class Participant(SerializableAttrs['Participant']):
-    id: str
+class ChatInfo(SerializableAttrs['ChatInfo']):
+    id: int
     name: str
-
-
-@dataclass
-class ChatInfo(ChatListInfo, SerializableAttrs['ChatInfo']):
+    sender_id: int
+    last_msg_id: int
+    last_msg_timestamp: int
     participants: List[Participant]
+    avatar: Optional[str]
     readonly: bool
 
 
@@ -48,8 +46,12 @@ class ChatInfo(ChatListInfo, SerializableAttrs['ChatInfo']):
 class Message(SerializableAttrs['Message']):
     id: int
     chat_id: int
+    user_id: int
     is_outgoing: bool
-    timestamp: int = None
+    is_echo: bool
+    timestamp: int
+    temp_id: Optional[str] = None
+    subject: Optional[str] = None
     text: Optional[str] = None
     image: Optional[str] = None
 
@@ -57,6 +59,6 @@ class Message(SerializableAttrs['Message']):
 @dataclass
 class StartStatus(SerializableAttrs['StartStatus']):
     started: bool
-    is_logged_in: bool
-    is_connected: bool
-    is_permanently_disconnected: bool
+    # is_logged_in: bool
+    # is_connected: bool
+    # is_permanently_disconnected: bool