From ceb54a7e3cfb79845e2b5e4f562fff7bae9f303f Mon Sep 17 00:00:00 2001 From: Sumner Evans <sumner.evans@automattic.com> Date: Fri, 28 Feb 2025 13:38:45 -0700 Subject: [PATCH] legacymigrate: implement Signed-off-by: Sumner Evans <sumner.evans@automattic.com> --- ROADMAP.md | 10 +- cmd/mautrix-linkedin/legacymigrate.go | 15 +++ cmd/mautrix-linkedin/legacymigrate.sql | 166 +++++++++++++++++++++++++ cmd/mautrix-linkedin/main.go | 71 +++++++++++ pkg/connector/client.go | 4 +- pkg/connector/dbmeta.go | 2 +- pkg/connector/ids.go | 6 +- pkg/connector/sync.go | 2 +- pkg/linkedingo/client.go | 4 +- pkg/linkedingo/request.go | 2 +- pkg/linkedingo/stringcookiejar.go | 45 ++++--- pkg/linkedingo/stringcookiejar_test.go | 6 +- 12 files changed, 295 insertions(+), 38 deletions(-) create mode 100644 cmd/mautrix-linkedin/legacymigrate.sql diff --git a/ROADMAP.md b/ROADMAP.md index 3c24cc1..c55ae3e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -42,7 +42,7 @@ * [x] Message edits * [x] Message delete * [x] Message reactions - * [ ] Message history + * [x] Message history * [x] Real-time messages * [ ] ~~Presence~~ (impossible for now, see https://github.com/mautrix/go/issues/295) * [x] Typing notifications @@ -62,10 +62,10 @@ * Misc * [ ] Multi-user support * [ ] Shared group chat portals - * [ ] Automatic portal creation - * [ ] At startup - * [ ] When added to chat + * [x] Automatic portal creation + * [x] At startup + * [x] When added to chat * [x] When receiving message * [ ] Private chat creation by inviting Matrix puppet of LinkedIn user to new room * [ ] Option to use own Matrix account for messages sent from other LinkedIn clients (relay mode) - * [ ] Split portal support + * [x] Split portal support diff --git a/cmd/mautrix-linkedin/legacymigrate.go b/cmd/mautrix-linkedin/legacymigrate.go index a7879ef..05464ce 100644 --- a/cmd/mautrix-linkedin/legacymigrate.go +++ b/cmd/mautrix-linkedin/legacymigrate.go @@ -17,9 +17,24 @@ package main import ( + _ "embed" + up "go.mau.fi/util/configupgrade" ) +const legacyMigrateRenameTables = ` +ALTER TABLE cookie RENAME TO cookie_old; +ALTER TABLE message RENAME TO message_old; +ALTER TABLE portal RENAME TO portal_old; +ALTER TABLE puppet RENAME TO puppet_old; +ALTER TABLE reaction RENAME TO reaction_old; +ALTER TABLE "user" RENAME TO user_old; +ALTER TABLE user_portal RENAME TO user_portal_old; +` + +//go:embed legacymigrate.sql +var legacyMigrateCopyData string + func migrateLegacyConfig(helper up.Helper) { helper.Set(up.Str, "mautrix.bridge.e2ee", "encryption", "pickle_key") } diff --git a/cmd/mautrix-linkedin/legacymigrate.sql b/cmd/mautrix-linkedin/legacymigrate.sql new file mode 100644 index 0000000..cd723fb --- /dev/null +++ b/cmd/mautrix-linkedin/legacymigrate.sql @@ -0,0 +1,166 @@ +INSERT INTO "user" (bridge_id, mxid, management_room) +SELECT '', mxid, notice_room FROM user_old; + +INSERT INTO user_login (bridge_id, user_mxid, id, remote_name, remote_profile, space_room, metadata) +SELECT + '', -- bridge_id + mxid, -- user_mxid + li_member_urn, -- id + li_member_urn, -- remote_name + '{}', -- remote_profile + space_mxid, -- space_room + '{}' -- metadata +FROM user_old; + +INSERT INTO ghost ( + bridge_id, id, name, avatar_id, avatar_hash, avatar_mxc, + name_set, avatar_set, contact_info_set, is_bot, identifiers, metadata +) +SELECT + '', -- bridge_id + li_member_urn, -- id + COALESCE(name, ''), -- name + COALESCE(photo_id, ''), -- avatar_id + '', -- avatar_hash + COALESCE(photo_mxc, ''), -- avatar_mxc + name_set, + avatar_set, + contact_info_set, + false, -- is_bot + '["linkedin:' || li_member_urn || '"]', -- identifiers + '{}' -- metadata +FROM puppet_old; + +INSERT INTO portal ( + bridge_id, id, receiver, mxid, other_user_id, name, topic, avatar_id, avatar_hash, avatar_mxc, + name_set, avatar_set, topic_set, name_is_custom, in_space, room_type, metadata +) +SELECT + '', -- bridge_id + 'urn:li:msg_conversation:(urn:li:fsd_profile:' || li_receiver_urn || ',' || li_thread_urn || ')', -- id + CASE WHEN li_is_group_chat=0 THEN li_receiver_urn ELSE '' END, -- receiver + mxid, -- mxid + CASE WHEN NOT li_is_group_chat THEN li_other_user_urn END, -- other_user_id + COALESCE(name, ''), -- name + COALESCE(topic, ''), -- topic + COALESCE(photo_id, ''), -- avatar_id + '', -- avatar_hash + COALESCE(avatar_url, ''), -- avatar_mxc + name_set, -- name_set + avatar_set, -- avatar_set + topic_set, -- topic_set + li_is_group_chat, -- name_is_custom + false, -- in_space + CASE WHEN li_is_group_chat THEN 'dm' ELSE 'group_dm' END, -- room_type + '{}' -- metadata +FROM portal_old; + +INSERT INTO user_portal (bridge_id, user_mxid, login_id, portal_id, portal_receiver, in_space, preferred) +SELECT + '', -- bridge_id + user_old.mxid, -- user_mxid + user_old.li_member_urn, -- login_id + 'urn:li:msg_conversation:(urn:li:fsd_profile:' || portal_old.li_receiver_urn || ',' || portal_old.li_thread_urn || ')', -- portal_id + CASE WHEN NOT li_is_group_chat THEN li_receiver_urn ELSE '' END, -- portal_receiver + false, -- in_space + false -- preferred +FROM portal_old +JOIN user_old ON user_old.li_member_urn = portal_old.li_receiver_urn; + +INSERT INTO message ( + bridge_id, id, part_id, mxid, room_id, room_receiver, sender_id, sender_mxid, timestamp, edit_count, metadata +) +SELECT + '', -- bridge_id + ( + 'urn:li:msg_message:(urn:li:fsd_profile:' || + li_receiver_urn || + SUBSTR(li_message_urn, INSTR(li_message_urn, ',')) || + ')' + ), -- id + '', -- part_id + mxid, + 'urn:li:msg_conversation:(urn:li:fsd_profile:' || li_receiver_urn || ',' || li_thread_urn || ')', -- room_id + ( + SELECT CASE WHEN li_is_group_chat=0 THEN li_receiver_urn ELSE '' END + FROM portal_old + WHERE li_thread_urn=message_old.li_thread_urn + ), -- room_receiver + li_sender_urn, -- sender_id + '', -- sender_mxid + timestamp * 1000000, -- timestamp + 0, -- edit_count + '{}' -- metadata +FROM message_old +WHERE true +ON CONFLICT DO NOTHING; + +INSERT INTO reaction ( + bridge_id, message_id, message_part_id, sender_id, emoji_id, room_id, room_receiver, mxid, timestamp, emoji, metadata +) +SELECT + '', -- bridge_id + ( + 'urn:li:msg_message:(urn:li:fsd_profile:' || + li_receiver_urn || + SUBSTR(li_message_urn, INSTR(li_message_urn, ',')) || + ')' + ), -- message_id + '', -- message_part_id + li_sender_urn, -- sender_id + reaction, -- emoji_id + 'urn:li:msg_conversation:(urn:li:fsd_profile:' || li_receiver_urn || ',' || SUBSTR(li_message_urn, 0, INSTR(li_message_urn, ',')) || ')', -- room_id + ( + SELECT CASE WHEN li_is_group_chat=0 THEN li_receiver_urn ELSE '' END + FROM portal_old + WHERE li_thread_urn=SUBSTR(li_message_urn, 0, INSTR(li_message_urn, ',')) + ), -- room_receiver + mxid, + ( + SELECT (timestamp * 1000000) + 1 + FROM message_old + WHERE message_old.li_message_urn=reaction_old.li_message_urn + AND "index"=0 + AND message_old.li_receiver_urn=reaction_old.li_receiver_urn + ), -- timestamp + reaction, -- emoji + '{}' -- metadata +FROM reaction_old; + +CREATE TABLE IF NOT EXISTS database_owner ( + key INTEGER PRIMARY KEY DEFAULT 0, + owner TEXT NOT NULL +); +INSERT INTO database_owner (key, owner) VALUES (0, "megabridge/mautrix-linkedin"); + +-- Python -> Go mx_ table migration +ALTER TABLE mx_room_state DROP COLUMN is_encrypted; +ALTER TABLE mx_room_state RENAME COLUMN has_full_member_list TO members_fetched; +UPDATE mx_room_state SET members_fetched=false WHERE members_fetched IS NULL; + +-- only: postgres until "end only" +ALTER TABLE mx_room_state ALTER COLUMN power_levels TYPE jsonb USING power_levels::jsonb; +ALTER TABLE mx_room_state ALTER COLUMN encryption TYPE jsonb USING encryption::jsonb; +ALTER TABLE mx_room_state ALTER COLUMN members_fetched SET DEFAULT false; +ALTER TABLE mx_room_state ALTER COLUMN members_fetched SET NOT NULL; +-- end only postgres + +ALTER TABLE mx_user_profile ADD COLUMN name_skeleton bytea; +CREATE INDEX mx_user_profile_membership_idx ON mx_user_profile (room_id, membership); +CREATE INDEX mx_user_profile_name_skeleton_idx ON mx_user_profile (room_id, name_skeleton); + +UPDATE mx_user_profile SET displayname='' WHERE displayname IS NULL; +UPDATE mx_user_profile SET avatar_url='' WHERE avatar_url IS NULL; + +CREATE TABLE mx_registrations ( + user_id TEXT PRIMARY KEY +); + +UPDATE mx_version SET version=7; + +DROP TABLE message_old; +DROP TABLE puppet_old; +DROP TABLE reaction_old; +DROP TABLE user_portal_old; +DROP TABLE user_old; +DROP TABLE portal_old; diff --git a/cmd/mautrix-linkedin/main.go b/cmd/mautrix-linkedin/main.go index 1a4b778..085c133 100644 --- a/cmd/mautrix-linkedin/main.go +++ b/cmd/mautrix-linkedin/main.go @@ -17,12 +17,16 @@ package main import ( + "context" "net/http" "maunium.net/go/mautrix/bridgev2/bridgeconfig" "maunium.net/go/mautrix/bridgev2/matrix/mxmain" + "go.mau.fi/util/dbutil" + "go.mau.fi/mautrix-linkedin/pkg/connector" + "go.mau.fi/mautrix-linkedin/pkg/linkedingo" ) // Information to find out exactly which commit the bridge was built from. @@ -49,7 +53,74 @@ func main() { m.Matrix.Provisioning.Router.HandleFunc("/v1/api/logout", legacyProvLogout).Methods(http.MethodPost) } } + m.PostInit = func() { + log := m.Log.With().Str("component", "database_migrator").Logger() + ctx := log.WithContext(context.TODO()) + + // The old bridge does not have a database_owner table, so use that to + // detect if the migration has already happened. + exists, err := m.DB.TableExists(ctx, "database_owner") + if err != nil { + log.Err(err).Msg("Failed to check if database_owner table exists") + return + } else if exists { + log.Debug().Msg("Database owner table exists, assuming database is already migrated") + return + } + expectedVersion := 10 + var dbVersion int + err = m.DB.QueryRow(ctx, "SELECT version FROM version").Scan(&dbVersion) + if err != nil { + log.Fatal().Err(err).Msg("Failed to get database version") + } else if dbVersion < expectedVersion { + log.Fatal(). + Int("expected_version", expectedVersion). + Int("version", dbVersion). + Msg("Unsupported database version. Please upgrade to beeper/linkedin v0.5.4 or higher before upgrading to v0.6.0.") + return + } else if dbVersion > expectedVersion { + log.Fatal(). + Int("expected_version", expectedVersion). + Int("version", dbVersion). + Msg("Unsupported database version (higher than expected)") + return + } + log.Info().Msg("Detected legacy database, migrating...") + err = m.DB.DoTxn(ctx, nil, func(ctx context.Context) error { + if err := m.LegacyMigrateSimple(legacyMigrateRenameTables, legacyMigrateCopyData, 16)(ctx); err != nil { + return err + } + rows, err := m.DB.Query(ctx, "SELECT mxid, name, value FROM cookie_old") + if err != nil { + return err + } + cookies := map[string]*linkedingo.StringCookieJar{} + for rows.Next() { + var mxid, name, value string + if err := rows.Scan(&mxid, &name, &value); err != nil { + return err + } + if _, ok := cookies[mxid]; !ok { + cookies[mxid] = linkedingo.NewEmptyStringCookieJar() + } + cookies[mxid].AddCookie(&http.Cookie{Name: name, Value: value}) + } + for mxid, jar := range cookies { + metadata := connector.UserLoginMetadata{Cookies: jar} + if _, err := m.DB.Exec(ctx, "UPDATE user_login SET metadata = $1 WHERE user_mxid = $2", dbutil.JSON{Data: metadata}, mxid); err != nil { + return err + } + } + _, err = m.DB.Exec(ctx, "DROP TABLE cookie_old;") + return err + }) + if err != nil { + m.LogDBUpgradeErrorAndExit("main", err, "Failed to migrate legacy database") + } else { + log.Info().Msg("Successfully migrated legacy database") + } + } m.InitVersion(Tag, Commit, BuildTime) m.Run() } diff --git a/pkg/connector/client.go b/pkg/connector/client.go index 7e60f1c..3ac6d26 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -206,7 +206,7 @@ func (l *LinkedInClient) onRealtimeMessage(ctx context.Context, msg linkedingo.M Stringer("entity_urn", msg.EntityURN). Stringer("sender", msg.Sender.EntityURN) }, - PortalKey: l.makePortalKey(msg.Conversation.EntityURN), + PortalKey: l.makePortalKey(msg.Conversation), CreatePortal: true, Sender: l.makeSender(msg.Sender), Timestamp: msg.DeliveredAt.Time, @@ -255,7 +255,7 @@ func (l *LinkedInClient) onRealtimeTypingIndicator(decoratedEvent *linkedingo.De Stringer("conversation_urn", typingIndicator.Conversation.EntityURN). Stringer("typing_participant_urn", typingIndicator.TypingParticipant.EntityURN) }, - PortalKey: l.makePortalKey(typingIndicator.Conversation.EntityURN), + PortalKey: l.makePortalKey(typingIndicator.Conversation), Sender: l.makeSender(typingIndicator.TypingParticipant), Timestamp: decoratedEvent.LeftServerAt.Time, } diff --git a/pkg/connector/dbmeta.go b/pkg/connector/dbmeta.go index 30d7bca..0fd760b 100644 --- a/pkg/connector/dbmeta.go +++ b/pkg/connector/dbmeta.go @@ -33,5 +33,5 @@ func (lc *LinkedInConnector) GetDBMetaTypes() database.MetaTypes { } type UserLoginMetadata struct { - Cookies *linkedingo.Jar `json:"cookies,omitempty"` + Cookies *linkedingo.StringCookieJar `json:"cookies,omitempty"` } diff --git a/pkg/connector/ids.go b/pkg/connector/ids.go index 3e8f107..9d0cd86 100644 --- a/pkg/connector/ids.go +++ b/pkg/connector/ids.go @@ -7,9 +7,9 @@ import ( "go.mau.fi/mautrix-linkedin/pkg/linkedingo" ) -func (l *LinkedInClient) makePortalKey(entityURN linkedingo.URN) (key networkid.PortalKey) { - key.ID = networkid.PortalID(entityURN.String()) - if l.main.Bridge.Config.SplitPortals { +func (l *LinkedInClient) makePortalKey(conv linkedingo.Conversation) (key networkid.PortalKey) { + key.ID = networkid.PortalID(conv.EntityURN.String()) + if !conv.GroupChat || l.main.Bridge.Config.SplitPortals { key.Receiver = l.userLogin.ID } return key diff --git a/pkg/connector/sync.go b/pkg/connector/sync.go index a527907..04c68fd 100644 --- a/pkg/connector/sync.go +++ b/pkg/connector/sync.go @@ -49,7 +49,7 @@ func (l *LinkedInClient) syncConversations(ctx context.Context) { updatedBefore = conv.LastActivityAt.Time } - portalKey := l.makePortalKey(conv.EntityURN) + portalKey := l.makePortalKey(conv) portal, err := l.main.Bridge.GetPortalByKey(ctx, portalKey) if err != nil { log.Err(err).Msg("Failed to get portal") diff --git a/pkg/linkedingo/client.go b/pkg/linkedingo/client.go index ec82a13..f23a83d 100644 --- a/pkg/linkedingo/client.go +++ b/pkg/linkedingo/client.go @@ -25,7 +25,7 @@ import ( type Client struct { http *http.Client - jar *Jar + jar *StringCookieJar userEntityURN URN realtimeSessionID uuid.UUID @@ -41,7 +41,7 @@ type Client struct { i18nLocale string } -func NewClient(ctx context.Context, userEntityURN URN, jar *Jar, handlers Handlers) *Client { +func NewClient(ctx context.Context, userEntityURN URN, jar *StringCookieJar, handlers Handlers) *Client { return &Client{ userEntityURN: userEntityURN, jar: jar, diff --git a/pkg/linkedingo/request.go b/pkg/linkedingo/request.go index 2e4045d..39cde05 100644 --- a/pkg/linkedingo/request.go +++ b/pkg/linkedingo/request.go @@ -46,7 +46,7 @@ func (c *Client) newAuthedRequest(method, urlStr string) *authedRequest { ar.header.Add("sec-ch-prefers-color-scheme", "light") ar.header.Add("sec-ch-ua", `"Chromium";v="131", "Not_A Brand";v="24"`) ar.header.Add("sec-ch-ua-mobile", "?0") - ar.header.Add("sec-ch-ua-platform", `"Linux"`) + ar.header.Add("sec-ch-ua-platform", `"macOS"`) return &ar } diff --git a/pkg/linkedingo/stringcookiejar.go b/pkg/linkedingo/stringcookiejar.go index 37dcf28..d1baedd 100644 --- a/pkg/linkedingo/stringcookiejar.go +++ b/pkg/linkedingo/stringcookiejar.go @@ -25,53 +25,58 @@ import ( "slices" ) -// Jar is an [http.CookieJar] implementation that is backed by a dictionary of -// name -> [http.Cookie] values. It also implements [json.Marshaler] and -// [json.Unmarshaler] which allow it to be saved as a string. +// StringCookieJar is an [http.CookieJar] implementation that is backed by a +// dictionary of name -> [http.Cookie] values. It also implements +// [json.Marshaler] and [json.Unmarshaler] which allow it to be saved as a +// string. // -// The zero value is not a valid [Jar]. Use [NewEmptyJar] to create a new -// [Jar]. -type Jar struct { +// The zero value is not a valid [StringCookieJar]. Use [NewEmptyStringCookieJar] to create +// a new [StringCookieJar]. +type StringCookieJar struct { cookies map[string]*http.Cookie } -var _ http.CookieJar = (*Jar)(nil) -var _ json.Marshaler = (*Jar)(nil) -var _ json.Unmarshaler = (*Jar)(nil) +var _ http.CookieJar = (*StringCookieJar)(nil) +var _ json.Marshaler = (*StringCookieJar)(nil) +var _ json.Unmarshaler = (*StringCookieJar)(nil) -// NewEmptyJar creates an empty [Jar]. -func NewEmptyJar() *Jar { - return &Jar{ +// NewEmptyStringCookieJar creates an empty [StringCookieJar]. +func NewEmptyStringCookieJar() *StringCookieJar { + return &StringCookieJar{ cookies: map[string]*http.Cookie{}, } } -// NewJarFromCookieHeader creates a [Jar] from a cookie header string. It +// NewJarFromCookieHeader creates a [StringCookieJar] from a cookie header string. It // errors if parsing the cookie header fails. -func NewJarFromCookieHeader(cookieHeader string) (*Jar, error) { +func NewJarFromCookieHeader(cookieHeader string) (*StringCookieJar, error) { cookies, err := parseCookieHeaderString(cookieHeader) - return &Jar{cookies: cookies}, err + return &StringCookieJar{cookies: cookies}, err } -func (s *Jar) Cookies(u *url.URL) []*http.Cookie { +func (s *StringCookieJar) Cookies(u *url.URL) []*http.Cookie { return slices.Collect(maps.Values(s.cookies)) } -func (s *Jar) SetCookies(u *url.URL, cookies []*http.Cookie) { +func (s *StringCookieJar) SetCookies(u *url.URL, cookies []*http.Cookie) { s.cookies = map[string]*http.Cookie{} for _, c := range cookies { s.cookies[c.Name] = c } } -func (s *Jar) GetCookie(name string) (value string) { +func (s *StringCookieJar) AddCookie(cookie *http.Cookie) { + s.cookies[cookie.Name] = cookie +} + +func (s *StringCookieJar) GetCookie(name string) (value string) { if c, ok := s.cookies[name]; ok { value = c.Value } return } -func (s *Jar) UnmarshalJSON(data []byte) (err error) { +func (s *StringCookieJar) UnmarshalJSON(data []byte) (err error) { var cookieHeader string err = json.Unmarshal(data, &cookieHeader) if err != nil { @@ -81,7 +86,7 @@ func (s *Jar) UnmarshalJSON(data []byte) (err error) { return } -func (s *Jar) MarshalJSON() ([]byte, error) { +func (s *StringCookieJar) MarshalJSON() ([]byte, error) { req, err := http.NewRequest(http.MethodGet, "", nil) if err != nil { return nil, fmt.Errorf("failed to create request") diff --git a/pkg/linkedingo/stringcookiejar_test.go b/pkg/linkedingo/stringcookiejar_test.go index ec4c70d..8463668 100644 --- a/pkg/linkedingo/stringcookiejar_test.go +++ b/pkg/linkedingo/stringcookiejar_test.go @@ -44,7 +44,7 @@ func TestCookieJarFromHeader(t *testing.T) { } func TestCookieJarSetCookies(t *testing.T) { - jar := linkedingo.NewEmptyJar() + jar := linkedingo.NewEmptyStringCookieJar() assert.Len(t, jar.Cookies(nil), 0) @@ -88,7 +88,7 @@ func TestCookieJarSetCookies(t *testing.T) { } func TestMarshal(t *testing.T) { - jar := linkedingo.NewEmptyJar() + jar := linkedingo.NewEmptyStringCookieJar() jar.SetCookies(nil, []*http.Cookie{ {Name: "123", Value: "this is a test with spaces"}, {Name: "234", Value: "I'm a value with a quote"}, @@ -105,7 +105,7 @@ func TestMarshal(t *testing.T) { } type container struct { - Cookies *linkedingo.Jar `json:"cookies,omitempty"` + Cookies *linkedingo.StringCookieJar `json:"cookies,omitempty"` } func TestUnmarshal(t *testing.T) { -- GitLab