From c728541c24836c1490189a8b125c8e194264f998 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 18:52:14 +0200 Subject: [PATCH 01/19] Run CI lint on Go 1.21 --- .github/workflows/go.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 0fdb755..3d392af 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -8,7 +8,8 @@ jobs: strategy: fail-fast: false matrix: - go-version: ["1.21"] + go-version: ["1.21", "1.22"] + name: Lint ${{ matrix.go-version == '1.22' && '(latest)' || '(old)' }} steps: - uses: actions/checkout@v4 From 43b6398744592dde83405ab5230cb2d822133a00 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 1 Feb 2024 14:00:48 +0100 Subject: [PATCH 02/19] Start adding Messenger-side encryption using WhatsApp protocol --- database/portal.go | 53 ++++-- database/puppet.go | 23 ++- database/upgrades/00-latest.sql | 12 +- database/upgrades/04-wa-device-id.sql | 2 + database/upgrades/05-wa-server-name.sql | 3 + database/user.go | 16 +- go.mod | 24 ++- go.sum | 52 +++--- main.go | 9 + messagix/client.go | 5 + messagix/e2ee-client.go | 77 ++++++++ messagix/e2ee-register.go | 232 ++++++++++++++++++++++++ messagix/types/account.go | 4 +- user.go | 123 +++++++++++++ 14 files changed, 575 insertions(+), 60 deletions(-) create mode 100644 database/upgrades/04-wa-device-id.sql create mode 100644 database/upgrades/05-wa-server-name.sql create mode 100644 messagix/e2ee-client.go create mode 100644 messagix/e2ee-register.go diff --git a/database/portal.go b/database/portal.go index 3906753..110cd1a 100644 --- a/database/portal.go +++ b/database/portal.go @@ -19,8 +19,10 @@ package database import ( "context" "database/sql" + "strconv" "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" "maunium.net/go/mautrix/id" "go.mau.fi/mautrix-meta/messagix/table" @@ -30,7 +32,8 @@ const ( portalBaseSelect = ` SELECT thread_id, receiver, thread_type, mxid, name, avatar_id, avatar_url, name_set, avatar_set, - encrypted, relay_user_id, oldest_message_id, oldest_message_ts, more_to_backfill + whatsapp_server, encrypted, relay_user_id, + oldest_message_id, oldest_message_ts, more_to_backfill FROM portal ` getPortalByMXIDQuery = portalBaseSelect + `WHERE mxid=$1` @@ -47,14 +50,16 @@ const ( INSERT INTO portal ( thread_id, receiver, thread_type, mxid, name, avatar_id, avatar_url, name_set, avatar_set, - encrypted, relay_user_id, oldest_message_id, oldest_message_ts, more_to_backfill - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + whatsapp_server, encrypted, relay_user_id, + oldest_message_id, oldest_message_ts, more_to_backfill + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) ` updatePortalQuery = ` UPDATE portal SET thread_type=$3, mxid=$4, name=$5, avatar_id=$6, avatar_url=$7, name_set=$8, avatar_set=$9, - encrypted=$10, relay_user_id=$11, oldest_message_id=$12, oldest_message_ts=$13, more_to_backfill=$14 + whatsapp_server=$10, encrypted=$11, relay_user_id=$12, + oldest_message_id=$13, oldest_message_ts=$14, more_to_backfill=$15 WHERE thread_id=$1 AND receiver=$2 ` deletePortalQuery = `DELETE FROM portal WHERE thread_id=$1 AND receiver=$2` @@ -73,15 +78,17 @@ type Portal struct { qh *dbutil.QueryHelper[*Portal] PortalKey - ThreadType table.ThreadType - MXID id.RoomID - Name string - AvatarID string - AvatarURL id.ContentURI - NameSet bool - AvatarSet bool - Encrypted bool - RelayUserID id.UserID + ThreadType table.ThreadType + MXID id.RoomID + Name string + AvatarID string + AvatarURL id.ContentURI + NameSet bool + AvatarSet bool + + WhatsAppServer string + Encrypted bool + RelayUserID id.UserID OldestMessageID string OldestMessageTS int64 @@ -128,6 +135,24 @@ func (p *Portal) IsPrivateChat() bool { return p.ThreadType.IsOneToOne() } +func (p *Portal) JID() types.JID { + jid := types.JID{ + User: strconv.FormatInt(p.ThreadID, 10), + Server: p.WhatsAppServer, + } + if jid.Server == "" { + switch p.ThreadType { + case table.ENCRYPTED_OVER_WA_GROUP: + jid.Server = types.GroupServer + //case table.ENCRYPTED_OVER_WA_ONE_TO_ONE: + // jid.Server = types.DefaultUserServer + default: + jid.Server = types.MessengerServer + } + } + return jid +} + func (p *Portal) Scan(row dbutil.Scannable) (*Portal, error) { var mxid sql.NullString err := row.Scan( @@ -140,6 +165,7 @@ func (p *Portal) Scan(row dbutil.Scannable) (*Portal, error) { &p.AvatarURL, &p.NameSet, &p.AvatarSet, + &p.WhatsAppServer, &p.Encrypted, &p.RelayUserID, &p.OldestMessageID, @@ -164,6 +190,7 @@ func (p *Portal) sqlVariables() []any { &p.AvatarURL, p.NameSet, p.AvatarSet, + p.WhatsAppServer, p.Encrypted, p.RelayUserID, p.OldestMessageID, diff --git a/database/puppet.go b/database/puppet.go index fecf719..a4bdd77 100644 --- a/database/puppet.go +++ b/database/puppet.go @@ -21,13 +21,14 @@ import ( "database/sql" "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" "maunium.net/go/mautrix/id" ) const ( puppetBaseSelect = ` SELECT id, name, username, avatar_id, avatar_url, name_set, avatar_set, - contact_info_set, custom_mxid, access_token + contact_info_set, whatsapp_server, custom_mxid, access_token FROM puppet ` getPuppetByMetaIDQuery = puppetBaseSelect + `WHERE id=$1` @@ -36,15 +37,15 @@ const ( updatePuppetQuery = ` UPDATE puppet SET name=$2, username=$3, avatar_id=$4, avatar_url=$5, name_set=$6, avatar_set=$7, - contact_info_set=$8, custom_mxid=$9, access_token=$10 + contact_info_set=$8, whatsapp_server=$9, custom_mxid=$10, access_token=$11 WHERE id=$1 ` insertPuppetQuery = ` INSERT INTO puppet ( id, name, username, avatar_id, avatar_url, name_set, avatar_set, - contact_info_set, custom_mxid, access_token + contact_info_set, whatsapp_server, custom_mxid, access_token ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ` ) @@ -64,6 +65,7 @@ type Puppet struct { AvatarSet bool ContactInfoSet bool + WhatsAppServer string CustomMXID id.UserID AccessToken string @@ -96,6 +98,7 @@ func (p *Puppet) Scan(row dbutil.Scannable) (*Puppet, error) { &p.NameSet, &p.AvatarSet, &p.ContactInfoSet, + &p.WhatsAppServer, &customMXID, &p.AccessToken, ) @@ -106,6 +109,17 @@ func (p *Puppet) Scan(row dbutil.Scannable) (*Puppet, error) { return p, nil } +func (p *Puppet) JID() types.JID { + jid := types.JID{ + User: p.Username, + Server: p.WhatsAppServer, + } + if jid.Server == "" { + jid.Server = types.MessengerServer + } + return jid +} + func (p *Puppet) sqlVariables() []any { return []any{ p.ID, @@ -116,6 +130,7 @@ func (p *Puppet) sqlVariables() []any { p.NameSet, p.AvatarSet, p.ContactInfoSet, + p.WhatsAppServer, dbutil.StrPtr(p.CustomMXID), p.AccessToken, } diff --git a/database/upgrades/00-latest.sql b/database/upgrades/00-latest.sql index 4fa6057..405e43b 100644 --- a/database/upgrades/00-latest.sql +++ b/database/upgrades/00-latest.sql @@ -1,4 +1,4 @@ --- v0 -> v3: Latest revision +-- v0 -> v5 (compatible with v3+): Latest revision CREATE TABLE portal ( thread_id BIGINT NOT NULL, @@ -12,6 +12,8 @@ CREATE TABLE portal ( name_set BOOLEAN NOT NULL DEFAULT false, avatar_set BOOLEAN NOT NULL DEFAULT false, + whatsapp_server TEXT NOT NULL DEFAULT '', + encrypted BOOLEAN NOT NULL DEFAULT false, relay_user_id TEXT NOT NULL, @@ -33,6 +35,7 @@ CREATE TABLE puppet ( avatar_set BOOLEAN NOT NULL DEFAULT false, contact_info_set BOOLEAN NOT NULL DEFAULT false, + whatsapp_server TEXT NOT NULL DEFAULT '', custom_mxid TEXT, access_token TEXT NOT NULL, @@ -41,9 +44,10 @@ CREATE TABLE puppet ( ); CREATE TABLE "user" ( - mxid TEXT NOT NULL PRIMARY KEY, - meta_id BIGINT, - cookies jsonb, + mxid TEXT NOT NULL PRIMARY KEY, + meta_id BIGINT, + wa_device_id INTEGER, + cookies jsonb, inbox_fetched BOOLEAN NOT NULL, diff --git a/database/upgrades/04-wa-device-id.sql b/database/upgrades/04-wa-device-id.sql new file mode 100644 index 0000000..1e064d6 --- /dev/null +++ b/database/upgrades/04-wa-device-id.sql @@ -0,0 +1,2 @@ +-- v4 (compatible with v3+): Store WhatsApp device ID for users +ALTER TABLE "user" ADD COLUMN wa_device_id INTEGER; diff --git a/database/upgrades/05-wa-server-name.sql b/database/upgrades/05-wa-server-name.sql new file mode 100644 index 0000000..523e5ae --- /dev/null +++ b/database/upgrades/05-wa-server-name.sql @@ -0,0 +1,3 @@ +-- v5 (compatible with v3+): Store WhatsApp server name for portals and puppets +ALTER TABLE portal ADD COLUMN whatsapp_server TEXT NOT NULL DEFAULT ''; +ALTER TABLE puppet ADD COLUMN whatsapp_server TEXT NOT NULL DEFAULT ''; diff --git a/database/user.go b/database/user.go index 08bb88c..0dc9db8 100644 --- a/database/user.go +++ b/database/user.go @@ -28,11 +28,11 @@ import ( ) const ( - getUserByMXIDQuery = `SELECT mxid, meta_id, cookies, inbox_fetched, management_room, space_room FROM "user" WHERE mxid=$1` - getUserByMetaIDQuery = `SELECT mxid, meta_id, cookies, inbox_fetched, management_room, space_room FROM "user" WHERE meta_id=$1` - getAllLoggedInUsersQuery = `SELECT mxid, meta_id, cookies, inbox_fetched, management_room, space_room FROM "user" WHERE cookies IS NOT NULL` - insertUserQuery = `INSERT INTO "user" (mxid, meta_id, cookies, inbox_fetched, management_room, space_room) VALUES ($1, $2, $3, $4, $5, $6)` - updateUserQuery = `UPDATE "user" SET meta_id=$2, cookies=$3, inbox_fetched=$4, management_room=$5, space_room=$6 WHERE mxid=$1` + getUserByMXIDQuery = `SELECT mxid, meta_id, wa_device_id, cookies, inbox_fetched, management_room, space_room FROM "user" WHERE mxid=$1` + getUserByMetaIDQuery = `SELECT mxid, meta_id, wa_device_id, cookies, inbox_fetched, management_room, space_room FROM "user" WHERE meta_id=$1` + getAllLoggedInUsersQuery = `SELECT mxid, meta_id, wa_device_id, cookies, inbox_fetched, management_room, space_room FROM "user" WHERE cookies IS NOT NULL` + insertUserQuery = `INSERT INTO "user" (mxid, meta_id, wa_device_id, cookies, inbox_fetched, management_room, space_room) VALUES ($1, $2, $3, $4, $5, $6, $7)` + updateUserQuery = `UPDATE "user" SET meta_id=$2, wa_device_id=$3, cookies=$4, inbox_fetched=$5, management_room=$6, space_room=$7 WHERE mxid=$1` ) type UserQuery struct { @@ -44,6 +44,7 @@ type User struct { MXID id.UserID MetaID int64 + WADeviceID uint16 Cookies cookies.Cookies InboxFetched bool ManagementRoom id.RoomID @@ -74,7 +75,7 @@ func (uq *UserQuery) GetAllLoggedIn(ctx context.Context) ([]*User, error) { } func (u *User) sqlVariables() []any { - return []any{u.MXID, dbutil.NumPtr(u.MetaID), dbutil.JSON{Data: u.Cookies}, u.InboxFetched, dbutil.StrPtr(u.ManagementRoom), dbutil.StrPtr(u.SpaceRoom)} + return []any{u.MXID, dbutil.NumPtr(u.MetaID), u.WADeviceID, dbutil.JSON{Data: u.Cookies}, u.InboxFetched, dbutil.StrPtr(u.ManagementRoom), dbutil.StrPtr(u.SpaceRoom)} } func (u *User) Insert(ctx context.Context) error { @@ -90,10 +91,12 @@ var NewCookies func() cookies.Cookies func (u *User) Scan(row dbutil.Scannable) (*User, error) { var managementRoom, spaceRoom sql.NullString var metaID sql.NullInt64 + var waDeviceID sql.NullInt32 scannedCookies := NewCookies() err := row.Scan( &u.MXID, &metaID, + &waDeviceID, &dbutil.JSON{Data: scannedCookies}, &u.InboxFetched, &managementRoom, @@ -106,6 +109,7 @@ func (u *User) Scan(row dbutil.Scannable) (*User, error) { u.Cookies = scannedCookies } u.MetaID = metaID.Int64 + u.WADeviceID = uint16(waDeviceID.Int32) u.ManagementRoom = id.RoomID(managementRoom.String) u.SpaceRoom = id.RoomID(spaceRoom.String) return u, nil diff --git a/go.mod b/go.mod index ce4a33a..744fa3b 100644 --- a/go.mod +++ b/go.mod @@ -5,23 +5,27 @@ go 1.21 require ( github.com/beeper/libserv v0.0.0-20231231202820-c7303abfc32c github.com/google/go-querystring v1.1.0 - github.com/google/uuid v1.5.0 + github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.0 github.com/lib/pq v1.10.9 github.com/mattn/go-colorable v0.1.13 - github.com/mattn/go-sqlite3 v1.14.19 - github.com/rs/zerolog v1.31.0 + github.com/mattn/go-sqlite3 v1.14.22 + github.com/rs/zerolog v1.32.0 github.com/tidwall/gjson v1.17.0 github.com/zyedidia/clipboard v1.0.4 - go.mau.fi/util v0.3.0 - golang.org/x/crypto v0.18.0 - golang.org/x/exp v0.0.0-20240119083558-1b970713d09a + go.mau.fi/libsignal v0.1.0 + go.mau.fi/util v0.3.1-0.20240209114727-da0b16df0446 + go.mau.fi/whatsmeow v0.0.0-20240209184912-c3e911dd6cfe + golang.org/x/crypto v0.19.0 + golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 golang.org/x/image v0.15.0 - golang.org/x/net v0.20.0 - maunium.net/go/mautrix v0.17.1-0.20240119201531-97d19484a396 + golang.org/x/net v0.21.0 + google.golang.org/protobuf v1.32.0 + maunium.net/go/mautrix v0.17.1-0.20240209172009-b369efbc06b2 ) require ( + filippo.io/edwards25519 v1.0.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/mattn/go-isatty v0.0.19 // indirect @@ -29,9 +33,9 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.5 // indirect - github.com/yuin/goldmark v1.6.0 // indirect + github.com/yuin/goldmark v1.7.0 // indirect go.mau.fi/zeroconfig v0.1.2 // indirect - golang.org/x/sys v0.16.0 // indirect + golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 2df3326..404aefb 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ -github.com/DATA-DOG/go-sqlmock v1.5.1 h1:FK6RCIUSfmbnI/imIICmboyQBkOckutaa6R5YYlLZyo= -github.com/DATA-DOG/go-sqlmock v1.5.1/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek= +filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/beeper/libserv v0.0.0-20231231202820-c7303abfc32c h1:WqjRVgUO039eiISCjsZC4F9onOEV93DJAk6v33rsZzY= github.com/beeper/libserv v0.0.0-20231231202820-c7303abfc32c/go.mod h1:b9FFm9y4mEm36G8ytVmS1vkNzJa0KepmcdVY+qf7qRU= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= @@ -12,8 +14,8 @@ github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -25,15 +27,15 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= -github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= -github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= +github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -45,30 +47,36 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/yuin/goldmark v1.6.0 h1:boZcn2GTjpsynOsC0iJHnBWa4Bi0qzfJjthwauItG68= -github.com/yuin/goldmark v1.6.0/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.0 h1:EfOIvIMZIzHdB/R/zVrikYLPPwJlfMcNczJFMs1m6sA= +github.com/yuin/goldmark v1.7.0/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= github.com/zyedidia/clipboard v1.0.4 h1:r6GUQOyPtIaApRLeD56/U+2uJbXis6ANGbKWCljULEo= github.com/zyedidia/clipboard v1.0.4/go.mod h1:zykFnZUXX0ErxqvYLUFEq7QDJKId8rmh2FgD0/Y8cjA= -go.mau.fi/util v0.3.0 h1:Lt3lbRXP6ZBqTINK0EieRWor3zEwwwrDT14Z5N8RUCs= -go.mau.fi/util v0.3.0/go.mod h1:9dGsBCCbZJstx16YgnVMVi3O2bOizELoKpugLD4FoGs= +go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= +go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= +go.mau.fi/util v0.3.1-0.20240209114727-da0b16df0446 h1:goGC5wdUKIAdD/rMcA282viTTwUzvLwI/94cxYaKAjI= +go.mau.fi/util v0.3.1-0.20240209114727-da0b16df0446/go.mod h1:rRypwgXVEPILomtFPyQcnbOeuRqf+nRN84vh/CICq4w= +go.mau.fi/whatsmeow v0.0.0-20240209184912-c3e911dd6cfe h1:ZosSMvW46/bX5tEdXXgF+weMIPj+r8QG5ry7JSdDWt8= +go.mau.fi/whatsmeow v0.0.0-20240209184912-c3e911dd6cfe/go.mod h1:lQHbhaG/fI+6hfGqz5Vzn2OBJBEZ05H0kCP6iJXriN4= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 h1:/RIbNt/Zr7rVhIkQhooTxCxFcdWLGIKnZA4IXNFSrvo= +golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= @@ -79,5 +87,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.17.1-0.20240119201531-97d19484a396 h1:n6GsfAoI12vq1wJCExPGNG7Gr5LmzfVj3YnVtqS+F1M= -maunium.net/go/mautrix v0.17.1-0.20240119201531-97d19484a396/go.mod h1:j+puTEQCEydlVxhJ/dQP5chfa26TdvBO7X6F3Ataav8= +maunium.net/go/mautrix v0.17.1-0.20240209172009-b369efbc06b2 h1:MN615kW9QLcvZBv8g1ZU2wZ/hfdBtZ2akJD2gc8tawo= +maunium.net/go/mautrix v0.17.1-0.20240209172009-b369efbc06b2/go.mod h1:tMIBWuMXrtjXAqMtaD1VHiT0B3TCxraYlqtncLIyKF0= diff --git a/main.go b/main.go index 7c23a8f..093ddd2 100644 --- a/main.go +++ b/main.go @@ -38,6 +38,8 @@ import ( "go.mau.fi/mautrix-meta/messagix/table" "go.mau.fi/mautrix-meta/messagix/types" "go.mau.fi/mautrix-meta/msgconv" + "go.mau.fi/whatsmeow/store/sqlstore" + waLog "go.mau.fi/whatsmeow/util/log" ) //go:embed example-config.yaml @@ -57,6 +59,8 @@ type MetaBridge struct { Config *config.Config DB *database.Database + DeviceStore *sqlstore.Container + provisioning *ProvisioningAPI usersByMXID map[id.UserID]*User @@ -140,6 +144,7 @@ func (br *MetaBridge) Init() { br.RegisterCommands() br.DB = database.New(br.Bridge.DB) + br.DeviceStore = sqlstore.NewWithDB(br.DB.RawDB, br.DB.Dialect.String(), waLog.Stdout("DATABASE", "DEBUG", true)) ss := br.Config.Bridge.Provisioning.SharedSecret if len(ss) > 0 && ss != "disable" { @@ -148,6 +153,10 @@ func (br *MetaBridge) Init() { } func (br *MetaBridge) Start() { + err := br.DeviceStore.Upgrade() + if err != nil { + br.ZLog.Fatal().Err(err).Msg("Failed to upgrade whatsmeow device store") + } if br.provisioning != nil { br.ZLog.Debug().Msg("Initializing provisioning API") br.provisioning.Init() diff --git a/messagix/client.go b/messagix/client.go index 7eac4fa..a5f6946 100644 --- a/messagix/client.go +++ b/messagix/client.go @@ -18,6 +18,8 @@ import ( "github.com/google/go-querystring/query" "github.com/rs/zerolog" + "go.mau.fi/whatsmeow" + "go.mau.fi/whatsmeow/store" "golang.org/x/net/proxy" "go.mau.fi/mautrix-meta/messagix/cookies" @@ -58,6 +60,9 @@ type Client struct { socksProxy proxy.Dialer GetNewProxy func(reason string) (string, error) + device *store.Device + e2eeClient *whatsmeow.Client + lsRequests int graphQLRequests int platform types.Platform diff --git a/messagix/e2ee-client.go b/messagix/e2ee-client.go new file mode 100644 index 0000000..cb13c4d --- /dev/null +++ b/messagix/e2ee-client.go @@ -0,0 +1,77 @@ +// mautrix-meta - A Matrix-Facebook Messenger and Instagram DM puppeting bridge. +// Copyright (C) 2024 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package messagix + +import ( + "strconv" + + "google.golang.org/protobuf/proto" + + "go.mau.fi/whatsmeow" + waProto "go.mau.fi/whatsmeow/binary/proto" + waLog "go.mau.fi/whatsmeow/util/log" +) + +func (c *Client) PrepareE2EEClient() *whatsmeow.Client { + if c.device == nil { + panic("PrepareE2EEClient called without device") + } + e2eeClient := whatsmeow.NewClient(c.device, waLog.Zerolog(c.Logger.With().Str("component", "whatsmeow").Logger())) + e2eeClient.GetClientPayload = c.getClientPayload + e2eeClient.MessengerConfig = &whatsmeow.MessengerConfig{ + UserAgent: UserAgent, + BaseURL: c.getEndpoint("base_url"), + } + return e2eeClient +} + +func (c *Client) getClientPayload() *waProto.ClientPayload { + userID, _ := strconv.ParseUint(c.device.ID.User, 10, 64) + return &waProto.ClientPayload{ + Device: proto.Uint32(uint32(c.device.ID.Device)), + FbCat: []byte(c.configs.browserConfigTable.MessengerWebInitData.CryptoAuthToken.EncryptedSerializedCat), + FbUserAgent: []byte(UserAgent), + Product: waProto.ClientPayload_MESSENGER.Enum(), + Username: proto.Uint64(userID), + + ConnectReason: waProto.ClientPayload_USER_ACTIVATED.Enum(), + ConnectType: waProto.ClientPayload_WIFI_UNKNOWN.Enum(), + Passive: proto.Bool(false), + Pull: proto.Bool(true), + UserAgent: &waProto.ClientPayload_UserAgent{ + Device: proto.String("Firefox"), + AppVersion: &waProto.ClientPayload_UserAgent_AppVersion{ + Primary: proto.Uint32(301), + Secondary: proto.Uint32(0), + Tertiary: proto.Uint32(2), + }, + LocaleCountryIso31661Alpha2: proto.String("en"), + LocaleLanguageIso6391: proto.String("en"), + //Hardware: proto.String("Linux"), + Manufacturer: proto.String("Linux"), + Mcc: proto.String("000"), + Mnc: proto.String("000"), + OsBuildNumber: proto.String("6.0.0"), + OsVersion: proto.String("6.0.0"), + //SimMcc: proto.String("000"), + //SimMnc: proto.String("000"), + + Platform: waProto.ClientPayload_UserAgent_WEB.Enum(), // or BLUE_WEB? + ReleaseChannel: waProto.ClientPayload_UserAgent_DEBUG.Enum(), + }, + } +} diff --git a/messagix/e2ee-register.go b/messagix/e2ee-register.go new file mode 100644 index 0000000..2621e87 --- /dev/null +++ b/messagix/e2ee-register.go @@ -0,0 +1,232 @@ +// mautrix-meta - A Matrix-Facebook Messenger and Instagram DM puppeting bridge. +// Copyright (C) 2024 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package messagix + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/rs/zerolog" + "go.mau.fi/libsignal/ecc" + "google.golang.org/protobuf/proto" + + "go.mau.fi/whatsmeow/binary/armadillo/waArmadilloICDC" + waProto "go.mau.fi/whatsmeow/binary/proto" + "go.mau.fi/whatsmeow/store" + "go.mau.fi/whatsmeow/types" +) + +type ICDCFetchResponse struct { + DeviceIdentities []string `json:"device_identities"` + ICDCSeq int `json:"icdc_seq"` + Status int `json:"status"` +} + +func (ifr *ICDCFetchResponse) DeviceIdentityBytes() (deviceIdentityBytes [][32]byte, err error) { + deviceIdentityBytes = make([][32]byte, len(ifr.DeviceIdentities)) + for i, deviceIdentity := range ifr.DeviceIdentities { + var ident []byte + ident, err = base64.RawURLEncoding.DecodeString(deviceIdentity) + if err != nil { + break + } else if len(ident) != 32 { + err = fmt.Errorf("device identity is not 32 bytes long") + break + } + deviceIdentityBytes[i] = [32]byte(ident) + } + return +} + +type ICDCRegisterResponse struct { + ICDCSuccess bool `json:"icdc_success"` // not always present? + Product string `json:"product"` // msgr + Status int `json:"status"` // 200 + Type string `json:"type"` // "new" + WADeviceID int `json:"wa_device_id"` +} + +func (c *Client) doE2EERequest(ctx context.Context, path string, body url.Values, into any) error { + url := "https://reg-e2ee.facebook.com" + path + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(body.Encode())) + if err != nil { + return fmt.Errorf("failed to prepare request: %w", err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") + req.Header.Set("accept-language", "en-US,en;q=0.9") + req.Header.Set("User-Agent", UserAgent) + req.Header.Set("sec-fetch-dest", "empty") + req.Header.Set("sec-fetch-mode", "cors") + req.Header.Set("sec-fetch-site", "cross-site") + req.Header.Set("origin", c.getEndpoint("base_url")) + req.Header.Set("referer", c.getEndpoint("messages")+"/") + zerolog.Ctx(ctx).Trace(). + Str("url", url). + Any("body", body). + Msg("ICDC request") + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + respBody, err := io.ReadAll(resp.Body) + logEvt := zerolog.Ctx(ctx).Trace(). + Str("url", url). + Int("status_code", resp.StatusCode) + if json.Valid(respBody) { + logEvt.RawJSON("response_body", respBody) + } else { + logEvt.Str("response_body", base64.StdEncoding.EncodeToString(respBody)) + } + logEvt.Msg("ICDC response") + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } else if resp.StatusCode >= 300 || resp.StatusCode < 200 { + return fmt.Errorf("unexpected status code %d", resp.StatusCode) + } else if err = json.Unmarshal(respBody, into); err != nil { + return fmt.Errorf("failed to unmarshal response: %w", err) + } + return nil +} + +func (c *Client) fetchICDC(ctx context.Context, fbid int64, deviceUUID uuid.UUID) (*ICDCFetchResponse, error) { + formBody := url.Values{ + "fbid": {strconv.FormatInt(fbid, 10)}, + "fb_cat": {c.configs.browserConfigTable.MessengerWebInitData.CryptoAuthToken.EncryptedSerializedCat}, + "app_id": {strconv.FormatInt(c.configs.browserConfigTable.MessengerWebInitData.AppID, 10)}, + "device_id": {deviceUUID.String()}, + } + var icdcResp ICDCFetchResponse + err := c.doE2EERequest(ctx, "/v2/fb_icdc_fetch", formBody, &icdcResp) + return &icdcResp, err +} + +func calculateIdentitiesHash(identities [][32]byte) []byte { + identitiesSlice := sliceifyIdentities(identities) + slices.SortFunc(identitiesSlice, bytes.Compare) + ihash := sha256.Sum256(bytes.Join(identitiesSlice, nil)) + return ihash[:10] +} + +func sliceifyIdentities(identities [][32]byte) [][]byte { + sliceIdentities := make([][]byte, len(identities)) + for i, identity := range identities { + sliceIdentities[i] = identity[:] + } + return sliceIdentities +} + +func (c *Client) SetDevice(dev *store.Device) { + c.device = dev +} + +func (c *Client) RegisterE2EE(ctx context.Context) error { + if c.device == nil { + return fmt.Errorf("cannot register for E2EE without a device") + } + var fbid int64 + if c.platform.IsMessenger() { + fbid = c.configs.browserConfigTable.CurrentUserInitialData.GetFBID() + } else { + fbid = c.configs.browserConfigTable.PolarisViewer.GetFBID() + } + if c.device.FacebookUUID == uuid.Nil { + c.device.FacebookUUID = uuid.New() + } + icdcMeta, err := c.fetchICDC(ctx, fbid, c.device.FacebookUUID) + if err != nil { + return fmt.Errorf("failed to fetch ICDC metadata: %w", err) + } + deviceIdentities, err := icdcMeta.DeviceIdentityBytes() + ownIdentityIndex := slices.Index(deviceIdentities, *c.device.IdentityKey.Pub) + if ownIdentityIndex == -1 { + ownIdentityIndex = len(deviceIdentities) + deviceIdentities = append(deviceIdentities, *c.device.IdentityKey.Pub) + // TODO does this need to be incremented when reregistering? + icdcMeta.ICDCSeq++ + } + icdcTS := time.Now().Unix() + unsignedList, err := proto.Marshal(&waArmadilloICDC.ICDCIdentityList{ + Seq: int32(icdcMeta.ICDCSeq), + Timestamp: icdcTS, + Devices: sliceifyIdentities(deviceIdentities), + SigningDeviceIndex: int32(ownIdentityIndex), + }) + if err != nil { + return fmt.Errorf("failed to marshal ICDC identity list: %w", err) + } + signature := ecc.CalculateSignature(ecc.NewDjbECPrivateKey(*c.device.IdentityKey.Priv), unsignedList) + signedList, err := proto.Marshal(&waArmadilloICDC.SignedICDCIdentityList{ + Details: unsignedList, + Signature: signature[:], + }) + if err != nil { + return fmt.Errorf("failed to marshal signed ICDC identity list: %w", err) + } + formBody := url.Values{ + "fbid": {strconv.FormatInt(fbid, 10)}, + "fb_cat": {c.configs.browserConfigTable.MessengerWebInitData.CryptoAuthToken.EncryptedSerializedCat}, + "app_id": {strconv.FormatInt(c.configs.browserConfigTable.MessengerWebInitData.AppID, 10)}, + "device_id": {c.device.FacebookUUID.String()}, + "e_regid": {base64.StdEncoding.EncodeToString(binary.BigEndian.AppendUint32(nil, c.device.RegistrationID))}, + "e_keytype": {base64.StdEncoding.EncodeToString([]byte{ecc.DjbType})}, + "e_ident": {base64.StdEncoding.EncodeToString(c.device.IdentityKey.Pub[:])}, + "e_skey_id": {base64.StdEncoding.EncodeToString(binary.BigEndian.AppendUint32(nil, c.device.SignedPreKey.KeyID)[1:])}, + "e_skey_val": {base64.StdEncoding.EncodeToString(c.device.SignedPreKey.Pub[:])}, + "e_skey_sig": {base64.StdEncoding.EncodeToString(c.device.SignedPreKey.Signature[:])}, + "icdc_list": {base64.StdEncoding.EncodeToString(signedList)}, + "icdc_ts": {strconv.FormatInt(icdcTS, 10)}, + "icdc_seq": {strconv.Itoa(icdcMeta.ICDCSeq)}, + "ihash": {base64.StdEncoding.EncodeToString(calculateIdentitiesHash(deviceIdentities))}, + } + var icdcResp ICDCRegisterResponse + err = c.doE2EERequest(ctx, "/v2/fb_register_v2", formBody, &icdcResp) + if err != nil { + return err + } else if icdcResp.Status != 200 { + return fmt.Errorf("ICDC registration returned non-200 inner status %d", icdcResp.Status) + } + c.device.ID = &types.JID{ + User: strconv.FormatInt(fbid, 10), + Device: uint16(icdcResp.WADeviceID), + Server: types.MessengerServer, + } + // This is a hack since currently whatsmeow requires it to be set + c.device.Account = &waProto.ADVSignedDeviceIdentity{ + Details: make([]byte, 0), + AccountSignatureKey: make([]byte, 32), + AccountSignature: make([]byte, 64), + DeviceSignature: make([]byte, 64), + } + zerolog.Ctx(ctx).Info(). + Stringer("jid", c.device.ID). + Msg("ICDC registration successful") + return err +} diff --git a/messagix/types/account.go b/messagix/types/account.go index 120f7b6..25dd5fc 100644 --- a/messagix/types/account.go +++ b/messagix/types/account.go @@ -40,7 +40,9 @@ type CurrentBusinessAccount struct { } type MessengerWebInitData struct { - AccountKey string `json:"accountKey,omitempty"` + AccountKey string `json:"accountKey,omitempty"` + //ActiveThreadKeys + //AllActiveThreadKeys AppID int64 `json:"appId,omitempty"` CryptoAuthToken CryptoAuthToken `json:"cryptoAuthToken,omitempty"` LogoutToken string `json:"logoutToken,omitempty"` diff --git a/user.go b/user.go index 519773f..f2d5207 100644 --- a/user.go +++ b/user.go @@ -29,6 +29,10 @@ import ( "sync/atomic" "github.com/rs/zerolog" + "go.mau.fi/whatsmeow" + "go.mau.fi/whatsmeow/store" + waTypes "go.mau.fi/whatsmeow/types" + "go.mau.fi/whatsmeow/types/events" "golang.org/x/exp/maps" "maunium.net/go/mautrix" "maunium.net/go/mautrix/appservice" @@ -172,6 +176,10 @@ type User struct { Client *messagix.Client + WADevice *store.Device + E2EEClient *whatsmeow.Client + e2eeConnectLock sync.Mutex + BridgeState *bridge.BridgeStateQueue bridgeStateLock sync.Mutex @@ -758,6 +766,103 @@ func (user *User) FillBridgeState(state status.BridgeState) status.BridgeState { return state } +func (user *User) connectE2EE() error { + user.e2eeConnectLock.Lock() + defer user.e2eeConnectLock.Unlock() + if user.E2EEClient != nil { + return fmt.Errorf("already connected to e2ee") + } + var err error + if user.WADevice == nil && user.WADeviceID != 0 { + user.WADevice, err = user.bridge.DeviceStore.GetDevice(waTypes.JID{User: strconv.FormatInt(user.MetaID, 10), Device: user.WADeviceID, Server: waTypes.MessengerServer}) + if err != nil { + return fmt.Errorf("failed to get whatsmeow device: %w", err) + } else if user.WADevice == nil { + user.log.Warn().Uint16("device_id", user.WADeviceID).Msg("Existing device not found in store") + } + } + isNew := false + if user.WADevice == nil { + isNew = true + user.WADevice = user.bridge.DeviceStore.NewDevice() + } + user.Client.SetDevice(user.WADevice) + + ctx := user.log.With().Str("component", "e2ee").Logger().WithContext(context.TODO()) + if isNew { + user.log.Info().Msg("Registering new e2ee device") + err = user.Client.RegisterE2EE(ctx) + if err != nil { + return fmt.Errorf("failed to register e2ee device: %w", err) + } + user.WADeviceID = user.WADevice.ID.Device + err = user.WADevice.Save() + if err != nil { + return fmt.Errorf("failed to save whatsmeow device store: %w", err) + } + err = user.Update(ctx) + if err != nil { + return fmt.Errorf("failed to save device ID to user: %w", err) + } + } + user.E2EEClient = user.Client.PrepareE2EEClient() + user.E2EEClient.AddEventHandler(user.e2eeEventHandler) + err = user.E2EEClient.Connect() + if err != nil { + return fmt.Errorf("failed to connect to e2ee socket: %w", err) + } + return nil +} + +func (user *User) e2eeEventHandler(rawEvt any) { + switch evt := rawEvt.(type) { + case *events.FBConsumerMessage: + log := user.log.With(). + Str("action", "handle whatsapp message"). + Stringer("chat_jid", evt.Info.Chat). + Stringer("sender_jid", evt.Info.Sender). + Str("message_id", evt.Info.ID). + Logger() + ctx := log.WithContext(context.TODO()) + threadID := int64(evt.Info.Chat.UserInt()) + if threadID == 0 { + log.Warn().Msg("Ignoring encrypted message with unsupported jid") + return + } + var expectedType table.ThreadType + switch evt.Info.Chat.Server { + case waTypes.GroupServer: + expectedType = table.ENCRYPTED_OVER_WA_GROUP + case waTypes.MessengerServer, waTypes.DefaultUserServer: + expectedType = table.ENCRYPTED_OVER_WA_ONE_TO_ONE + } + portal := user.GetPortalByThreadID(threadID, expectedType) + changed := false + if portal.ThreadType != expectedType { + log.Info(). + Int64("old_thread_type", int64(portal.ThreadType)). + Int64("new_thread_type", int64(expectedType)). + Msg("Updating thread type") + portal.ThreadType = expectedType + changed = true + } + if portal.WhatsAppServer != evt.Info.Chat.Server { + log.Info(). + Str("old_server", portal.WhatsAppServer). + Str("new_server", evt.Info.Chat.Server). + Msg("Updating WhatsApp server") + portal.WhatsAppServer = evt.Info.Chat.Server + changed = true + } + if changed { + err := portal.Update(ctx) + if err != nil { + log.Err(err).Msg("Failed to update portal") + } + } + } +} + func (user *User) eventHandler(rawEvt any) { switch evt := rawEvt.(type) { case *messagix.Event_PublishResponse: @@ -797,6 +902,14 @@ func (user *User) eventHandler(rawEvt any) { user.BridgeState.Send(status.BridgeState{StateEvent: status.StateConnected}) user.tryAutomaticDoublePuppeting() user.handleTable(evt.Table) + if user.bridge.Config.Meta.Mode.IsMessenger() { + go func() { + err := user.connectE2EE() + if err != nil { + user.log.Err(err).Msg("Error connecting to e2ee") + } + }() + } go user.BackfillLoop() case *messagix.Event_SocketError: user.BridgeState.Send(status.BridgeState{ @@ -842,8 +955,12 @@ func (user *User) unlockedDisconnect() { if user.Client != nil { user.Client.Disconnect() } + if user.E2EEClient != nil { + user.E2EEClient.Disconnect() + } user.StopBackfillLoop() user.Client = nil + user.E2EEClient = nil } func (user *User) Disconnect() error { @@ -857,6 +974,12 @@ func (user *User) DeleteSession() { user.Lock() defer user.Unlock() user.unlockedDisconnect() + if user.WADevice != nil { + err := user.WADevice.Delete() + if err != nil { + user.log.Err(err).Msg("Failed to delete whatsmeow device") + } + } user.Cookies = nil user.MetaID = 0 doublePuppet := user.bridge.GetPuppetByCustomMXID(user.MXID) From f3f566fa59a90638b6696d9d92aee3cbb7b06b05 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 Feb 2024 22:27:35 +0200 Subject: [PATCH 03/19] Add support for incoming WhatsApp text messages --- messagix/table/enums.go | 9 +++ msgconv/from-whatsapp.go | 132 +++++++++++++++++++++++++++++++++++++++ portal.go | 131 +++++++++++++++++++++++++++++++++++--- user.go | 1 + 4 files changed, 265 insertions(+), 8 deletions(-) create mode 100644 msgconv/from-whatsapp.go diff --git a/messagix/table/enums.go b/messagix/table/enums.go index 874bee0..b574c1a 100644 --- a/messagix/table/enums.go +++ b/messagix/table/enums.go @@ -241,6 +241,15 @@ func (tt ThreadType) IsOneToOne() bool { } } +func (tt ThreadType) IsWhatsApp() bool { + switch tt { + case ENCRYPTED_OVER_WA_GROUP, ENCRYPTED_OVER_WA_ONE_TO_ONE: + return true + default: + return false + } +} + const ( UNKNOWN_THREAD_TYPE ThreadType = 0 ONE_TO_ONE ThreadType = 1 diff --git a/msgconv/from-whatsapp.go b/msgconv/from-whatsapp.go new file mode 100644 index 0000000..e50416f --- /dev/null +++ b/msgconv/from-whatsapp.go @@ -0,0 +1,132 @@ +// mautrix-meta - A Matrix-Facebook Messenger and Instagram DM puppeting bridge. +// Copyright (C) 2024 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package msgconv + +import ( + "context" + "fmt" + _ "image/gif" + _ "image/jpeg" + _ "image/png" + "slices" + "strings" + + "github.com/rs/zerolog" + "go.mau.fi/whatsmeow/binary/armadillo/waCommon" + "go.mau.fi/whatsmeow/binary/armadillo/waConsumerApplication" + "go.mau.fi/whatsmeow/types" + "go.mau.fi/whatsmeow/types/events" + _ "golang.org/x/image/webp" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +func (mc *MessageConverter) whatsappTextToMatrix(ctx context.Context, text *waCommon.MessageText) *ConvertedMessagePart { + content := &event.MessageEventContent{ + MsgType: event.MsgText, + Body: text.GetText(), + Mentions: &event.Mentions{}, + } + silent := false + if len(text.Commands) > 0 { + for _, cmd := range text.Commands { + switch cmd.CommandType { + case waCommon.Command_SILENT: + silent = true + content.Mentions.Room = false + case waCommon.Command_EVERYONE: + if !silent { + content.Mentions.Room = true + } + case waCommon.Command_AI: + // TODO ??? + } + } + } + if len(text.GetMentionedJID()) > 0 { + content.Format = event.FormatHTML + content.FormattedBody = event.TextToHTML(content.Body) + for _, jid := range text.GetMentionedJID() { + parsed, err := types.ParseJID(jid) + if err != nil { + zerolog.Ctx(ctx).Err(err).Str("jid", jid).Msg("Failed to parse mentioned JID") + continue + } + mxid := mc.GetUserMXID(ctx, int64(parsed.UserInt())) + if !silent { + content.Mentions.UserIDs = append(content.Mentions.UserIDs, mxid) + } + mentionText := "@" + jid + content.Body = strings.ReplaceAll(content.Body, mentionText, mxid.String()) + content.FormattedBody = strings.ReplaceAll(content.FormattedBody, mentionText, fmt.Sprintf(`%s`, mxid.URI().MatrixToURL(), mxid.String())) + } + } + return &ConvertedMessagePart{ + Type: event.EventMessage, + Content: content, + } +} + +func (mc *MessageConverter) WhatsAppToMatrix(ctx context.Context, evt *events.FBConsumerMessage) *ConvertedMessage { + cm := &ConvertedMessage{ + Parts: make([]*ConvertedMessagePart, 0), + } + switch content := evt.Message.GetPayload().GetContent().GetContent().(type) { + case *waConsumerApplication.ConsumerApplication_Content_MessageText: + cm.Parts = append(cm.Parts, mc.whatsappTextToMatrix(ctx, content.MessageText)) + case *waConsumerApplication.ConsumerApplication_Content_ExtendedTextMessage: + // TODO convert url previews + cm.Parts = append(cm.Parts, mc.whatsappTextToMatrix(ctx, content.ExtendedTextMessage.GetText())) + case *waConsumerApplication.ConsumerApplication_Content_ImageMessage: + case *waConsumerApplication.ConsumerApplication_Content_StickerMessage: + case *waConsumerApplication.ConsumerApplication_Content_ViewOnceMessage: + case *waConsumerApplication.ConsumerApplication_Content_DocumentMessage: + case *waConsumerApplication.ConsumerApplication_Content_AudioMessage: + case *waConsumerApplication.ConsumerApplication_Content_VideoMessage: + case *waConsumerApplication.ConsumerApplication_Content_LocationMessage: + case *waConsumerApplication.ConsumerApplication_Content_LiveLocationMessage: + case *waConsumerApplication.ConsumerApplication_Content_ContactMessage: + case *waConsumerApplication.ConsumerApplication_Content_ContactsArrayMessage: + } + if len(cm.Parts) == 0 { + cm.Parts = append(cm.Parts, &ConvertedMessagePart{ + Type: event.EventMessage, + Content: &event.MessageEventContent{ + MsgType: event.MsgNotice, + Body: "Unsupported message", + }, + }) + } + var replyTo id.EventID + var sender id.UserID + if qm := evt.Application.GetMetadata().GetQuotedMessage(); qm != nil { + pcp, _ := types.ParseJID(qm.GetParticipant()) + replyTo, sender = mc.GetMatrixReply(ctx, qm.GetStanzaID(), int64(pcp.UserInt())) + } + for _, part := range cm.Parts { + if part.Content.Mentions == nil { + part.Content.Mentions = &event.Mentions{} + } + if replyTo != "" { + part.Content.RelatesTo = (&event.RelatesTo{}).SetReplyTo(replyTo) + if !slices.Contains(part.Content.Mentions.UserIDs, sender) { + part.Content.Mentions.UserIDs = append(part.Content.Mentions.UserIDs, sender) + } + } + } + return cm +} diff --git a/portal.go b/portal.go index 7cf9fa4..6ffd6b4 100644 --- a/portal.go +++ b/portal.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "reflect" + "slices" "strconv" "sync" "sync/atomic" @@ -28,6 +29,9 @@ import ( "github.com/rs/zerolog" "go.mau.fi/util/variationselector" + "go.mau.fi/whatsmeow/binary/armadillo/waConsumerApplication" + "go.mau.fi/whatsmeow/types" + "go.mau.fi/whatsmeow/types/events" "maunium.net/go/mautrix" "maunium.net/go/mautrix/appservice" "maunium.net/go/mautrix/bridge" @@ -326,6 +330,10 @@ func (portal *Portal) getBridgeInfo() (string, CustomBridgeInfoContent) { bridgeInfo.Protocol.ExternalURL = "https://www.messenger.com/" bridgeInfo.Channel.ExternalURL = fmt.Sprintf("https://www.messenger.com/t/%d", portal.ThreadID) } + if portal.ThreadType.IsWhatsApp() { + // TODO store fb-side thread ID? (the whatsapp chat id is not the same as the fb-side thread id used in urls) + bridgeInfo.Channel.ExternalURL = "" + } var roomType string if portal.IsPrivateChat() { roomType = "dm" @@ -847,7 +855,7 @@ func (portal *Portal) GetMatrixReply(ctx context.Context, replyToID string, repl } } else { replyTo = message.MXID - if message.Sender != replyToUser { + if replyToUser != 0 && message.Sender != replyToUser { log.Warn(). Int64("message_sender", message.Sender). Int64("reply_to_user", replyToUser). @@ -898,6 +906,8 @@ func (portal *Portal) GetUserMXID(ctx context.Context, userID int64) id.UserID { func (portal *Portal) handleMetaMessage(portalMessage portalMetaMessage) { switch typedEvt := portalMessage.evt.(type) { + case *events.FBConsumerMessage: + portal.handleEncryptedMessage(portalMessage.user, typedEvt) case *table.WrappedMessage: portal.handleMetaInsertMessage(portalMessage.user, typedEvt) case *table.UpsertMessages: @@ -955,6 +965,41 @@ func (portal *Portal) checkPendingMessage(ctx context.Context, messageID string, return true } +func (portal *Portal) handleEncryptedMessage(source *User, evt *events.FBConsumerMessage) { + sender := portal.bridge.GetPuppetByID(int64(evt.Info.Sender.UserInt())) + log := portal.log.With(). + Str("action", "handle whatsapp message"). + Stringer("chat_jid", evt.Info.Chat). + Stringer("sender_jid", evt.Info.Sender). + Str("message_id", evt.Info.ID). + Logger() + ctx := log.WithContext(context.TODO()) + + switch payload := evt.Message.GetPayload().GetPayload().(type) { + case *waConsumerApplication.ConsumerApplication_Payload_Content: + switch payload.Content.GetContent().(type) { + case *waConsumerApplication.ConsumerApplication_Content_EditMessage: + log.Warn().Msg("Unsupported edit message payload message") + case *waConsumerApplication.ConsumerApplication_Content_ReactionMessage: + log.Warn().Msg("Unsupported reaction message payload message") + default: + portal.handleMetaOrWhatsAppMessage(ctx, source, sender, evt, nil) + } + case *waConsumerApplication.ConsumerApplication_Payload_ApplicationData: + switch applicationContent := payload.ApplicationData.GetApplicationContent().(type) { + case *waConsumerApplication.ConsumerApplication_ApplicationData_Revoke: + default: + log.Warn().Type("content_type", applicationContent).Msg("Unrecognized application content type") + } + case *waConsumerApplication.ConsumerApplication_Payload_Signal: + log.Warn().Msg("Unsupported signal payload message") + case *waConsumerApplication.ConsumerApplication_Payload_SubProtocol: + log.Warn().Msg("Unsupported subprotocol payload message") + default: + log.Warn().Type("payload_type", payload).Msg("Unrecognized payload type") + } +} + func (portal *Portal) handleMetaInsertMessage(source *User, message *table.WrappedMessage) { sender := portal.bridge.GetPuppetByID(message.SenderId) log := portal.log.With(). @@ -964,6 +1009,11 @@ func (portal *Portal) handleMetaInsertMessage(source *User, message *table.Wrapp Str("otid", message.OfflineThreadingId). Logger() ctx := log.WithContext(context.TODO()) + portal.handleMetaOrWhatsAppMessage(ctx, source, sender, nil, message) +} + +func (portal *Portal) handleMetaOrWhatsAppMessage(ctx context.Context, source *User, sender *Puppet, waMsg *events.FBConsumerMessage, metaMsg *table.WrappedMessage) { + log := zerolog.Ctx(ctx) if portal.MXID == "" { log.Debug().Msg("Creating Matrix room from incoming message") @@ -973,13 +1023,22 @@ func (portal *Portal) handleMetaInsertMessage(source *User, message *table.Wrapp } } - otidInt, _ := strconv.ParseInt(message.OfflineThreadingId, 10, 64) - messageTime := time.UnixMilli(message.TimestampMs) - if portal.checkPendingMessage(ctx, message.MessageId, otidInt, sender.ID, messageTime) { - return + var messageID string + var messageTime time.Time + var otidInt int64 + if waMsg != nil { + messageID = waMsg.Info.ID + messageTime = waMsg.Info.Timestamp + } else { + messageID = metaMsg.MessageId + otidInt, _ = strconv.ParseInt(metaMsg.OfflineThreadingId, 10, 64) + messageTime = time.UnixMilli(metaMsg.TimestampMs) + if portal.checkPendingMessage(ctx, metaMsg.MessageId, otidInt, sender.ID, messageTime) { + return + } } - existingMessage, err := portal.bridge.DB.Message.GetByID(ctx, message.MessageId, 0, portal.Receiver) + existingMessage, err := portal.bridge.DB.Message.GetByID(ctx, messageID, 0, portal.Receiver) if err != nil { log.Err(err).Msg("Failed to check if message was already bridged") return @@ -991,7 +1050,12 @@ func (portal *Portal) handleMetaInsertMessage(source *User, message *table.Wrapp intent := sender.IntentFor(portal) ctx = context.WithValue(ctx, msgconvContextKeyIntent, intent) ctx = context.WithValue(ctx, msgconvContextKeyClient, source.Client) - converted := portal.MsgConv.ToMatrix(ctx, message) + var converted *msgconv.ConvertedMessage + if waMsg != nil { + converted = portal.MsgConv.WhatsAppToMatrix(ctx, waMsg) + } else { + converted = portal.MsgConv.ToMatrix(ctx, metaMsg) + } if portal.bridge.Config.Bridge.CaptionInMessage { converted.MergeCaption() } @@ -1005,7 +1069,7 @@ func (portal *Portal) handleMetaInsertMessage(source *User, message *table.Wrapp log.Err(err).Int("part_index", i).Msg("Failed to send message to Matrix") continue } - portal.storeMessageInDB(ctx, resp.EventID, message.MessageId, otidInt, sender.ID, messageTime, i) + portal.storeMessageInDB(ctx, resp.EventID, messageID, otidInt, sender.ID, messageTime, i) } } @@ -1389,6 +1453,14 @@ func (portal *Portal) CreateMatrixRoom(ctx context.Context, user *User) error { if autoJoinInvites { invite = append(invite, user.MXID) } + var waGroupInfo *types.GroupInfo + var participants []id.UserID + if portal.ThreadType == table.ENCRYPTED_OVER_WA_GROUP { + waGroupInfo, participants = portal.UpdateWAGroupInfo(ctx, user, nil) + invite = append(invite, participants...) + slices.Sort(invite) + invite = slices.Compact(invite) + } if portal.bridge.Config.Bridge.Encryption.Default { initialState = append(initialState, &event.Event{ @@ -1461,6 +1533,9 @@ func (portal *Portal) CreateMatrixRoom(ctx context.Context, user *User) error { if portal.IsPrivateChat() { user.AddDirectChat(ctx, portal.MXID, portal.GetDMPuppet().MXID) } + if waGroupInfo != nil && !autoJoinInvites { + portal.SyncWAParticipants(ctx, waGroupInfo.Participants) + } return nil } @@ -1483,6 +1558,46 @@ func (portal *Portal) UpdateInfoFromPuppet(ctx context.Context, puppet *Puppet) } } +func (portal *Portal) UpdateWAGroupInfo(ctx context.Context, source *User, groupInfo *types.GroupInfo) (*types.GroupInfo, []id.UserID) { + log := zerolog.Ctx(ctx) + if groupInfo == nil { + var err error + groupInfo, err = source.E2EEClient.GetGroupInfo(portal.JID()) + if err != nil { + log.Err(err).Msg("Failed to fetch WhatsApp group info") + return nil, nil + } + } + update := false + update = portal.updateName(ctx, groupInfo.Name) || update + //update = portal.updateTopic(ctx, groupInfo.Topic) || update + //update = portal.updateWAAvatar(ctx) + participants := portal.SyncWAParticipants(ctx, groupInfo.Participants) + if update { + err := portal.Update(ctx) + if err != nil { + log.Err(err).Msg("Failed to save portal in database after updating group info") + } + portal.UpdateBridgeInfo(ctx) + } + return groupInfo, participants +} + +func (portal *Portal) SyncWAParticipants(ctx context.Context, participants []types.GroupParticipant) []id.UserID { + var userIDs []id.UserID + for _, pcp := range participants { + puppet := portal.bridge.GetPuppetByID(int64(pcp.JID.UserInt())) + userIDs = append(userIDs, puppet.IntentFor(portal).UserID) + if portal.MXID != "" { + err := puppet.IntentFor(portal).EnsureJoined(ctx, portal.MXID) + if err != nil { + zerolog.Ctx(ctx).Err(err).Msg("Failed to ensure participant is joined to group") + } + } + } + return userIDs +} + func (portal *Portal) UpdateInfo(ctx context.Context, info table.ThreadInfo) { log := zerolog.Ctx(ctx).With(). Str("function", "UpdateInfo"). diff --git a/user.go b/user.go index f2d5207..ff6c059 100644 --- a/user.go +++ b/user.go @@ -860,6 +860,7 @@ func (user *User) e2eeEventHandler(rawEvt any) { log.Err(err).Msg("Failed to update portal") } } + portal.metaMessages <- portalMetaMessage{user: user, evt: evt} } } From 01208b999fc31d9a8d35c824fe1191c59ca6fe74 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Feb 2024 16:17:30 +0200 Subject: [PATCH 04/19] Add support for outgoing WhatsApp text messages --- msgconv/to-whatsapp.go | 60 +++++++++++++++++++++++++++ portal.go | 94 +++++++++++++++++++++++++++--------------- user.go | 3 ++ 3 files changed, 124 insertions(+), 33 deletions(-) create mode 100644 msgconv/to-whatsapp.go diff --git a/msgconv/to-whatsapp.go b/msgconv/to-whatsapp.go new file mode 100644 index 0000000..f37f12e --- /dev/null +++ b/msgconv/to-whatsapp.go @@ -0,0 +1,60 @@ +// mautrix-meta - A Matrix-Facebook Messenger and Instagram DM puppeting bridge. +// Copyright (C) 2024 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package msgconv + +import ( + "context" + "fmt" + + "maunium.net/go/mautrix/event" + + "go.mau.fi/whatsmeow/binary/armadillo/waCommon" + "go.mau.fi/whatsmeow/binary/armadillo/waConsumerApplication" +) + +func (mc *MessageConverter) ToWhatsApp(ctx context.Context, evt *event.Event, content *event.MessageEventContent, relaybotFormatted bool) (*waConsumerApplication.ConsumerApplication, error) { + if content.MsgType == event.MsgEmote && !relaybotFormatted { + content.Body = "/me " + content.Body + if content.FormattedBody != "" { + content.FormattedBody = "/me " + content.FormattedBody + } + } + var waContent waConsumerApplication.ConsumerApplication_Content + switch content.MsgType { + case event.MsgText, event.MsgNotice, event.MsgEmote: + // TODO mentions + waContent.Content = &waConsumerApplication.ConsumerApplication_Content_MessageText{ + MessageText: &waCommon.MessageText{ + Text: content.Body, + }, + } + case event.MsgImage, event.MsgVideo, event.MsgAudio, event.MsgFile: + fallthrough + case event.MsgLocation: + fallthrough + default: + return nil, fmt.Errorf("%w %s", ErrUnsupportedMsgType, content.MsgType) + } + return &waConsumerApplication.ConsumerApplication{ + Payload: &waConsumerApplication.ConsumerApplication_Payload{ + Payload: &waConsumerApplication.ConsumerApplication_Payload_Content{ + Content: &waContent, + }, + }, + Metadata: nil, + }, nil +} diff --git a/portal.go b/portal.go index 6ffd6b4..4bae1d3 100644 --- a/portal.go +++ b/portal.go @@ -29,6 +29,7 @@ import ( "github.com/rs/zerolog" "go.mau.fi/util/variationselector" + "go.mau.fi/whatsmeow" "go.mau.fi/whatsmeow/binary/armadillo/waConsumerApplication" "go.mau.fi/whatsmeow/types" "go.mau.fi/whatsmeow/types/events" @@ -499,13 +500,27 @@ func (portal *Portal) handleMatrixMessage(ctx context.Context, sender *User, evt } if editTarget := content.RelatesTo.GetReplaceID(); editTarget != "" { + if portal.ThreadType.IsWhatsApp() { + // TODO implement + go ms.sendMessageMetrics(evt, fmt.Errorf("whatsapp edits aren't supported yet"), "Ignoring", true) + return + } portal.handleMatrixEdit(ctx, sender, isRelay, realSenderMXID, &ms, evt, content) return } relaybotFormatted := isRelay && portal.addRelaybotFormat(ctx, realSenderMXID, evt, content) - ctx = context.WithValue(ctx, msgconvContextKeyClient, sender.Client) - tasks, otid, err := portal.MsgConv.ToMeta(ctx, evt, content, relaybotFormatted) + var otid int64 + var tasks []socket.Task + var waMsg *waConsumerApplication.ConsumerApplication + var err error + if portal.ThreadType.IsWhatsApp() { + ctx = context.WithValue(ctx, msgconvContextKeyE2EEClient, sender.E2EEClient) + waMsg, err = portal.MsgConv.ToWhatsApp(ctx, evt, content, relaybotFormatted) + } else { + ctx = context.WithValue(ctx, msgconvContextKeyClient, sender.Client) + tasks, otid, err = portal.MsgConv.ToMeta(ctx, evt, content, relaybotFormatted) + } if err != nil { log.Err(err).Msg("Failed to convert message") go ms.sendMessageMetrics(evt, err, "Error converting", true) @@ -515,43 +530,54 @@ func (portal *Portal) handleMatrixMessage(ctx context.Context, sender *User, evt timings.convert = time.Since(start) start = time.Now() - otidStr := strconv.FormatInt(otid, 10) - portal.pendingMessages[otid] = evt.ID - messageTS := time.Now() - resp, err := sender.Client.ExecuteTasks(tasks...) - log.Trace().Any("response", resp).Msg("Meta send response") - var msgID string - if err == nil { - for _, replace := range resp.LSReplaceOptimsiticMessage { - if replace.OfflineThreadingId == otidStr { - msgID = replace.MessageId + if waMsg != nil { + messageID := sender.E2EEClient.GenerateMessageID() + var resp whatsmeow.SendResponse + resp, err = sender.E2EEClient.SendFBMessage(ctx, portal.JID(), waMsg, nil, whatsmeow.SendRequestExtra{ + ID: messageID, + }) + // TODO save message in db before sending and only update timestamp later + portal.storeMessageInDB(ctx, evt.ID, messageID, 0, sender.MetaID, resp.Timestamp, 0) + } else { + otidStr := strconv.FormatInt(otid, 10) + portal.pendingMessages[otid] = evt.ID + messageTS := time.Now() + var resp *table.LSTable + resp, err = sender.Client.ExecuteTasks(tasks...) + log.Trace().Any("response", resp).Msg("Meta send response") + var msgID string + if err == nil { + for _, replace := range resp.LSReplaceOptimsiticMessage { + if replace.OfflineThreadingId == otidStr { + msgID = replace.MessageId + } } - } - if len(msgID) == 0 { - for _, failed := range resp.LSMarkOptimisticMessageFailed { - if failed.OTID == otidStr { - log.Warn().Str("message", failed.Message).Msg("Sending message failed") - go ms.sendMessageMetrics(evt, fmt.Errorf("%w: %s", errServerRejected, failed.Message), "Error sending", true) - return + if len(msgID) == 0 { + for _, failed := range resp.LSMarkOptimisticMessageFailed { + if failed.OTID == otidStr { + log.Warn().Str("message", failed.Message).Msg("Sending message failed") + go ms.sendMessageMetrics(evt, fmt.Errorf("%w: %s", errServerRejected, failed.Message), "Error sending", true) + return + } } + log.Warn().Msg("Message send response didn't include message ID") + } + } + if msgID != "" { + portal.pendingMessagesLock.Lock() + _, ok = portal.pendingMessages[otid] + if ok { + portal.storeMessageInDB(ctx, evt.ID, msgID, otid, sender.MetaID, messageTS, 0) + delete(portal.pendingMessages, otid) + } else { + log.Debug().Msg("Not storing message send response: pending message was already removed from map") } - log.Warn().Msg("Message send response didn't include message ID") + portal.pendingMessagesLock.Unlock() } } timings.totalSend = time.Since(start) go ms.sendMessageMetrics(evt, err, "Error sending", true) - if msgID != "" { - portal.pendingMessagesLock.Lock() - _, ok = portal.pendingMessages[otid] - if ok { - portal.storeMessageInDB(ctx, evt.ID, msgID, otid, sender.MetaID, messageTS, 0) - delete(portal.pendingMessages, otid) - } else { - log.Debug().Msg("Not storing message send response: pending message was already removed from map") - } - portal.pendingMessagesLock.Unlock() - } } func (portal *Portal) handleMatrixEdit(ctx context.Context, sender *User, isRelay bool, realSenderMXID id.UserID, ms *metricSender, evt *event.Event, content *event.MessageEventContent) { @@ -784,6 +810,7 @@ type msgconvContextKey int const ( msgconvContextKeyIntent msgconvContextKey = iota msgconvContextKeyClient + msgconvContextKeyE2EEClient msgconvContextKeyBackfill ) @@ -1049,11 +1076,12 @@ func (portal *Portal) handleMetaOrWhatsAppMessage(ctx context.Context, source *U intent := sender.IntentFor(portal) ctx = context.WithValue(ctx, msgconvContextKeyIntent, intent) - ctx = context.WithValue(ctx, msgconvContextKeyClient, source.Client) var converted *msgconv.ConvertedMessage if waMsg != nil { + ctx = context.WithValue(ctx, msgconvContextKeyE2EEClient, source.E2EEClient) converted = portal.MsgConv.WhatsAppToMatrix(ctx, waMsg) } else { + ctx = context.WithValue(ctx, msgconvContextKeyClient, source.Client) converted = portal.MsgConv.ToMatrix(ctx, metaMsg) } if portal.bridge.Config.Bridge.CaptionInMessage { @@ -1604,7 +1632,7 @@ func (portal *Portal) UpdateInfo(ctx context.Context, info table.ThreadInfo) { Logger() ctx = log.WithContext(ctx) update := false - if portal.ThreadType != info.GetThreadType() { + if portal.ThreadType != info.GetThreadType() && !portal.ThreadType.IsWhatsApp() { portal.ThreadType = info.GetThreadType() update = true } diff --git a/user.go b/user.go index ff6c059..f13e96f 100644 --- a/user.go +++ b/user.go @@ -835,6 +835,9 @@ func (user *User) e2eeEventHandler(rawEvt any) { expectedType = table.ENCRYPTED_OVER_WA_GROUP case waTypes.MessengerServer, waTypes.DefaultUserServer: expectedType = table.ENCRYPTED_OVER_WA_ONE_TO_ONE + default: + log.Warn().Msg("Unexpected chat server in encrypted message") + return } portal := user.GetPortalByThreadID(threadID, expectedType) changed := false From cc1784629d0223a6c49528492f6312e34ea2baab Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Feb 2024 20:50:48 +0200 Subject: [PATCH 05/19] Add support for incoming WhatsApp media --- msgconv/from-meta.go | 68 ++++++----- msgconv/from-whatsapp.go | 258 +++++++++++++++++++++++++++++++++++++-- msgconv/msgconv.go | 2 + portal.go | 4 + 4 files changed, 293 insertions(+), 39 deletions(-) diff --git a/msgconv/from-meta.go b/msgconv/from-meta.go index 958f0e1..e73a726 100644 --- a/msgconv/from-meta.go +++ b/msgconv/from-meta.go @@ -62,6 +62,7 @@ func (cm *ConvertedMessage) MergeCaption() { mediaContent.Body = textContent.Body mediaContent.Format = textContent.Format mediaContent.FormattedBody = textContent.FormattedBody + mediaContent.Mentions = textContent.Mentions cm.Parts = cm.Parts[:1] } @@ -356,6 +357,39 @@ func (mc *MessageConverter) xmaAttachmentToMatrix(ctx context.Context, att *tabl return mc.fetchFullXMA(ctx, att, converted) } +func (mc *MessageConverter) uploadAttachment(ctx context.Context, data []byte, fileName, mimeType string) (*event.MessageEventContent, error) { + var file *event.EncryptedFileInfo + uploadMime := mimeType + uploadFileName := fileName + if mc.GetData(ctx).Encrypted { + file = &event.EncryptedFileInfo{ + EncryptedFile: *attachment.NewEncryptedFile(), + URL: "", + } + file.EncryptInPlace(data) + uploadMime = "application/octet-stream" + uploadFileName = "" + } + mxc, err := mc.UploadMatrixMedia(ctx, data, uploadFileName, uploadMime) + if err != nil { + return nil, err + } + content := &event.MessageEventContent{ + Body: fileName, + Info: &event.FileInfo{ + MimeType: mimeType, + Size: len(data), + }, + } + if file != nil { + file.URL = mxc + content.File = file + } else { + content.URL = mxc + } + return content, nil +} + func (mc *MessageConverter) reuploadAttachment( ctx context.Context, attachmentType table.AttachmentType, url, fileName, mimeType string, @@ -388,32 +422,14 @@ func (mc *MessageConverter) reuploadAttachment( width, height = config.Width, config.Height } } - var file *event.EncryptedFileInfo - uploadMime := mimeType - uploadFileName := fileName - if mc.GetData(ctx).Encrypted { - file = &event.EncryptedFileInfo{ - EncryptedFile: *attachment.NewEncryptedFile(), - URL: "", - } - file.EncryptInPlace(data) - uploadMime = "application/octet-stream" - uploadFileName = "" - } - mxc, err := mc.UploadMatrixMedia(ctx, data, uploadFileName, uploadMime) + content, err := mc.uploadAttachment(ctx, data, fileName, mimeType) if err != nil { return nil, err } - content := &event.MessageEventContent{ - Body: fileName, - Info: &event.FileInfo{ - MimeType: mimeType, - Duration: duration, - Width: width, - Height: height, - Size: len(data), - }, - } + content.Info.Duration = duration + content.Info.Width = width + content.Info.Height = height + if attachmentType == table.AttachmentTypeAnimatedImage && mimeType == "video/mp4" { extra["info"] = map[string]any{ "fi.mau.gif": true, @@ -450,12 +466,6 @@ func (mc *MessageConverter) reuploadAttachment( if content.Body == "" { content.Body = strings.TrimPrefix(string(content.MsgType), "m.") + exmime.ExtensionFromMimetype(mimeType) } - if file != nil { - file.URL = mxc - content.File = file - } else { - content.URL = mxc - } return &ConvertedMessagePart{ Type: eventType, Content: content, diff --git a/msgconv/from-whatsapp.go b/msgconv/from-whatsapp.go index e50416f..f1c6e74 100644 --- a/msgconv/from-whatsapp.go +++ b/msgconv/from-whatsapp.go @@ -26,8 +26,12 @@ import ( "strings" "github.com/rs/zerolog" + "go.mau.fi/util/exmime" + "go.mau.fi/util/ffmpeg" + "go.mau.fi/whatsmeow" "go.mau.fi/whatsmeow/binary/armadillo/waCommon" "go.mau.fi/whatsmeow/binary/armadillo/waConsumerApplication" + "go.mau.fi/whatsmeow/binary/armadillo/waMediaTransport" "go.mau.fi/whatsmeow/types" "go.mau.fi/whatsmeow/types/events" _ "golang.org/x/image/webp" @@ -81,6 +85,194 @@ func (mc *MessageConverter) whatsappTextToMatrix(ctx context.Context, text *waCo } } +type MediaTransportContainer interface { + GetTransport() *waMediaTransport.WAMediaTransport +} + +type AttachmentTransport[Integral MediaTransportContainer, Ancillary any] interface { + GetIntegral() Integral + GetAncillary() Ancillary +} + +type AttachmentMessage[Integral MediaTransportContainer, Ancillary any, Transport AttachmentTransport[Integral, Ancillary]] interface { + Decode() (Transport, error) +} + +type AttachmentMessageWithCaption[Integral MediaTransportContainer, Ancillary any, Transport AttachmentTransport[Integral, Ancillary]] interface { + GetCaption() *waCommon.MessageText +} + +type convertFunc func(ctx context.Context, data []byte, mimeType string) ([]byte, string, string, error) + +func convertWhatsAppAttachment[ + Transport AttachmentTransport[Integral, Ancillary], + Integral MediaTransportContainer, + Ancillary any, +]( + ctx context.Context, + mc *MessageConverter, + msg AttachmentMessage[Integral, Ancillary, Transport], + mediaType whatsmeow.MediaType, + convert convertFunc, +) (metadata Ancillary, media, caption *ConvertedMessagePart, err error) { + var typedTransport Transport + typedTransport, err = msg.Decode() + if err != nil { + return + } + msgWithCaption, ok := msg.(AttachmentMessageWithCaption[Integral, Ancillary, Transport]) + if ok && len(msgWithCaption.GetCaption().GetText()) > 0 { + caption = mc.whatsappTextToMatrix(ctx, msgWithCaption.GetCaption()) + caption.Content.MsgType = event.MsgNotice + } + metadata = typedTransport.GetAncillary() + transport := typedTransport.GetIntegral().GetTransport() + media, err = mc.reuploadWhatsAppAttachment(ctx, transport, mediaType, convert) + return +} + +func (mc *MessageConverter) reuploadWhatsAppAttachment( + ctx context.Context, + transport *waMediaTransport.WAMediaTransport, + mediaType whatsmeow.MediaType, + convert convertFunc, +) (*ConvertedMessagePart, error) { + data, err := mc.GetE2EEClient(ctx).DownloadFB(transport.GetIntegral(), mediaType) + if err != nil { + return nil, fmt.Errorf("failed to download: %w", err) + } + var fileName string + mimeType := transport.GetAncillary().GetMimetype() + if convert != nil { + data, fileName, mimeType, err = convert(ctx, data, mimeType) + if err != nil { + return nil, fmt.Errorf("failed to convert: %w", err) + } + } + content, err := mc.uploadAttachment(ctx, data, fileName, mimeType) + if err != nil { + return nil, fmt.Errorf("failed to upload: %w", err) + } + return &ConvertedMessagePart{ + Type: event.EventMessage, + Content: content, + Extra: make(map[string]any), + }, nil +} + +func (mc *MessageConverter) convertWhatsAppImage(ctx context.Context, image *waConsumerApplication.ConsumerApplication_ImageMessage) (converted, caption *ConvertedMessagePart, err error) { + metadata, converted, caption, err := convertWhatsAppAttachment[*waMediaTransport.ImageTransport](ctx, mc, image, whatsmeow.MediaImage, func(ctx context.Context, data []byte, mimeType string) ([]byte, string, string, error) { + fileName := "image" + exmime.ExtensionFromMimetype(mimeType) + return data, mimeType, fileName, nil + }) + if converted != nil { + converted.Content.MsgType = event.MsgImage + converted.Content.Info.Width = int(metadata.GetWidth()) + converted.Content.Info.Height = int(metadata.GetHeight()) + } + return +} + +func (mc *MessageConverter) convertWhatsAppSticker(ctx context.Context, sticker *waConsumerApplication.ConsumerApplication_StickerMessage) (converted, caption *ConvertedMessagePart, err error) { + metadata, converted, caption, err := convertWhatsAppAttachment[*waMediaTransport.StickerTransport](ctx, mc, sticker, whatsmeow.MediaImage, func(ctx context.Context, data []byte, mimeType string) ([]byte, string, string, error) { + fileName := "sticker" + exmime.ExtensionFromMimetype(mimeType) + return data, mimeType, fileName, nil + }) + if converted != nil { + converted.Type = event.EventSticker + converted.Content.Info.Width = int(metadata.GetWidth()) + converted.Content.Info.Height = int(metadata.GetHeight()) + } + return +} + +func (mc *MessageConverter) convertWhatsAppDocument(ctx context.Context, document *waConsumerApplication.ConsumerApplication_DocumentMessage) (converted, caption *ConvertedMessagePart, err error) { + _, converted, caption, err = convertWhatsAppAttachment[*waMediaTransport.DocumentTransport](ctx, mc, document, whatsmeow.MediaDocument, func(ctx context.Context, data []byte, mimeType string) ([]byte, string, string, error) { + fileName := document.GetFileName() + if fileName == "" { + fileName = "file" + exmime.ExtensionFromMimetype(mimeType) + } + return data, mimeType, fileName, nil + }) + if converted != nil { + converted.Content.MsgType = event.MsgFile + } + return +} + +func (mc *MessageConverter) convertWhatsAppAudio(ctx context.Context, audio *waConsumerApplication.ConsumerApplication_AudioMessage) (converted, caption *ConvertedMessagePart, err error) { + metadata, converted, caption, err := convertWhatsAppAttachment[*waMediaTransport.AudioTransport](ctx, mc, audio, whatsmeow.MediaAudio, func(ctx context.Context, data []byte, mimeType string) ([]byte, string, string, error) { + fileName := "audio" + exmime.ExtensionFromMimetype(mimeType) + if audio.GetPTT() && !strings.HasPrefix(mimeType, "audio/ogg") { + data, err = ffmpeg.ConvertBytes(ctx, data, ".ogg", []string{}, []string{"-c:a", "libopus"}, mimeType) + if err != nil { + return data, mimeType, fileName, fmt.Errorf("failed to convert audio to ogg/opus: %w", err) + } + fileName += ".ogg" + mimeType = "audio/ogg" + } + return data, mimeType, fileName, nil + }) + if converted != nil { + converted.Content.MsgType = event.MsgAudio + converted.Content.Info.Duration = int(metadata.GetSeconds() * 1000) + if audio.GetPTT() { + converted.Extra["org.matrix.msc3245.voice"] = map[string]any{} + converted.Extra["org.matrix.msc1767.audio"] = map[string]any{} + } + } + return +} + +func (mc *MessageConverter) convertWhatsAppVideo(ctx context.Context, video *waConsumerApplication.ConsumerApplication_VideoMessage) (converted, caption *ConvertedMessagePart, err error) { + metadata, converted, caption, err := convertWhatsAppAttachment[*waMediaTransport.VideoTransport](ctx, mc, video, whatsmeow.MediaVideo, func(ctx context.Context, data []byte, mimeType string) ([]byte, string, string, error) { + fileName := "video" + exmime.ExtensionFromMimetype(mimeType) + return data, mimeType, fileName, nil + }) + if converted != nil { + converted.Content.MsgType = event.MsgVideo + converted.Content.Info.Width = int(metadata.GetWidth()) + converted.Content.Info.Height = int(metadata.GetHeight()) + converted.Content.Info.Duration = int(metadata.GetSeconds() * 1000) + if metadata.GetGifPlayback() { + converted.Extra["info"] = map[string]any{ + "fi.mau.gif": true, + "fi.mau.loop": true, + "fi.mau.autoplay": true, + "fi.mau.hide_controls": true, + "fi.mau.no_audio": true, + } + } + } + return +} + +func (mc *MessageConverter) convertWhatsAppMedia(ctx context.Context, evt *events.FBConsumerMessage) (converted, caption *ConvertedMessagePart, err error) { + switch content := evt.Message.GetPayload().GetContent().GetContent().(type) { + case *waConsumerApplication.ConsumerApplication_Content_ImageMessage: + return mc.convertWhatsAppImage(ctx, content.ImageMessage) + case *waConsumerApplication.ConsumerApplication_Content_StickerMessage: + return mc.convertWhatsAppSticker(ctx, content.StickerMessage) + case *waConsumerApplication.ConsumerApplication_Content_ViewOnceMessage: + switch realContent := content.ViewOnceMessage.GetViewOnceContent().(type) { + case *waConsumerApplication.ConsumerApplication_ViewOnceMessage_ImageMessage: + return mc.convertWhatsAppImage(ctx, realContent.ImageMessage) + case *waConsumerApplication.ConsumerApplication_ViewOnceMessage_VideoMessage: + return mc.convertWhatsAppVideo(ctx, realContent.VideoMessage) + default: + return nil, nil, fmt.Errorf("unrecognized view once message type %T", realContent) + } + case *waConsumerApplication.ConsumerApplication_Content_DocumentMessage: + return mc.convertWhatsAppDocument(ctx, content.DocumentMessage) + case *waConsumerApplication.ConsumerApplication_Content_AudioMessage: + return mc.convertWhatsAppAudio(ctx, content.AudioMessage) + case *waConsumerApplication.ConsumerApplication_Content_VideoMessage: + return mc.convertWhatsAppVideo(ctx, content.VideoMessage) + default: + return nil, nil, fmt.Errorf("unrecognized media message type %T", content) + } +} + func (mc *MessageConverter) WhatsAppToMatrix(ctx context.Context, evt *events.FBConsumerMessage) *ConvertedMessage { cm := &ConvertedMessage{ Parts: make([]*ConvertedMessagePart, 0), @@ -89,25 +281,71 @@ func (mc *MessageConverter) WhatsAppToMatrix(ctx context.Context, evt *events.FB case *waConsumerApplication.ConsumerApplication_Content_MessageText: cm.Parts = append(cm.Parts, mc.whatsappTextToMatrix(ctx, content.MessageText)) case *waConsumerApplication.ConsumerApplication_Content_ExtendedTextMessage: + part := mc.whatsappTextToMatrix(ctx, content.ExtendedTextMessage.GetText()) // TODO convert url previews - cm.Parts = append(cm.Parts, mc.whatsappTextToMatrix(ctx, content.ExtendedTextMessage.GetText())) - case *waConsumerApplication.ConsumerApplication_Content_ImageMessage: - case *waConsumerApplication.ConsumerApplication_Content_StickerMessage: - case *waConsumerApplication.ConsumerApplication_Content_ViewOnceMessage: - case *waConsumerApplication.ConsumerApplication_Content_DocumentMessage: - case *waConsumerApplication.ConsumerApplication_Content_AudioMessage: - case *waConsumerApplication.ConsumerApplication_Content_VideoMessage: + cm.Parts = append(cm.Parts, part) + case *waConsumerApplication.ConsumerApplication_Content_ImageMessage, + *waConsumerApplication.ConsumerApplication_Content_StickerMessage, + *waConsumerApplication.ConsumerApplication_Content_ViewOnceMessage, + *waConsumerApplication.ConsumerApplication_Content_DocumentMessage, + *waConsumerApplication.ConsumerApplication_Content_AudioMessage, + *waConsumerApplication.ConsumerApplication_Content_VideoMessage: + converted, caption, err := mc.convertWhatsAppMedia(ctx, evt) + if err != nil { + zerolog.Ctx(ctx).Err(err).Msg("Failed to convert media message") + converted = &ConvertedMessagePart{ + Type: event.EventMessage, + Content: &event.MessageEventContent{ + MsgType: event.MsgNotice, + Body: "Failed to transfer media", + }, + } + } + cm.Parts = append(cm.Parts, converted) + if caption != nil { + cm.Parts = append(cm.Parts, caption) + } case *waConsumerApplication.ConsumerApplication_Content_LocationMessage: + cm.Parts = append(cm.Parts, &ConvertedMessagePart{ + Type: event.EventMessage, + Content: &event.MessageEventContent{ + MsgType: event.MsgLocation, + Body: content.LocationMessage.GetLocation().GetName() + "\n" + content.LocationMessage.GetAddress(), + GeoURI: fmt.Sprintf("geo:%f,%f", content.LocationMessage.GetLocation().GetDegreesLatitude(), content.LocationMessage.GetLocation().GetDegreesLongitude()), + }, + }) case *waConsumerApplication.ConsumerApplication_Content_LiveLocationMessage: + cm.Parts = append(cm.Parts, &ConvertedMessagePart{ + Type: event.EventMessage, + Content: &event.MessageEventContent{ + MsgType: event.MsgLocation, + Body: "Live location sharing started", + GeoURI: fmt.Sprintf("geo:%f,%f", content.LiveLocationMessage.GetLocation().GetDegreesLatitude(), content.LiveLocationMessage.GetLocation().GetDegreesLongitude()), + }, + }) case *waConsumerApplication.ConsumerApplication_Content_ContactMessage: + cm.Parts = append(cm.Parts, &ConvertedMessagePart{ + Type: event.EventMessage, + Content: &event.MessageEventContent{ + MsgType: event.MsgNotice, + Body: "Unsupported message (contact)", + }, + }) case *waConsumerApplication.ConsumerApplication_Content_ContactsArrayMessage: - } - if len(cm.Parts) == 0 { cm.Parts = append(cm.Parts, &ConvertedMessagePart{ Type: event.EventMessage, Content: &event.MessageEventContent{ MsgType: event.MsgNotice, - Body: "Unsupported message", + Body: "Unsupported message (contacts array)", + }, + }) + default: + zerolog.Ctx(ctx).Warn().Type("content_type", content).Msg("Unrecognized content type") + cm.Parts = append(cm.Parts, &ConvertedMessagePart{ + Type: event.EventMessage, + Content: &event.MessageEventContent{ + MsgType: event.MsgNotice, + Body: "Unsupported message (unknown type)", }, }) } diff --git a/msgconv/msgconv.go b/msgconv/msgconv.go index 6f1a54a..04ba50e 100644 --- a/msgconv/msgconv.go +++ b/msgconv/msgconv.go @@ -19,6 +19,7 @@ package msgconv import ( "context" + "go.mau.fi/whatsmeow" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" @@ -36,6 +37,7 @@ type PortalMethods interface { ShouldFetchXMA(ctx context.Context) bool GetClient(ctx context.Context) *messagix.Client + GetE2EEClient(ctx context.Context) *whatsmeow.Client GetData(ctx context.Context) *database.Portal } diff --git a/portal.go b/portal.go index 4bae1d3..a2930f3 100644 --- a/portal.go +++ b/portal.go @@ -864,6 +864,10 @@ func (portal *Portal) GetClient(ctx context.Context) *messagix.Client { return ctx.Value(msgconvContextKeyClient).(*messagix.Client) } +func (portal *Portal) GetE2EEClient(ctx context.Context) *whatsmeow.Client { + return ctx.Value(msgconvContextKeyE2EEClient).(*whatsmeow.Client) +} + func (portal *Portal) GetMatrixReply(ctx context.Context, replyToID string, replyToUser int64) (replyTo id.EventID, replyTargetSender id.UserID) { if replyToID == "" { return From b2f9ae1732a56fc1ffc651abdf59656af4840016 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 8 Feb 2024 14:25:03 +0200 Subject: [PATCH 06/19] Add support for outgoing WhatsApp media and replies --- messagix/socket/threads.go | 1 + msgconv/from-matrix.go | 22 +++- msgconv/from-whatsapp.go | 2 +- msgconv/to-whatsapp.go | 263 ++++++++++++++++++++++++++++++++++++- portal.go | 7 +- 5 files changed, 280 insertions(+), 15 deletions(-) diff --git a/messagix/socket/threads.go b/messagix/socket/threads.go index 828e461..01804a3 100644 --- a/messagix/socket/threads.go +++ b/messagix/socket/threads.go @@ -41,6 +41,7 @@ type ReplyMetaData struct { ReplyMessageId string `json:"reply_source_id"` ReplySourceType int64 `json:"reply_source_type"` // 1 ? ReplyType int64 `json:"reply_type"` // ? + ReplySender int64 `json:"-"` } type MentionData struct { diff --git a/msgconv/from-matrix.go b/msgconv/from-matrix.go index 0073a9f..d46cced 100644 --- a/msgconv/from-matrix.go +++ b/msgconv/from-matrix.go @@ -101,32 +101,42 @@ func (mc *MessageConverter) ToMeta(ctx context.Context, evt *event.Event, conten return []socket.Task{task, readTask}, task.Otid, nil } -func (mc *MessageConverter) reuploadFileToMeta(ctx context.Context, evt *event.Event, content *event.MessageEventContent) (*types.MercuryUploadResponse, error) { +func (mc *MessageConverter) downloadMatrixMedia(ctx context.Context, content *event.MessageEventContent) (data []byte, mimeType, fileName string, err error) { mxc := content.URL if content.File != nil { mxc = content.File.URL } - data, err := mc.DownloadMatrixMedia(ctx, mxc) + data, err = mc.DownloadMatrixMedia(ctx, mxc) if err != nil { - return nil, exerrors.NewDualError(ErrMediaDownloadFailed, err) + err = exerrors.NewDualError(ErrMediaDownloadFailed, err) + return } if content.File != nil { err = content.File.DecryptInPlace(data) if err != nil { - return nil, exerrors.NewDualError(ErrMediaDecryptFailed, err) + err = exerrors.NewDualError(ErrMediaDecryptFailed, err) + return } } - mimeType := content.GetInfo().MimeType + mimeType = content.GetInfo().MimeType if mimeType == "" { mimeType = http.DetectContentType(data) } - fileName := content.FileName + fileName = content.FileName if fileName == "" { fileName = content.Body if fileName == "" { fileName = string(content.MsgType)[2:] + exmime.ExtensionFromMimetype(mimeType) } } + return +} + +func (mc *MessageConverter) reuploadFileToMeta(ctx context.Context, evt *event.Event, content *event.MessageEventContent) (*types.MercuryUploadResponse, error) { + data, mimeType, fileName, err := mc.downloadMatrixMedia(ctx, content) + if err != nil { + return nil, err + } _, isVoice := evt.Content.Raw["org.matrix.msc3245.voice"] if isVoice { data, err = ffmpeg.ConvertBytes(ctx, data, ".wav", []string{}, []string{"-c:a", "pcm_u8", "-ar", "48000"}, mimeType) diff --git a/msgconv/from-whatsapp.go b/msgconv/from-whatsapp.go index f1c6e74..06b4386 100644 --- a/msgconv/from-whatsapp.go +++ b/msgconv/from-whatsapp.go @@ -144,7 +144,7 @@ func (mc *MessageConverter) reuploadWhatsAppAttachment( var fileName string mimeType := transport.GetAncillary().GetMimetype() if convert != nil { - data, fileName, mimeType, err = convert(ctx, data, mimeType) + data, mimeType, fileName, err = convert(ctx, data, mimeType) if err != nil { return nil, fmt.Errorf("failed to convert: %w", err) } diff --git a/msgconv/to-whatsapp.go b/msgconv/to-whatsapp.go index f37f12e..1dbc3d1 100644 --- a/msgconv/to-whatsapp.go +++ b/msgconv/to-whatsapp.go @@ -17,16 +17,34 @@ package msgconv import ( + "bytes" "context" "fmt" + "image" + "strconv" + "strings" + "time" + "go.mau.fi/util/ffmpeg" + "go.mau.fi/whatsmeow" + "go.mau.fi/whatsmeow/binary/armadillo/waMediaTransport" + "go.mau.fi/whatsmeow/binary/armadillo/waMsgApplication" + "go.mau.fi/whatsmeow/types" "maunium.net/go/mautrix/event" "go.mau.fi/whatsmeow/binary/armadillo/waCommon" "go.mau.fi/whatsmeow/binary/armadillo/waConsumerApplication" ) -func (mc *MessageConverter) ToWhatsApp(ctx context.Context, evt *event.Event, content *event.MessageEventContent, relaybotFormatted bool) (*waConsumerApplication.ConsumerApplication, error) { +func (mc *MessageConverter) ToWhatsApp( + ctx context.Context, + evt *event.Event, + content *event.MessageEventContent, + relaybotFormatted bool, +) (*waConsumerApplication.ConsumerApplication, *waMsgApplication.MessageApplication_Metadata, error) { + if evt.Type == event.EventSticker { + content.MsgType = event.MessageType(event.EventSticker.Type) + } if content.MsgType == event.MsgEmote && !relaybotFormatted { content.Body = "/me " + content.Body if content.FormattedBody != "" { @@ -42,12 +60,49 @@ func (mc *MessageConverter) ToWhatsApp(ctx context.Context, evt *event.Event, co Text: content.Body, }, } - case event.MsgImage, event.MsgVideo, event.MsgAudio, event.MsgFile: - fallthrough + case event.MsgImage, event.MsgVideo, event.MsgAudio, event.MsgFile, event.MessageType(event.EventSticker.Type): + reuploaded, fileName, err := mc.reuploadMediaToWhatsApp(ctx, evt, content) + if err != nil { + return nil, nil, err + } + var caption waCommon.MessageText + if content.FileName != "" && content.Body != content.FileName { + // TODO also mentions here + caption.Text = content.Body + } + waContent.Content, err = mc.wrapWhatsAppMedia(evt, content, reuploaded, &caption, fileName) + if err != nil { + return nil, nil, err + } case event.MsgLocation: - fallthrough + lat, long, err := parseGeoURI(content.GeoURI) + if err != nil { + return nil, nil, err + } + // TODO does this actually work with any of the messenger clients? + waContent.Content = &waConsumerApplication.ConsumerApplication_Content_LocationMessage{ + LocationMessage: &waConsumerApplication.ConsumerApplication_LocationMessage{ + Location: &waConsumerApplication.ConsumerApplication_Location{ + DegreesLatitude: lat, + DegreesLongitude: long, + Name: content.Body, + }, + Address: "Earth", + }, + } default: - return nil, fmt.Errorf("%w %s", ErrUnsupportedMsgType, content.MsgType) + return nil, nil, fmt.Errorf("%w %s", ErrUnsupportedMsgType, content.MsgType) + } + var meta waMsgApplication.MessageApplication_Metadata + if replyTo := mc.GetMetaReply(ctx, content); replyTo != nil { + meta.QuotedMessage = &waMsgApplication.MessageApplication_Metadata_QuotedMessage{ + StanzaID: replyTo.ReplyMessageId, + RemoteJID: mc.GetData(ctx).JID().String(), + // TODO: this is hacky since it hardcodes the server + // TODO 2: should this be included for DMs? + Participant: types.JID{User: strconv.FormatInt(replyTo.ReplySender, 10), Server: types.MessengerServer}.String(), + Payload: nil, + } } return &waConsumerApplication.ConsumerApplication{ Payload: &waConsumerApplication.ConsumerApplication_Payload{ @@ -56,5 +111,201 @@ func (mc *MessageConverter) ToWhatsApp(ctx context.Context, evt *event.Event, co }, }, Metadata: nil, - }, nil + }, &meta, nil +} + +func parseGeoURI(uri string) (lat, long float64, err error) { + if !strings.HasPrefix(uri, "geo:") { + err = fmt.Errorf("uri doesn't have geo: prefix") + return + } + // Remove geo: prefix and anything after ; + coordinates := strings.Split(strings.TrimPrefix(uri, "geo:"), ";")[0] + + if splitCoordinates := strings.Split(coordinates, ","); len(splitCoordinates) != 2 { + err = fmt.Errorf("didn't find exactly two numbers separated by a comma") + } else if lat, err = strconv.ParseFloat(splitCoordinates[0], 64); err != nil { + err = fmt.Errorf("latitude is not a number: %w", err) + } else if long, err = strconv.ParseFloat(splitCoordinates[1], 64); err != nil { + err = fmt.Errorf("longitude is not a number: %w", err) + } + return +} + +func clampTo400(w, h int) (int, int) { + if w > 400 { + h = h * 400 / w + w = 400 + } + if h > 400 { + w = w * 400 / h + h = 400 + } + return w, h +} + +func (mc *MessageConverter) reuploadMediaToWhatsApp(ctx context.Context, evt *event.Event, content *event.MessageEventContent) (*waMediaTransport.WAMediaTransport, string, error) { + data, mimeType, fileName, err := mc.downloadMatrixMedia(ctx, content) + if err != nil { + return nil, "", err + } + _, isVoice := evt.Content.Raw["org.matrix.msc3245.voice"] + if isVoice { + data, err = ffmpeg.ConvertBytes(ctx, data, ".m4a", []string{}, []string{"-c:a", "aac"}, mimeType) + if err != nil { + return nil, "", fmt.Errorf("%w voice message to m4a: %w", ErrMediaConvertFailed, err) + } + mimeType = "audio/mp4" + fileName += ".m4a" + } else if mimeType == "image/gif" && content.MsgType == event.MsgImage { + data, err = ffmpeg.ConvertBytes(ctx, data, ".mp4", []string{"-f", "gif"}, []string{ + "-pix_fmt", "yuv420p", "-c:v", "libx264", "-movflags", "+faststart", + "-filter:v", "crop='floor(in_w/2)*2:floor(in_h/2)*2'", + }, mimeType) + if err != nil { + return nil, "", fmt.Errorf("%w gif to mp4: %w", ErrMediaConvertFailed, err) + } + mimeType = "video/mp4" + fileName += ".mp4" + content.MsgType = event.MsgVideo + customInfo, ok := evt.Content.Raw["info"].(map[string]any) + if !ok { + customInfo = make(map[string]any) + evt.Content.Raw["info"] = customInfo + } + customInfo["fi.mau.gif"] = true + } + if content.MsgType == event.MsgImage && content.Info.Width == 0 { + cfg, _, _ := image.DecodeConfig(bytes.NewReader(data)) + content.Info.Width, content.Info.Height = cfg.Width, cfg.Height + } + mediaType := msgToMediaType(content.MsgType) + uploaded, err := mc.GetE2EEClient(ctx).Upload(ctx, data, mediaType) + if err != nil { + return nil, "", err + } + w, h := clampTo400(content.Info.Width, content.Info.Height) + if w == 0 && content.MsgType == event.MsgImage { + w, h = 400, 400 + } + mediaTransport := &waMediaTransport.WAMediaTransport{ + Integral: &waMediaTransport.WAMediaTransport_Integral{ + FileSHA256: uploaded.FileSHA256, + MediaKey: uploaded.MediaKey, + FileEncSHA256: uploaded.FileEncSHA256, + DirectPath: uploaded.DirectPath, + MediaKeyTimestamp: time.Now().Unix(), + }, + Ancillary: &waMediaTransport.WAMediaTransport_Ancillary{ + FileLength: uint64(len(data)), + Mimetype: mimeType, + // This field is extremely required for some reason. + // Messenger iOS & Android will refuse to display the media if it's not present. + // iOS also requires that width and height are non-empty. + Thumbnail: &waMediaTransport.WAMediaTransport_Ancillary_Thumbnail{ + ThumbnailWidth: uint32(w), + ThumbnailHeight: uint32(h), + }, + ObjectID: uploaded.ObjectID, + }, + } + fmt.Printf("Uploaded media transport: %+v\n", mediaTransport) + return mediaTransport, fileName, nil +} + +func (mc *MessageConverter) wrapWhatsAppMedia( + evt *event.Event, + content *event.MessageEventContent, + reuploaded *waMediaTransport.WAMediaTransport, + caption *waCommon.MessageText, + fileName string, +) (output waConsumerApplication.ConsumerApplication_Content_Content, err error) { + switch content.MsgType { + case event.MsgImage: + imageMsg := &waConsumerApplication.ConsumerApplication_ImageMessage{ + Caption: caption, + } + err = imageMsg.Set(&waMediaTransport.ImageTransport{ + Integral: &waMediaTransport.ImageTransport_Integral{ + Transport: reuploaded, + }, + Ancillary: &waMediaTransport.ImageTransport_Ancillary{ + Height: uint32(content.Info.Height), + Width: uint32(content.Info.Width), + }, + }) + output = &waConsumerApplication.ConsumerApplication_Content_ImageMessage{ImageMessage: imageMsg} + case event.MessageType(event.EventSticker.Type): + stickerMsg := &waConsumerApplication.ConsumerApplication_StickerMessage{} + err = stickerMsg.Set(&waMediaTransport.StickerTransport{ + Integral: &waMediaTransport.StickerTransport_Integral{ + Transport: reuploaded, + }, + Ancillary: &waMediaTransport.StickerTransport_Ancillary{ + Height: uint32(content.Info.Height), + Width: uint32(content.Info.Width), + }, + }) + output = &waConsumerApplication.ConsumerApplication_Content_StickerMessage{StickerMessage: stickerMsg} + case event.MsgVideo: + videoMsg := &waConsumerApplication.ConsumerApplication_VideoMessage{ + Caption: caption, + } + customInfo, _ := evt.Content.Raw["info"].(map[string]any) + isGif, _ := customInfo["fi.mau.gif"].(bool) + + err = videoMsg.Set(&waMediaTransport.VideoTransport{ + Integral: &waMediaTransport.VideoTransport_Integral{ + Transport: reuploaded, + }, + Ancillary: &waMediaTransport.VideoTransport_Ancillary{ + Height: uint32(content.Info.Height), + Width: uint32(content.Info.Width), + Seconds: uint32(content.Info.Duration / 1000), + GifPlayback: isGif, + }, + }) + output = &waConsumerApplication.ConsumerApplication_Content_VideoMessage{VideoMessage: videoMsg} + case event.MsgAudio: + _, isVoice := evt.Content.Raw["org.matrix.msc3245.voice"] + audioMsg := &waConsumerApplication.ConsumerApplication_AudioMessage{ + PTT: isVoice, + } + err = audioMsg.Set(&waMediaTransport.AudioTransport{ + Integral: &waMediaTransport.AudioTransport_Integral{ + Transport: reuploaded, + }, + Ancillary: &waMediaTransport.AudioTransport_Ancillary{ + Seconds: uint32(content.Info.Duration / 1000), + }, + }) + output = &waConsumerApplication.ConsumerApplication_Content_AudioMessage{AudioMessage: audioMsg} + case event.MsgFile: + documentMsg := &waConsumerApplication.ConsumerApplication_DocumentMessage{ + FileName: fileName, + } + err = documentMsg.Set(&waMediaTransport.DocumentTransport{ + Integral: &waMediaTransport.DocumentTransport_Integral{ + Transport: reuploaded, + }, + Ancillary: &waMediaTransport.DocumentTransport_Ancillary{}, + }) + output = &waConsumerApplication.ConsumerApplication_Content_DocumentMessage{DocumentMessage: documentMsg} + } + return +} + +func msgToMediaType(msgType event.MessageType) whatsmeow.MediaType { + switch msgType { + case event.MsgImage, event.MessageType(event.EventSticker.Type): + return whatsmeow.MediaImage + case event.MsgVideo: + return whatsmeow.MediaVideo + case event.MsgAudio: + return whatsmeow.MediaAudio + case event.MsgFile: + fallthrough + default: + return whatsmeow.MediaDocument + } } diff --git a/portal.go b/portal.go index a2930f3..de97796 100644 --- a/portal.go +++ b/portal.go @@ -31,6 +31,7 @@ import ( "go.mau.fi/util/variationselector" "go.mau.fi/whatsmeow" "go.mau.fi/whatsmeow/binary/armadillo/waConsumerApplication" + "go.mau.fi/whatsmeow/binary/armadillo/waMsgApplication" "go.mau.fi/whatsmeow/types" "go.mau.fi/whatsmeow/types/events" "maunium.net/go/mautrix" @@ -513,10 +514,11 @@ func (portal *Portal) handleMatrixMessage(ctx context.Context, sender *User, evt var otid int64 var tasks []socket.Task var waMsg *waConsumerApplication.ConsumerApplication + var waMeta *waMsgApplication.MessageApplication_Metadata var err error if portal.ThreadType.IsWhatsApp() { ctx = context.WithValue(ctx, msgconvContextKeyE2EEClient, sender.E2EEClient) - waMsg, err = portal.MsgConv.ToWhatsApp(ctx, evt, content, relaybotFormatted) + waMsg, waMeta, err = portal.MsgConv.ToWhatsApp(ctx, evt, content, relaybotFormatted) } else { ctx = context.WithValue(ctx, msgconvContextKeyClient, sender.Client) tasks, otid, err = portal.MsgConv.ToMeta(ctx, evt, content, relaybotFormatted) @@ -533,7 +535,7 @@ func (portal *Portal) handleMatrixMessage(ctx context.Context, sender *User, evt if waMsg != nil { messageID := sender.E2EEClient.GenerateMessageID() var resp whatsmeow.SendResponse - resp, err = sender.E2EEClient.SendFBMessage(ctx, portal.JID(), waMsg, nil, whatsmeow.SendRequestExtra{ + resp, err = sender.E2EEClient.SendFBMessage(ctx, portal.JID(), waMsg, waMeta, whatsmeow.SendRequestExtra{ ID: messageID, }) // TODO save message in db before sending and only update timestamp later @@ -922,6 +924,7 @@ func (portal *Portal) GetMetaReply(ctx context.Context, content *event.MessageEv ReplyMessageId: replyToMsg.ID, ReplySourceType: 1, ReplyType: 0, + ReplySender: replyToMsg.Sender, } } return nil From a0d2db59ed84c2b078b4033043595fb17f0a9495 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 16:47:06 +0200 Subject: [PATCH 07/19] Update roadmap --- .editorconfig | 2 ++ ROADMAP.md | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 39eb3d4..35f30c8 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,6 +10,8 @@ insert_final_newline = true [*.md] trim_trailing_whitespace = false +indent_size = 2 +indent_style = space [*.{yaml,yml,sql}] indent_style = space diff --git a/ROADMAP.md b/ROADMAP.md index 28875c9..612e4c8 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -39,6 +39,7 @@ * [x] Files * [x] Voice messages * [x] Locations + * [ ] Polls * [ ] Live location sharing * [x] Story/reel/clip shares * [x] Profile shares @@ -67,10 +68,81 @@ * [x] Name * [ ] Per-chat nickname * [x] Avatar +* Matrix → WhatsApp + * [ ] Message content + * [x] Text + * [ ] Media + * [x] Files + * [x] Voice messages + * [x] Videos + * [x] Images + * [x] Stickers + * [x] Gifs + * [ ] Locations + * [ ] Polls + * [ ] Formatting (Messenger only) + * [x] Replies + * [ ] Mentions + * [ ] Message redactions + * [ ] Message reactions + * [ ] Message edits + * [ ] Writing to chat backup + * [ ] Presence + * [ ] Typing notifications + * [ ] Read receipts + * [ ] Power level + * [ ] Membership actions + * [ ] Invite + * [ ] Kick + * [ ] Leave + * [ ] Room metadata changes + * [ ] Name + * [ ] Avatar + * [ ] Per-room user nick +* WhatsApp → Matrix + * [ ] Message content + * [x] Text + * [ ] Media + * [x] Images + * [x] Videos + * [x] Gifs + * [x] Stickers + * [x] Files + * [x] Voice messages + * [x] Locations + * [ ] Polls + * [ ] Live location sharing + * [ ] Story/reel/clip shares + * [ ] Profile shares + * [ ] Product shares + * [ ] Formatting (Messenger only) + * [x] Replies + * [x] Mentions + * [ ] Polls + * [ ] Message unsend + * [ ] Message reactions + * [ ] Message edits + * [ ] Message history/Reading chat backup + * [ ] Presence + * [ ] Typing notifications + * [ ] Read receipts + * [ ] Admin status + * [ ] Membership actions + * [ ] Add member + * [ ] Remove member + * [ ] Leave + * [ ] Chat metadata changes + * [ ] Title + * [ ] Avatar + * [ ] Initial chat metadata + * [ ] User metadata + * [ ] Name + * [ ] Per-chat nickname + * [ ] Avatar * Misc * [x] Multi-user support * [x] Shared group chat portals - * [ ] Messenger encryption + * [x] Messenger encryption * [x] Matrix encryption * [x] Automatic portal creation * [x] At startup From 8f3725900262c324ad0613d9d0aab47f34ba6171 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 18:07:43 +0200 Subject: [PATCH 08/19] Add support for outgoing read receipts, edits, reactions and redactions to WhatsApp --- ROADMAP.md | 8 +- database/message.go | 9 + database/upgrades/00-latest.sql | 3 +- .../upgrades/06-user-portal-last-read.sql | 2 + database/user.go | 10 +- database/userportal.go | 49 ++++ msgconv/to-whatsapp.go | 21 +- portal.go | 273 ++++++++++++++---- user.go | 4 + 9 files changed, 313 insertions(+), 66 deletions(-) create mode 100644 database/upgrades/06-user-portal-last-read.sql diff --git a/ROADMAP.md b/ROADMAP.md index 612e4c8..f7abde6 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -83,13 +83,13 @@ * [ ] Formatting (Messenger only) * [x] Replies * [ ] Mentions - * [ ] Message redactions - * [ ] Message reactions - * [ ] Message edits + * [x] Message redactions + * [x] Message reactions + * [x] Message edits * [ ] Writing to chat backup * [ ] Presence * [ ] Typing notifications - * [ ] Read receipts + * [x] Read receipts * [ ] Power level * [ ] Membership actions * [ ] Invite diff --git a/database/message.go b/database/message.go index 2d61658..ec60eb5 100644 --- a/database/message.go +++ b/database/message.go @@ -51,6 +51,11 @@ const ( SELECT id, part_index, thread_id, thread_receiver, msg_sender, otid, mxid, mx_room, timestamp, edit_count FROM message WHERE id=$1 AND thread_receiver=$2 ` + getMessagesBetweenTimeQuery = ` + SELECT id, part_index, thread_id, thread_receiver, msg_sender, otid, mxid, mx_room, timestamp, edit_count FROM message + WHERE thread_id=$1 AND thread_receiver=$2 AND timestamp>$3 AND timestamp<=$4 AND part_index=0 + ORDER BY timestamp ASC + ` findEditTargetPortalFromMessageQuery = ` SELECT thread_id, thread_receiver FROM message WHERE id=$1 AND (thread_receiver=$2 OR thread_receiver=0) AND part_index=0 @@ -121,6 +126,10 @@ func (mq *MessageQuery) GetAllPartsByID(ctx context.Context, id string, receiver return mq.QueryMany(ctx, getAllMessagePartsByIDQuery, id, receiver) } +func (mq *MessageQuery) GetAllBetweenTimestamps(ctx context.Context, key PortalKey, min, max time.Time) ([]*Message, error) { + return mq.QueryMany(ctx, getMessagesBetweenTimeQuery, key.ThreadID, key.Receiver, min.UnixMilli(), max.UnixMilli()) +} + func (mq *MessageQuery) FindEditTargetPortal(ctx context.Context, id string, receiver int64) (key PortalKey, err error) { err = mq.GetDB().QueryRow(ctx, findEditTargetPortalFromMessageQuery, id, receiver).Scan(&key.ThreadID, &key.Receiver) if errors.Is(err, sql.ErrNoRows) { diff --git a/database/upgrades/00-latest.sql b/database/upgrades/00-latest.sql index 405e43b..38d6616 100644 --- a/database/upgrades/00-latest.sql +++ b/database/upgrades/00-latest.sql @@ -1,4 +1,4 @@ --- v0 -> v5 (compatible with v3+): Latest revision +-- v0 -> v6 (compatible with v3+): Latest revision CREATE TABLE portal ( thread_id BIGINT NOT NULL, @@ -62,6 +62,7 @@ CREATE TABLE user_portal ( portal_thread_id BIGINT NOT NULL, portal_receiver BIGINT NOT NULL, + last_read_ts BIGINT NOT NULL DEFAULT 0, in_space BOOLEAN NOT NULL DEFAULT false, PRIMARY KEY (user_mxid, portal_thread_id, portal_receiver), diff --git a/database/upgrades/06-user-portal-last-read.sql b/database/upgrades/06-user-portal-last-read.sql new file mode 100644 index 0000000..007d964 --- /dev/null +++ b/database/upgrades/06-user-portal-last-read.sql @@ -0,0 +1,2 @@ +-- v6 (compatible with v3+): Store last read timestamp for chats +ALTER TABLE user_portal ADD COLUMN last_read_ts BIGINT NOT NULL DEFAULT 0; diff --git a/database/user.go b/database/user.go index 0dc9db8..4c60931 100644 --- a/database/user.go +++ b/database/user.go @@ -20,6 +20,7 @@ import ( "context" "database/sql" "sync" + "time" "go.mau.fi/util/dbutil" "maunium.net/go/mautrix/id" @@ -50,15 +51,18 @@ type User struct { ManagementRoom id.RoomID SpaceRoom id.RoomID - inSpaceCache map[PortalKey]bool - inSpaceCacheLock sync.Mutex + lastReadCache map[PortalKey]time.Time + lastReadCacheLock sync.Mutex + inSpaceCache map[PortalKey]bool + inSpaceCacheLock sync.Mutex } func newUser(qh *dbutil.QueryHelper[*User]) *User { return &User{ qh: qh, - inSpaceCache: make(map[PortalKey]bool), + lastReadCache: make(map[PortalKey]time.Time), + inSpaceCache: make(map[PortalKey]bool), } } diff --git a/database/userportal.go b/database/userportal.go index 8c488e2..2888e14 100644 --- a/database/userportal.go +++ b/database/userportal.go @@ -20,11 +20,18 @@ import ( "context" "database/sql" "errors" + "time" "github.com/rs/zerolog" ) const ( + getLastReadTSQuery = `SELECT last_read_ts FROM user_portal WHERE user_mxid=$1 AND portal_thread_id=$2 AND portal_receiver=$3` + setLastReadTSQuery = ` + INSERT INTO user_portal (user_mxid, portal_thread_id, portal_receiver, last_read_ts) VALUES ($1, $2, $3, $4) + ON CONFLICT (user_mxid, portal_thread_id, portal_receiver) DO UPDATE + SET last_read_ts=excluded.last_read_ts WHERE user_portal.last_read_ts 0 { + sender.SetLastReadTS(ctx, portal.PortalKey, messages[len(messages)-1].Timestamp) + } + groupedMessages := make(map[types.JID][]types.MessageID) + for _, msg := range messages { + var key types.JID + if msg.Sender == sender.MetaID { + // Don't send read receipts for own messages or fake messages + continue + } else if !portal.IsPrivateChat() { + // TODO: this is hacky since it hardcodes the server + key = types.JID{User: strconv.FormatInt(msg.Sender, 10), Server: types.MessengerServer} + } // else: blank key (participant field isn't needed in direct chat read receipts) + groupedMessages[key] = append(groupedMessages[key], msg.ID) + } + // For explicit read receipts, log even if there are no targets. For implicit ones only log when there are targets + if len(groupedMessages) > 0 || isExplicit { + log.Debug(). + Time("last_read", prevTimestamp). + Bool("last_read_was_zero", lastReadIsZero). + Bool("explicit", isExplicit). + Any("receipts", groupedMessages). + Msg("Sending read receipts") + } + for messageSender, ids := range groupedMessages { + err = sender.E2EEClient.MarkRead(ids, receiptTimestamp, portal.JID(), messageSender) + if err != nil { + log.Err(err).Strs("ids", ids).Msg("Failed to mark messages as read") + } + } +} + const MaxEditCount = 5 -func (portal *Portal) handleMatrixMessage(ctx context.Context, sender *User, evt *event.Event) { +func (portal *Portal) handleMatrixMessage(ctx context.Context, sender *User, evt *event.Event, timings messageTimings) { log := zerolog.Ctx(ctx) - evtTS := time.UnixMilli(evt.Timestamp) - timings := messageTimings{ - initReceive: evt.Mautrix.ReceivedAt.Sub(evtTS), - decrypt: evt.Mautrix.DecryptionDuration, - totalReceive: time.Since(evtTS), - } - implicitRRStart := time.Now() - timings.implicitRR = time.Since(implicitRRStart) start := time.Now() messageAge := timings.totalReceive @@ -488,6 +571,7 @@ func (portal *Portal) handleMatrixMessage(ctx context.Context, sender *User, evt realSenderMXID := sender.MXID isRelay := false + // TODO check login for correct client (e2ee vs not e2ee) if !sender.IsLoggedIn() { sender = portal.GetRelayUser() if sender == nil { @@ -501,11 +585,6 @@ func (portal *Portal) handleMatrixMessage(ctx context.Context, sender *User, evt } if editTarget := content.RelatesTo.GetReplaceID(); editTarget != "" { - if portal.ThreadType.IsWhatsApp() { - // TODO implement - go ms.sendMessageMetrics(evt, fmt.Errorf("whatsapp edits aren't supported yet"), "Ignoring", true) - return - } portal.handleMatrixEdit(ctx, sender, isRelay, realSenderMXID, &ms, evt, content) return } @@ -609,12 +688,24 @@ func (portal *Portal) handleMatrixEdit(ctx context.Context, sender *User, isRela if isRelay { portal.addRelaybotFormat(ctx, realSenderMXID, evt, content) } - editTask := &socket.EditMessageTask{ - MessageID: editTargetMsg.ID, - Text: content.Body, + if portal.ThreadType.IsWhatsApp() { + consumerMsg := wrapEdit(&waConsumerApplication.ConsumerApplication_EditMessage{ + Key: portal.buildMessageKey(sender, editTargetMsg), + Message: portal.MsgConv.TextToWhatsApp(content), + TimestampMS: evt.Timestamp, + }) + var resp whatsmeow.SendResponse + resp, err = sender.E2EEClient.SendFBMessage(ctx, portal.JID(), consumerMsg, nil) + log.Trace().Any("response", resp).Msg("WhatsApp delete response") + } else { + editTask := &socket.EditMessageTask{ + MessageID: editTargetMsg.ID, + Text: content.Body, + } + var resp *table.LSTable + resp, err = sender.Client.ExecuteTasks(editTask) + log.Trace().Any("response", resp).Msg("Meta edit response") } - resp, err := sender.Client.ExecuteTasks(editTask) - log.Trace().Any("response", resp).Msg("Meta edit response") go ms.sendMessageMetrics(evt, err, "Error sending", true) if err == nil { // TODO does the response contain the edit count? @@ -652,14 +743,24 @@ func (portal *Portal) handleMatrixRedaction(ctx context.Context, sender *User, e portal.sendMessageStatusCheckpointFailed(ctx, evt, errRedactionTargetSentBySomeoneElse) return } - resp, err := sender.Client.ExecuteTasks(&socket.DeleteMessageTask{MessageId: dbMessage.ID}) + if portal.ThreadType.IsWhatsApp() { + consumerMsg := wrapRevoke(&waConsumerApplication.ConsumerApplication_RevokeMessage{ + Key: portal.buildMessageKey(sender, dbMessage), + }) + var resp whatsmeow.SendResponse + resp, err = sender.E2EEClient.SendFBMessage(ctx, portal.JID(), consumerMsg, nil) + log.Trace().Any("response", resp).Msg("WhatsApp delete response") + } else { + var resp *table.LSTable + resp, err = sender.Client.ExecuteTasks(&socket.DeleteMessageTask{MessageId: dbMessage.ID}) + // TODO does the response data need to be checked? + log.Trace().Any("response", resp).Msg("Instagram delete response") + } if err != nil { portal.sendMessageStatusCheckpointFailed(ctx, evt, err) log.Err(err).Msg("Failed to send message redaction to Meta") return } - // TODO does the response data need to be checked? - log.Trace().Any("response", resp).Msg("Instagram delete response") err = dbMessage.Delete(ctx) if err != nil { log.Err(err).Msg("Failed to delete redacted message from database") @@ -693,22 +794,22 @@ func (portal *Portal) handleMatrixRedaction(ctx context.Context, sender *User, e portal.sendMessageStatusCheckpointFailed(ctx, evt, errUnreactTargetSentBySomeoneElse) return } - resp, err := sender.Client.ExecuteTasks(&socket.SendReactionTask{ - ThreadKey: portal.ThreadID, - TimestampMs: evt.Timestamp, - MessageID: dbReaction.MessageID, - ActorID: dbReaction.Sender, - Reaction: "", - SyncGroup: 1, - SendAttribution: table.MESSENGER_INBOX_IN_THREAD, - }) + targetMsg, err := portal.bridge.DB.Message.GetByID(ctx, dbReaction.MessageID, 0, portal.Receiver) + if err != nil { + portal.sendMessageStatusCheckpointFailed(ctx, evt, err) + log.Err(err).Msg("Failed to get removed reaction target message") + return + } else if targetMsg == nil { + portal.sendMessageStatusCheckpointFailed(ctx, evt, errReactionTargetNotFound) + log.Warn().Msg("Reaction target message not found") + return + } + err = portal.sendReaction(ctx, sender, targetMsg, "", evt.Timestamp) if err != nil { portal.sendMessageStatusCheckpointFailed(ctx, evt, err) log.Err(err).Msg("Failed to send reaction redaction to Meta") return } - // TODO does the response data need to be checked? - log.Trace().Any("response", resp).Msg("Instagram reaction delete response") err = dbReaction.Delete(ctx) if err != nil { log.Err(err).Msg("Failed to delete redacted reaction from database") @@ -719,6 +820,88 @@ func (portal *Portal) handleMatrixRedaction(ctx context.Context, sender *User, e } } +func wrapEdit(message *waConsumerApplication.ConsumerApplication_EditMessage) *waConsumerApplication.ConsumerApplication { + return &waConsumerApplication.ConsumerApplication{ + Payload: &waConsumerApplication.ConsumerApplication_Payload{ + Payload: &waConsumerApplication.ConsumerApplication_Payload_Content{ + Content: &waConsumerApplication.ConsumerApplication_Content{ + Content: &waConsumerApplication.ConsumerApplication_Content_EditMessage{ + EditMessage: message, + }, + }, + }, + }, + } +} + +func wrapRevoke(message *waConsumerApplication.ConsumerApplication_RevokeMessage) *waConsumerApplication.ConsumerApplication { + return &waConsumerApplication.ConsumerApplication{ + Payload: &waConsumerApplication.ConsumerApplication_Payload{ + Payload: &waConsumerApplication.ConsumerApplication_Payload_ApplicationData{ + ApplicationData: &waConsumerApplication.ConsumerApplication_ApplicationData{ + ApplicationContent: &waConsumerApplication.ConsumerApplication_ApplicationData_Revoke{ + Revoke: message, + }, + }, + }, + }, + } +} + +func wrapReaction(message *waConsumerApplication.ConsumerApplication_ReactionMessage) *waConsumerApplication.ConsumerApplication { + return &waConsumerApplication.ConsumerApplication{ + Payload: &waConsumerApplication.ConsumerApplication_Payload{ + Payload: &waConsumerApplication.ConsumerApplication_Payload_Content{ + Content: &waConsumerApplication.ConsumerApplication_Content{ + Content: &waConsumerApplication.ConsumerApplication_Content_ReactionMessage{ + ReactionMessage: message, + }, + }, + }, + }, + } +} + +func (portal *Portal) buildMessageKey(user *User, targetMsg *database.Message) *waCommon.MessageKey { + var messageKeyParticipant string + if !portal.IsPrivateChat() { + // TODO: this is hacky since it hardcodes the server + messageKeyParticipant = types.JID{User: strconv.FormatInt(targetMsg.Sender, 10), Server: types.MessengerServer}.String() + } + return &waCommon.MessageKey{ + RemoteJID: portal.JID().String(), + FromMe: targetMsg.Sender == user.MetaID, + ID: targetMsg.ID, + Participant: messageKeyParticipant, + } +} + +func (portal *Portal) sendReaction(ctx context.Context, sender *User, targetMsg *database.Message, metaEmoji string, timestamp int64) error { + if portal.ThreadType.IsWhatsApp() { + consumerMsg := wrapReaction(&waConsumerApplication.ConsumerApplication_ReactionMessage{ + Key: portal.buildMessageKey(sender, targetMsg), + Text: metaEmoji, + SenderTimestampMS: timestamp, + }) + resp, err := sender.E2EEClient.SendFBMessage(ctx, portal.JID(), consumerMsg, nil) + zerolog.Ctx(ctx).Trace().Any("response", resp).Msg("WhatsApp reaction response") + return err + } else { + resp, err := sender.Client.ExecuteTasks(&socket.SendReactionTask{ + ThreadKey: portal.ThreadID, + TimestampMs: timestamp, + MessageID: targetMsg.ID, + ActorID: sender.MetaID, + Reaction: metaEmoji, + SyncGroup: 1, + SendAttribution: table.MESSENGER_INBOX_IN_THREAD, + }) + // TODO save the hidden thread message from the response too? + zerolog.Ctx(ctx).Trace().Any("response", resp).Msg("Instagram reaction response") + return err + } +} + func (portal *Portal) handleMatrixReaction(ctx context.Context, sender *User, evt *event.Event) { log := zerolog.Ctx(ctx) if !sender.IsLoggedIn() { @@ -739,22 +922,12 @@ func (portal *Portal) handleMatrixReaction(ctx context.Context, sender *User, ev emoji := evt.Content.AsReaction().RelatesTo.Key metaEmoji := variationselector.Remove(emoji) - resp, err := sender.Client.ExecuteTasks(&socket.SendReactionTask{ - ThreadKey: portal.ThreadID, - TimestampMs: evt.Timestamp, - MessageID: targetMsg.ID, - ActorID: sender.MetaID, - Reaction: metaEmoji, - SyncGroup: 1, - SendAttribution: table.MESSENGER_INBOX_IN_THREAD, - }) + err = portal.sendReaction(ctx, sender, targetMsg, metaEmoji, evt.Timestamp) if err != nil { portal.sendMessageStatusCheckpointFailed(ctx, evt, err) - log.Error().Msg("Failed to send reaction") + log.Err(err).Msg("Failed to send reaction") return } - // TODO save the hidden thread message from the response too? - log.Trace().Any("response", resp).Msg("Instagram reaction response") dbReaction, err := portal.bridge.DB.Reaction.GetByID( ctx, targetMsg.ID, diff --git a/user.go b/user.go index f13e96f..0b7059c 100644 --- a/user.go +++ b/user.go @@ -208,6 +208,10 @@ func (user *User) IsLoggedIn() bool { return user.Client != nil } +func (user *User) IsE2EEConnected() bool { + return user.E2EEClient != nil && user.E2EEClient.IsConnected() +} + func (user *User) GetManagementRoomID() id.RoomID { return user.ManagementRoom } From 98763cfc2de8e0723ee0da5fb4282cd9bcaef684 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 18:31:08 +0200 Subject: [PATCH 09/19] Add support for incoming read receipts from WhatsApp --- ROADMAP.md | 2 +- portal.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ user.go | 5 +++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/ROADMAP.md b/ROADMAP.md index f7abde6..a7a8412 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -125,7 +125,7 @@ * [ ] Message history/Reading chat backup * [ ] Presence * [ ] Typing notifications - * [ ] Read receipts + * [x] Read receipts * [ ] Admin status * [ ] Membership actions * [ ] Add member diff --git a/portal.go b/portal.go index c6a8535..35b6ea4 100644 --- a/portal.go +++ b/portal.go @@ -1115,6 +1115,8 @@ func (portal *Portal) handleMetaMessage(portalMessage portalMetaMessage) { switch typedEvt := portalMessage.evt.(type) { case *events.FBConsumerMessage: portal.handleEncryptedMessage(portalMessage.user, typedEvt) + case *events.Receipt: + portal.handleWhatsAppReceipt(portalMessage.user, typedEvt) case *table.WrappedMessage: portal.handleMetaInsertMessage(portalMessage.user, typedEvt) case *table.UpsertMessages: @@ -1172,6 +1174,58 @@ func (portal *Portal) checkPendingMessage(ctx context.Context, messageID string, return true } +func (portal *Portal) handleWhatsAppReceipt(source *User, receipt *events.Receipt) { + if receipt.Type != types.ReceiptTypeRead && receipt.Type != types.ReceiptTypeReadSelf { + return + } + senderID := int64(receipt.Sender.UserInt()) + if senderID == 0 { + return + } + log := portal.log.With(). + Str("action", "handle whatsapp receipt"). + Stringer("chat_jid", receipt.Chat). + Stringer("receipt_sender_jid", receipt.Sender). + Strs("message_ids", receipt.MessageIDs). + Time("receipt_timestamp", receipt.Timestamp). + Logger() + ctx := log.WithContext(context.TODO()) + markAsRead := make([]*database.Message, 0, 1) + var bestTimestamp time.Time + for _, msgID := range receipt.MessageIDs { + msg, err := portal.bridge.DB.Message.GetLastPartByID(ctx, msgID, portal.Receiver) + if err != nil { + log.Err(err).Msg("Failed to get message from database") + } + if msg == nil { + continue + } + if msg.Timestamp.After(bestTimestamp) { + bestTimestamp = msg.Timestamp + markAsRead = append(markAsRead[:0], msg) + } else if msg != nil && msg.Timestamp.Equal(bestTimestamp) { + markAsRead = append(markAsRead, msg) + } + } + if senderID == source.MetaID { + if len(markAsRead) > 0 { + source.SetLastReadTS(ctx, portal.PortalKey, markAsRead[0].Timestamp) + } else { + source.SetLastReadTS(ctx, portal.PortalKey, receipt.Timestamp) + } + } + sender := portal.bridge.GetPuppetByID(senderID) + for _, msg := range markAsRead { + // TODO bridge read-self as m.read.private? + err := portal.SendReadReceipt(ctx, sender, msg.MXID) + if err != nil { + log.Err(err).Stringer("event_id", msg.MXID).Msg("Failed to mark event as read") + } else { + log.Debug().Stringer("event_id", msg.MXID).Msg("Marked event as read") + } + } +} + func (portal *Portal) handleEncryptedMessage(source *User, evt *events.FBConsumerMessage) { sender := portal.bridge.GetPuppetByID(int64(evt.Info.Sender.UserInt())) log := portal.log.With(). diff --git a/user.go b/user.go index 0b7059c..f812d88 100644 --- a/user.go +++ b/user.go @@ -868,6 +868,11 @@ func (user *User) e2eeEventHandler(rawEvt any) { } } portal.metaMessages <- portalMetaMessage{user: user, evt: evt} + case *events.Receipt: + portal := user.GetExistingPortalByThreadID(int64(evt.Chat.UserInt())) + if portal != nil { + portal.metaMessages <- portalMetaMessage{user: user, evt: evt} + } } } From 3c3a6ed4ba5a156f599e06eeef970012fd17c5f3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 19:21:55 +0200 Subject: [PATCH 10/19] Add support for incoming message edits from WhatsApp --- ROADMAP.md | 2 +- database/message.go | 8 ++++++++ msgconv/from-whatsapp.go | 8 ++++---- portal.go | 38 +++++++++++++++++++++++++++++++++++--- 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index a7a8412..bf17e31 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -121,7 +121,7 @@ * [ ] Polls * [ ] Message unsend * [ ] Message reactions - * [ ] Message edits + * [x] Message edits * [ ] Message history/Reading chat backup * [ ] Presence * [ ] Typing notifications diff --git a/database/message.go b/database/message.go index ec60eb5..bd1e553 100644 --- a/database/message.go +++ b/database/message.go @@ -220,3 +220,11 @@ func (msg *Message) UpdateEditCount(ctx context.Context, count int64) error { msg.EditCount = count return msg.qh.Exec(ctx, updateMessageEditCountQuery, msg.ID, msg.ThreadReceiver, msg.PartIndex, msg.EditCount) } + +func (msg *Message) EditTimestamp() int64 { + return msg.EditCount +} + +func (msg *Message) UpdateEditTimestamp(ctx context.Context, ts int64) error { + return msg.UpdateEditCount(ctx, ts) +} diff --git a/msgconv/from-whatsapp.go b/msgconv/from-whatsapp.go index 06b4386..7ee8536 100644 --- a/msgconv/from-whatsapp.go +++ b/msgconv/from-whatsapp.go @@ -39,7 +39,7 @@ import ( "maunium.net/go/mautrix/id" ) -func (mc *MessageConverter) whatsappTextToMatrix(ctx context.Context, text *waCommon.MessageText) *ConvertedMessagePart { +func (mc *MessageConverter) WhatsAppTextToMatrix(ctx context.Context, text *waCommon.MessageText) *ConvertedMessagePart { content := &event.MessageEventContent{ MsgType: event.MsgText, Body: text.GetText(), @@ -122,7 +122,7 @@ func convertWhatsAppAttachment[ } msgWithCaption, ok := msg.(AttachmentMessageWithCaption[Integral, Ancillary, Transport]) if ok && len(msgWithCaption.GetCaption().GetText()) > 0 { - caption = mc.whatsappTextToMatrix(ctx, msgWithCaption.GetCaption()) + caption = mc.WhatsAppTextToMatrix(ctx, msgWithCaption.GetCaption()) caption.Content.MsgType = event.MsgNotice } metadata = typedTransport.GetAncillary() @@ -279,9 +279,9 @@ func (mc *MessageConverter) WhatsAppToMatrix(ctx context.Context, evt *events.FB } switch content := evt.Message.GetPayload().GetContent().GetContent().(type) { case *waConsumerApplication.ConsumerApplication_Content_MessageText: - cm.Parts = append(cm.Parts, mc.whatsappTextToMatrix(ctx, content.MessageText)) + cm.Parts = append(cm.Parts, mc.WhatsAppTextToMatrix(ctx, content.MessageText)) case *waConsumerApplication.ConsumerApplication_Content_ExtendedTextMessage: - part := mc.whatsappTextToMatrix(ctx, content.ExtendedTextMessage.GetText()) + part := mc.WhatsAppTextToMatrix(ctx, content.ExtendedTextMessage.GetText()) // TODO convert url previews cm.Parts = append(cm.Parts, part) case *waConsumerApplication.ConsumerApplication_Content_ImageMessage, diff --git a/portal.go b/portal.go index 35b6ea4..6adf422 100644 --- a/portal.go +++ b/portal.go @@ -676,7 +676,7 @@ func (portal *Portal) handleMatrixEdit(ctx context.Context, sender *User, isRela } else if editTargetMsg.Sender != sender.MetaID { go ms.sendMessageMetrics(evt, errEditDifferentSender, "Error converting", true) return - } else if editTargetMsg.EditCount >= MaxEditCount { + } else if !portal.ThreadType.IsWhatsApp() && editTargetMsg.EditCount >= MaxEditCount { go ms.sendMessageMetrics(evt, errEditCountExceeded, "Error converting", true) return } @@ -1233,14 +1233,15 @@ func (portal *Portal) handleEncryptedMessage(source *User, evt *events.FBConsume Stringer("chat_jid", evt.Info.Chat). Stringer("sender_jid", evt.Info.Sender). Str("message_id", evt.Info.ID). + Time("message_ts", evt.Info.Timestamp). Logger() ctx := log.WithContext(context.TODO()) switch payload := evt.Message.GetPayload().GetPayload().(type) { case *waConsumerApplication.ConsumerApplication_Payload_Content: - switch payload.Content.GetContent().(type) { + switch content := payload.Content.GetContent().(type) { case *waConsumerApplication.ConsumerApplication_Content_EditMessage: - log.Warn().Msg("Unsupported edit message payload message") + portal.handleWhatsAppEditMessage(ctx, sender, content.EditMessage) case *waConsumerApplication.ConsumerApplication_Content_ReactionMessage: log.Warn().Msg("Unsupported reaction message payload message") default: @@ -1335,6 +1336,37 @@ func (portal *Portal) handleMetaOrWhatsAppMessage(ctx context.Context, source *U } } +func (portal *Portal) handleWhatsAppEditMessage(ctx context.Context, sender *Puppet, edit *waConsumerApplication.ConsumerApplication_EditMessage) { + log := zerolog.Ctx(ctx).With(). + Int64("edit_ts", edit.TimestampMS). + Logger() + ctx = log.WithContext(context.TODO()) + targetMsg, err := portal.bridge.DB.Message.GetAllPartsByID(ctx, edit.GetKey().GetID(), portal.Receiver) + if err != nil { + log.Err(err).Msg("Failed to get edit target message") + return + } else if len(targetMsg) == 0 { + log.Warn().Msg("Edit target message not found") + return + } else if len(targetMsg) > 1 { + log.Warn().Msg("Ignoring edit of multipart message") + return + } else if targetMsg[0].EditTimestamp() <= edit.TimestampMS { + log.Debug().Int64("existing_edit_ts", targetMsg[0].EditTimestamp()).Msg("Ignoring duplicate edit") + return + } + converted := portal.MsgConv.WhatsAppTextToMatrix(ctx, edit.GetMessage()) + converted.Content.SetEdit(targetMsg[0].MXID) + resp, err := portal.sendMatrixEvent(ctx, sender.IntentFor(portal), converted.Type, converted.Content, converted.Extra, edit.TimestampMS) + if err != nil { + log.Err(err).Msg("Failed to send edit to Matrix") + } else if err := targetMsg[0].UpdateEditTimestamp(ctx, edit.TimestampMS); err != nil { + log.Err(err).Stringer("event_id", resp.EventID).Msg("Failed to save message edit count to database") + } else { + log.Debug().Stringer("event_id", resp.EventID).Msg("Handled Meta message edit") + } +} + func (portal *Portal) handleMetaEditMessage(edit *table.LSEditMessage) { log := portal.log.With(). Str("action", "edit meta message"). From 71dee6971b0fff82b5cb4a817f7fbd248fd91e48 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 19:28:14 +0200 Subject: [PATCH 11/19] Add support for incoming message reactions from WhatsApp --- ROADMAP.md | 2 +- portal.go | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index bf17e31..aaa2368 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -120,7 +120,7 @@ * [x] Mentions * [ ] Polls * [ ] Message unsend - * [ ] Message reactions + * [x] Message reactions * [x] Message edits * [ ] Message history/Reading chat backup * [ ] Presence diff --git a/portal.go b/portal.go index 6adf422..12afcb4 100644 --- a/portal.go +++ b/portal.go @@ -1243,7 +1243,7 @@ func (portal *Portal) handleEncryptedMessage(source *User, evt *events.FBConsume case *waConsumerApplication.ConsumerApplication_Content_EditMessage: portal.handleWhatsAppEditMessage(ctx, sender, content.EditMessage) case *waConsumerApplication.ConsumerApplication_Content_ReactionMessage: - log.Warn().Msg("Unsupported reaction message payload message") + portal.handleMetaOrWhatsAppReaction(ctx, sender, content.ReactionMessage.GetKey().GetID(), content.ReactionMessage.GetText(), content.ReactionMessage.GetSenderTimestampMS()) default: portal.handleMetaOrWhatsAppMessage(ctx, source, sender, evt, nil) } @@ -1407,13 +1407,18 @@ func (portal *Portal) handleMetaEditMessage(edit *table.LSEditMessage) { func (portal *Portal) handleMetaReaction(react *table.LSUpsertReaction) { sender := portal.bridge.GetPuppetByID(react.ActorId) - log := portal.log.With(). + ctx := portal.log.With(). Str("action", "upsert meta reaction"). Int64("sender_id", sender.ID). Str("target_msg_id", react.MessageId). - Logger() - ctx := log.WithContext(context.TODO()) - targetMsg, err := portal.bridge.DB.Message.GetByID(ctx, react.MessageId, 0, portal.Receiver) + Logger(). + WithContext(context.TODO()) + portal.handleMetaOrWhatsAppReaction(ctx, sender, react.MessageId, react.Reaction, react.TimestampMs) +} + +func (portal *Portal) handleMetaOrWhatsAppReaction(ctx context.Context, sender *Puppet, messageID, reaction string, timestamp int64) { + log := zerolog.Ctx(ctx) + targetMsg, err := portal.bridge.DB.Message.GetByID(ctx, messageID, 0, portal.Receiver) if err != nil { log.Err(err).Msg("Failed to get target message from database") return @@ -1425,7 +1430,7 @@ func (portal *Portal) handleMetaReaction(react *table.LSUpsertReaction) { if err != nil { log.Err(err).Msg("Failed to get existing reaction from database") return - } else if existingReaction != nil && existingReaction.Emoji == react.Reaction { + } else if existingReaction != nil && existingReaction.Emoji == reaction { // TODO should reactions be deduplicated by some ID instead of the emoji? log.Debug().Msg("Ignoring duplicate reaction") return @@ -1442,11 +1447,11 @@ func (portal *Portal) handleMetaReaction(react *table.LSUpsertReaction) { content := &event.ReactionEventContent{ RelatesTo: event.RelatesTo{ Type: event.RelAnnotation, - Key: variationselector.Add(react.Reaction), + Key: variationselector.Add(reaction), EventID: targetMsg.MXID, }, } - resp, err := portal.sendMatrixEvent(ctx, intent, event.EventReaction, content, nil, 0) + resp, err := portal.sendMatrixEvent(ctx, intent, event.EventReaction, content, nil, timestamp) if err != nil { log.Err(err).Msg("Failed to send reaction") return @@ -1459,14 +1464,14 @@ func (portal *Portal) handleMetaReaction(react *table.LSUpsertReaction) { dbReaction.ThreadID = portal.ThreadID dbReaction.ThreadReceiver = portal.Receiver dbReaction.Sender = sender.ID - dbReaction.Emoji = react.Reaction + dbReaction.Emoji = reaction // TODO save timestamp? err = dbReaction.Insert(ctx) if err != nil { log.Err(err).Msg("Failed to insert reaction to database") } } else { - existingReaction.Emoji = react.Reaction + existingReaction.Emoji = reaction existingReaction.MXID = resp.EventID err = existingReaction.Update(ctx) if err != nil { From fe742c7706678f7d3ff08096f3e9dc55aeff0d0a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 20:33:53 +0200 Subject: [PATCH 12/19] Add support for incoming message unsending from WhatsApp --- ROADMAP.md | 2 +- portal.go | 64 +++++++++++++++++++++++++++++------------------------- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index aaa2368..4a14156 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -119,7 +119,7 @@ * [x] Replies * [x] Mentions * [ ] Polls - * [ ] Message unsend + * [x] Message unsend * [x] Message reactions * [x] Message edits * [ ] Message history/Reading chat backup diff --git a/portal.go b/portal.go index 12afcb4..1fb3df4 100644 --- a/portal.go +++ b/portal.go @@ -1250,6 +1250,7 @@ func (portal *Portal) handleEncryptedMessage(source *User, evt *events.FBConsume case *waConsumerApplication.ConsumerApplication_Payload_ApplicationData: switch applicationContent := payload.ApplicationData.GetApplicationContent().(type) { case *waConsumerApplication.ConsumerApplication_ApplicationData_Revoke: + portal.handleMetaOrWhatsAppDelete(ctx, sender, applicationContent.Revoke.GetKey().GetID()) default: log.Warn().Type("content_type", applicationContent).Msg("Unrecognized application content type") } @@ -1413,7 +1414,18 @@ func (portal *Portal) handleMetaReaction(react *table.LSUpsertReaction) { Str("target_msg_id", react.MessageId). Logger(). WithContext(context.TODO()) - portal.handleMetaOrWhatsAppReaction(ctx, sender, react.MessageId, react.Reaction, react.TimestampMs) + portal.handleMetaOrWhatsAppReaction(ctx, sender, react.MessageId, react.Reaction, 0) +} + +func (portal *Portal) handleMetaReactionDelete(react *table.LSDeleteReaction) { + sender := portal.bridge.GetPuppetByID(react.ActorId) + log := portal.log.With(). + Str("action", "delete meta reaction"). + Int64("sender_id", sender.ID). + Str("target_msg_id", react.MessageId). + Logger() + ctx := log.WithContext(context.TODO()) + portal.handleMetaOrWhatsAppReaction(ctx, sender, react.MessageId, "", 0) } func (portal *Portal) handleMetaOrWhatsAppReaction(ctx context.Context, sender *Puppet, messageID, reaction string, timestamp int64) { @@ -1444,6 +1456,17 @@ func (portal *Portal) handleMetaOrWhatsAppReaction(ctx context.Context, sender * log.Err(err).Msg("Failed to redact reaction") } } + if reaction == "" { + if existingReaction == nil { + log.Warn().Msg("Existing reaction to delete not found") + return + } + err = existingReaction.Delete(ctx) + if err != nil { + log.Err(err).Msg("Failed to delete reaction from database") + } + return + } content := &event.ReactionEventContent{ RelatesTo: event.RelatesTo{ Type: event.RelAnnotation, @@ -1480,40 +1503,17 @@ func (portal *Portal) handleMetaOrWhatsAppReaction(ctx context.Context, sender * } } -func (portal *Portal) handleMetaReactionDelete(react *table.LSDeleteReaction) { - sender := portal.bridge.GetPuppetByID(react.ActorId) - log := portal.log.With(). - Str("action", "delete meta reaction"). - Int64("sender_id", sender.ID). - Str("target_msg_id", react.MessageId). - Logger() - ctx := log.WithContext(context.TODO()) - existingReaction, err := portal.bridge.DB.Reaction.GetByID(ctx, react.MessageId, portal.Receiver, sender.ID) - if err != nil { - log.Err(err).Msg("Failed to get existing reaction from database") - return - } else if existingReaction == nil { - log.Warn().Msg("Existing reaction to delete not found") - return - } - _, err = sender.IntentFor(portal).RedactEvent(ctx, portal.MXID, existingReaction.MXID, mautrix.ReqRedact{ - TxnID: "mxmeta_unreact_" + existingReaction.MXID.String(), - }) - if err != nil { - log.Err(err).Msg("Failed to redact reaction") - } - err = existingReaction.Delete(ctx) - if err != nil { - log.Err(err).Msg("Failed to delete reaction from database") - } -} - func (portal *Portal) handleMetaDelete(messageID string) { log := portal.log.With(). Str("action", "delete meta message"). Str("message_id", messageID). Logger() ctx := log.WithContext(context.TODO()) + portal.handleMetaOrWhatsAppDelete(ctx, nil, messageID) +} + +func (portal *Portal) handleMetaOrWhatsAppDelete(ctx context.Context, sender *Puppet, messageID string) { + log := zerolog.Ctx(ctx) targetMsg, err := portal.bridge.DB.Message.GetAllPartsByID(ctx, messageID, portal.Receiver) if err != nil { log.Err(err).Msg("Failed to get target message from database") @@ -1522,8 +1522,12 @@ func (portal *Portal) handleMetaDelete(messageID string) { log.Warn().Msg("Target message not found") return } + intent := portal.MainIntent() + if sender != nil { + intent = sender.IntentFor(portal) + } for _, part := range targetMsg { - _, err = portal.MainIntent().RedactEvent(ctx, portal.MXID, part.MXID, mautrix.ReqRedact{ + _, err = intent.RedactEvent(ctx, portal.MXID, part.MXID, mautrix.ReqRedact{ TxnID: "mxmeta_delete_" + part.MXID.String(), }) if err != nil { From ecf09609f8473bd680d38ebd13fd25b74266e710 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 21:30:14 +0200 Subject: [PATCH 13/19] Make reactions/redactions work for old unencrypted messages Also disable read receipts completely for those --- database/message.go | 4 ++++ portal.go | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/database/message.go b/database/message.go index bd1e553..49d0334 100644 --- a/database/message.go +++ b/database/message.go @@ -228,3 +228,7 @@ func (msg *Message) EditTimestamp() int64 { func (msg *Message) UpdateEditTimestamp(ctx context.Context, ts int64) error { return msg.UpdateEditCount(ctx, ts) } + +func (msg *Message) IsUnencrypted() bool { + return strings.HasPrefix(msg.ID, "mid.$") +} diff --git a/portal.go b/portal.go index 1fb3df4..adf98e4 100644 --- a/portal.go +++ b/portal.go @@ -484,8 +484,8 @@ func (portal *Portal) handleMatrixReadReceiptForWhatsApp(ctx context.Context, se groupedMessages := make(map[types.JID][]types.MessageID) for _, msg := range messages { var key types.JID - if msg.Sender == sender.MetaID { - // Don't send read receipts for own messages or fake messages + if msg.Sender == sender.MetaID || msg.IsUnencrypted() { + // Don't send read receipts for own messages or unencrypted messages continue } else if !portal.IsPrivateChat() { // TODO: this is hacky since it hardcodes the server @@ -743,7 +743,7 @@ func (portal *Portal) handleMatrixRedaction(ctx context.Context, sender *User, e portal.sendMessageStatusCheckpointFailed(ctx, evt, errRedactionTargetSentBySomeoneElse) return } - if portal.ThreadType.IsWhatsApp() { + if !dbMessage.IsUnencrypted() { consumerMsg := wrapRevoke(&waConsumerApplication.ConsumerApplication_RevokeMessage{ Key: portal.buildMessageKey(sender, dbMessage), }) @@ -877,7 +877,7 @@ func (portal *Portal) buildMessageKey(user *User, targetMsg *database.Message) * } func (portal *Portal) sendReaction(ctx context.Context, sender *User, targetMsg *database.Message, metaEmoji string, timestamp int64) error { - if portal.ThreadType.IsWhatsApp() { + if !targetMsg.IsUnencrypted() { consumerMsg := wrapReaction(&waConsumerApplication.ConsumerApplication_ReactionMessage{ Key: portal.buildMessageKey(sender, targetMsg), Text: metaEmoji, From ed82a5d6eede5c39012915cce5276c964bbe0c64 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 21:30:56 +0200 Subject: [PATCH 14/19] Add command to toggle messenger-side encryption in DMs --- commands.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/commands.go b/commands.go index 8ffd5e7..3b323a3 100644 --- a/commands.go +++ b/commands.go @@ -57,6 +57,7 @@ func (br *MetaBridge) RegisterCommands() { cmdLogin, cmdSyncSpace, cmdDeleteSession, + cmdToggleEncryption, cmdSetRelay, cmdUnsetRelay, cmdDeletePortal, @@ -77,6 +78,38 @@ func wrapCommand(handler func(*WrappedCommandEvent)) func(*commands.Event) { } } +var cmdToggleEncryption = &commands.FullHandler{ + Func: wrapCommand(fnToggleEncryption), + Name: "toggle-encryption", + Help: commands.HelpMeta{ + Section: HelpSectionPortalManagement, + Description: "Toggle Messenger-side encryption for the current room", + }, + RequiresPortal: true, + RequiresLogin: true, +} + +func fnToggleEncryption(ce *WrappedCommandEvent) { + if !ce.Bridge.Config.Meta.Mode.IsMessenger() { + ce.Reply("Encryption support is not yet enabled in Instagram mode") + return + } else if !ce.Portal.IsPrivateChat() { + ce.Reply("Only private chats can be toggled between encrypted and unencrypted") + return + } + if ce.Portal.ThreadType.IsWhatsApp() { + ce.Portal.ThreadType = table.ONE_TO_ONE + ce.Reply("Messages in this room will now be sent unencrypted over Messenger") + } else { + ce.Portal.ThreadType = table.ENCRYPTED_OVER_WA_ONE_TO_ONE + ce.Reply("Messages in this room will now be sent encrypted over WhatsApp") + } + err := ce.Portal.Update(ce.Ctx) + if err != nil { + ce.ZLog.Err(err).Msg("Failed to update portal in database") + } +} + var cmdSetRelay = &commands.FullHandler{ Func: wrapCommand(fnSetRelay), Name: "set-relay", From b341740535a293c4121f8021b514c3af8dbe6817 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 21:43:36 +0200 Subject: [PATCH 15/19] Update WhatsApp connection client payload --- messagix/client.go | 7 +++++-- messagix/e2ee-client.go | 10 +++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/messagix/client.go b/messagix/client.go index a5f6946..750a54e 100644 --- a/messagix/client.go +++ b/messagix/client.go @@ -30,13 +30,16 @@ import ( ) const DPR = "1" +const BrowserName = "Chrome" const ChromeVersion = "118" const ChromeVersionFull = ChromeVersion + ".0.5993.89" const UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/" + ChromeVersion + ".0.0.0 Safari/537.36" const SecCHUserAgent = `"Chromium";v="` + ChromeVersion + `", "Google Chrome";v="` + ChromeVersion + `", "Not-A.Brand";v="99"` const SecCHFullVersionList = `"Chromium";v="` + ChromeVersionFull + `", "Google Chrome";v="` + ChromeVersionFull + `", "Not-A.Brand";v="99.0.0.0"` -const SecCHPlatform = `"Linux"` -const SecCHPlatformVersion = `"6.5.0"` +const OSName = "Linux" +const OSVersion = "6.5.0" +const SecCHPlatform = `"` + OSName + `"` +const SecCHPlatformVersion = `"` + OSVersion + `"` const SecCHMobile = "?0" const SecCHModel = "" const SecCHPrefersColorScheme = "light" diff --git a/messagix/e2ee-client.go b/messagix/e2ee-client.go index cb13c4d..4e31741 100644 --- a/messagix/e2ee-client.go +++ b/messagix/e2ee-client.go @@ -53,7 +53,7 @@ func (c *Client) getClientPayload() *waProto.ClientPayload { Passive: proto.Bool(false), Pull: proto.Bool(true), UserAgent: &waProto.ClientPayload_UserAgent{ - Device: proto.String("Firefox"), + Device: proto.String(BrowserName), AppVersion: &waProto.ClientPayload_UserAgent_AppVersion{ Primary: proto.Uint32(301), Secondary: proto.Uint32(0), @@ -62,15 +62,15 @@ func (c *Client) getClientPayload() *waProto.ClientPayload { LocaleCountryIso31661Alpha2: proto.String("en"), LocaleLanguageIso6391: proto.String("en"), //Hardware: proto.String("Linux"), - Manufacturer: proto.String("Linux"), + Manufacturer: proto.String(OSName), Mcc: proto.String("000"), Mnc: proto.String("000"), - OsBuildNumber: proto.String("6.0.0"), - OsVersion: proto.String("6.0.0"), + OsBuildNumber: proto.String(""), + OsVersion: proto.String(""), //SimMcc: proto.String("000"), //SimMnc: proto.String("000"), - Platform: waProto.ClientPayload_UserAgent_WEB.Enum(), // or BLUE_WEB? + Platform: waProto.ClientPayload_UserAgent_BLUE_WEB.Enum(), ReleaseChannel: waProto.ClientPayload_UserAgent_DEBUG.Enum(), }, } From 5d4e77691a6b260d2e6e9892c89c105a9927e008 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 21:45:36 +0200 Subject: [PATCH 16/19] Remove unused e2eeClient variable in messagix.Client struct --- messagix/client.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/messagix/client.go b/messagix/client.go index 750a54e..8e87374 100644 --- a/messagix/client.go +++ b/messagix/client.go @@ -18,7 +18,6 @@ import ( "github.com/google/go-querystring/query" "github.com/rs/zerolog" - "go.mau.fi/whatsmeow" "go.mau.fi/whatsmeow/store" "golang.org/x/net/proxy" @@ -63,8 +62,7 @@ type Client struct { socksProxy proxy.Dialer GetNewProxy func(reason string) (string, error) - device *store.Device - e2eeClient *whatsmeow.Client + device *store.Device lsRequests int graphQLRequests int From f96df9c27cc12a7e2af5e535eb6abba84e22474c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 21:47:33 +0200 Subject: [PATCH 17/19] Log unhandled whatsmeow events --- user.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/user.go b/user.go index f812d88..49afbe8 100644 --- a/user.go +++ b/user.go @@ -873,6 +873,8 @@ func (user *User) e2eeEventHandler(rawEvt any) { if portal != nil { portal.metaMessages <- portalMetaMessage{user: user, evt: evt} } + default: + user.log.Debug().Type("event_type", rawEvt).Msg("Unhandled WhatsApp event") } } From bdf625dca5e6a18bec8c7c15d7b0a4af121c880b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 21:59:39 +0200 Subject: [PATCH 18/19] Use zerolog for whatsmeow db --- main.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 093ddd2..106398a 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,9 @@ import ( "maunium.net/go/mautrix/format" "maunium.net/go/mautrix/id" + "go.mau.fi/whatsmeow/store/sqlstore" + waLog "go.mau.fi/whatsmeow/util/log" + "go.mau.fi/mautrix-meta/config" "go.mau.fi/mautrix-meta/database" "go.mau.fi/mautrix-meta/messagix/cookies" @@ -38,8 +41,6 @@ import ( "go.mau.fi/mautrix-meta/messagix/table" "go.mau.fi/mautrix-meta/messagix/types" "go.mau.fi/mautrix-meta/msgconv" - "go.mau.fi/whatsmeow/store/sqlstore" - waLog "go.mau.fi/whatsmeow/util/log" ) //go:embed example-config.yaml @@ -144,7 +145,7 @@ func (br *MetaBridge) Init() { br.RegisterCommands() br.DB = database.New(br.Bridge.DB) - br.DeviceStore = sqlstore.NewWithDB(br.DB.RawDB, br.DB.Dialect.String(), waLog.Stdout("DATABASE", "DEBUG", true)) + br.DeviceStore = sqlstore.NewWithDB(br.DB.RawDB, br.DB.Dialect.String(), waLog.Zerolog(br.ZLog.With().Str("db_section", "whatsmeow").Logger())) ss := br.Config.Bridge.Provisioning.SharedSecret if len(ss) > 0 && ss != "disable" { From 683490bb3bb77863ee4c2edd34cb243c37b4414c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Feb 2024 22:36:22 +0200 Subject: [PATCH 19/19] Fetch user info from FB if necessary when receiving message --- messagix/socket/contacts.go | 2 +- portal.go | 16 ++++++++++------ puppet.go | 37 ++++++++++++++++++++++++++++++++----- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/messagix/socket/contacts.go b/messagix/socket/contacts.go index 23fd060..506b11b 100644 --- a/messagix/socket/contacts.go +++ b/messagix/socket/contacts.go @@ -21,7 +21,7 @@ func (t *GetContactsTask) Create() (interface{}, interface{}, bool) { } type GetContactsFullTask struct { - ContactId int64 `json:"contact_id"` + ContactID int64 `json:"contact_id"` } func (t *GetContactsFullTask) GetLabel() string { diff --git a/portal.go b/portal.go index adf98e4..e80afeb 100644 --- a/portal.go +++ b/portal.go @@ -1236,6 +1236,7 @@ func (portal *Portal) handleEncryptedMessage(source *User, evt *events.FBConsume Time("message_ts", evt.Info.Timestamp). Logger() ctx := log.WithContext(context.TODO()) + sender.FetchAndUpdateInfoIfNecessary(ctx, source) switch payload := evt.Message.GetPayload().GetPayload().(type) { case *waConsumerApplication.ConsumerApplication_Payload_Content: @@ -1778,8 +1779,10 @@ func (portal *Portal) CreateMatrixRoom(ctx context.Context, user *User) error { invite = append(invite, portal.bridge.Bot.UserID) } } - if portal.IsPrivateChat() { - portal.UpdateInfoFromPuppet(ctx, portal.GetDMPuppet()) + dmPuppet := portal.GetDMPuppet() + if dmPuppet != nil { + dmPuppet.FetchAndUpdateInfoIfNecessary(ctx, user) + portal.UpdateInfoFromPuppet(ctx, dmPuppet) } if !portal.AvatarURL.IsEmpty() { initialState = append(initialState, &event.Event{ @@ -1834,10 +1837,10 @@ func (portal *Portal) CreateMatrixRoom(ctx context.Context, user *User) error { go portal.addToPersonalSpace(portal.log.WithContext(context.TODO()), user) if portal.IsPrivateChat() { - user.AddDirectChat(ctx, portal.MXID, portal.GetDMPuppet().MXID) + user.AddDirectChat(ctx, portal.MXID, dmPuppet.MXID) } if waGroupInfo != nil && !autoJoinInvites { - portal.SyncWAParticipants(ctx, waGroupInfo.Participants) + portal.SyncWAParticipants(ctx, user, waGroupInfo.Participants) } return nil @@ -1875,7 +1878,7 @@ func (portal *Portal) UpdateWAGroupInfo(ctx context.Context, source *User, group update = portal.updateName(ctx, groupInfo.Name) || update //update = portal.updateTopic(ctx, groupInfo.Topic) || update //update = portal.updateWAAvatar(ctx) - participants := portal.SyncWAParticipants(ctx, groupInfo.Participants) + participants := portal.SyncWAParticipants(ctx, source, groupInfo.Participants) if update { err := portal.Update(ctx) if err != nil { @@ -1886,10 +1889,11 @@ func (portal *Portal) UpdateWAGroupInfo(ctx context.Context, source *User, group return groupInfo, participants } -func (portal *Portal) SyncWAParticipants(ctx context.Context, participants []types.GroupParticipant) []id.UserID { +func (portal *Portal) SyncWAParticipants(ctx context.Context, source *User, participants []types.GroupParticipant) []id.UserID { var userIDs []id.UserID for _, pcp := range participants { puppet := portal.bridge.GetPuppetByID(int64(pcp.JID.UserInt())) + puppet.FetchAndUpdateInfoIfNecessary(ctx, source) userIDs = append(userIDs, puppet.IntentFor(portal).UserID) if portal.MXID != "" { err := puppet.IntentFor(portal).EnsureJoined(ctx, portal.MXID) diff --git a/puppet.go b/puppet.go index c15167f..7f56602 100644 --- a/puppet.go +++ b/puppet.go @@ -31,6 +31,7 @@ import ( "go.mau.fi/mautrix-meta/config" "go.mau.fi/mautrix-meta/database" + "go.mau.fi/mautrix-meta/messagix/socket" "go.mau.fi/mautrix-meta/messagix/types" "go.mau.fi/mautrix-meta/msgconv" ) @@ -179,6 +180,8 @@ type Puppet struct { customUser *User syncLock sync.Mutex + + triedFetchingInfo bool } var userIDRegex *regexp.Regexp @@ -221,6 +224,35 @@ func (puppet *Puppet) GetAvatarURL() id.ContentURI { return puppet.AvatarURL } +func (puppet *Puppet) FetchAndUpdateInfoIfNecessary(ctx context.Context, via *User) { + if puppet.triedFetchingInfo || puppet.Name != "" { + return + } + puppet.triedFetchingInfo = true + zerolog.Ctx(ctx).Debug().Int64("via_user_meta_id", via.MetaID).Msg("Fetching and updating info for user") + resp, err := via.Client.ExecuteTasks(&socket.GetContactsFullTask{ + ContactID: puppet.ID, + }) + if err != nil { + zerolog.Ctx(ctx).Err(err).Int64("via_user_meta_id", via.MetaID).Msg("Failed to fetch info") + } else { + var gotInfo bool + for _, info := range resp.LSDeleteThenInsertContact { + if info.Id == puppet.ID { + puppet.UpdateInfo(ctx, info) + gotInfo = true + } else { + zerolog.Ctx(ctx).Warn().Int64("other_meta_id", info.Id).Msg("Got info for wrong user") + } + } + if !gotInfo { + zerolog.Ctx(ctx).Warn().Int64("via_user_meta_id", via.MetaID).Msg("Didn't get info for user") + } else { + zerolog.Ctx(ctx).Debug().Int64("via_user_meta_id", via.MetaID).Msg("Fetched and updated info for user") + } + } +} + func (puppet *Puppet) UpdateInfo(ctx context.Context, info types.UserInfo) { log := zerolog.Ctx(ctx).With(). Str("function", "Puppet.UpdateInfo"). @@ -228,11 +260,6 @@ func (puppet *Puppet) UpdateInfo(ctx context.Context, info types.UserInfo) { Logger() ctx = log.WithContext(ctx) var err error - if info == nil { - log.Debug().Msg("Not Fetching info to update puppet") - // TODO implement? - return - } log.Trace().Msg("Updating puppet info")