Merge mautrix#464

Includes migration from our downstream revision to the signalmeow store

config/bridge.go

@@ -35,6 +35,7 @@ type BridgeConfig struct {
 	DisplaynameTemplate   string `yaml:"displayname_template"`
 	PrivateChatPortalMeta string `yaml:"private_chat_portal_meta"`
 	UseContactAvatars     bool   `yaml:"use_contact_avatars"`
+	UseOutdatedProfiles   bool   `yaml:"use_outdated_profiles"`
 	NumberInTopic         bool   `yaml:"number_in_topic"`
 
 	NoteToSelfAvatar id.ContentURIString `yaml:"note_to_self_avatar"`

config/upgrade.go

@@ -84,6 +84,7 @@ func DoUpgrade(helper *up.Helper) {
 	}
 	helper.Copy(up.Str, "bridge", "private_chat_portal_meta")
 	helper.Copy(up.Bool, "bridge", "use_contact_avatars")
+	helper.Copy(up.Bool, "bridge", "use_outdated_profiles")
 	helper.Copy(up.Bool, "bridge", "number_in_topic")
 	helper.Copy(up.Str, "bridge", "note_to_self_avatar")
 	helper.Copy(up.Int, "bridge", "portal_message_buffer")

database/puppet.go

@@ -31,7 +31,7 @@ import (
 const (
 	puppetBaseSelect = `
         SELECT uuid, number, name, name_quality, avatar_path, avatar_hash, avatar_url, name_set, avatar_set,
-               contact_info_set, is_registered, custom_mxid, access_token, first_activity_ts, last_activity_ts
+               contact_info_set, is_registered, profile_fetched_at, custom_mxid, access_token, first_activity_ts, last_activity_ts
         FROM puppet
 	`
 	getPuppetBySignalIDQuery   = puppetBaseSelect + `WHERE uuid=$1`
@@ -41,18 +41,18 @@ const (
 	updatePuppetQuery          = `
 		UPDATE puppet SET
 			number=$2, name=$3, name_quality=$4, avatar_path=$5, avatar_hash=$6, avatar_url=$7,
-			name_set=$8, avatar_set=$9, contact_info_set=$10, is_registered=$11,
-			custom_mxid=$12, access_token=$13
+			name_set=$8, avatar_set=$9, contact_info_set=$10, is_registered=$11, profile_fetched_at=$12,
+			custom_mxid=$13, access_token=$14
 		WHERE uuid=$1
 	`
 	insertPuppetQuery = `
 		INSERT INTO puppet (
 			uuid, number, name, name_quality, avatar_path, avatar_hash, avatar_url,
-			name_set, avatar_set, contact_info_set, is_registered,
+			name_set, avatar_set, contact_info_set, is_registered, profile_fetched_at,
 			custom_mxid, access_token
 		)
 		VALUES (
-			$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13
+			$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14
 		)
 	`
 	oneDayMs = 24 * 60 * 60 * 1000
@@ -75,11 +75,12 @@ type Puppet struct {
 	NameSet     bool
 	AvatarSet   bool
 
-	IsRegistered bool
+	IsRegistered     bool
+	ContactInfoSet   bool
+	ProfileFetchedAt time.Time
 
-	CustomMXID     id.UserID
-	AccessToken    string
-	ContactInfoSet bool
+	CustomMXID  id.UserID
+	AccessToken string
 
 	FirstActivityTs int64
 	LastActivityTs  int64
@@ -107,6 +108,7 @@ func (pq *PuppetQuery) GetAllWithCustomMXID(ctx context.Context) ([]*Puppet, err
 
 func (p *Puppet) Scan(row dbutil.Scannable) (*Puppet, error) {
 	var number, customMXID sql.NullString
+	var profileFetchedAt sql.NullInt64
 	var firstActivityTs, lastActivityTs sql.NullInt64
 	err := row.Scan(
 		&p.SignalID,
@@ -120,16 +122,20 @@ func (p *Puppet) Scan(row dbutil.Scannable) (*Puppet, error) {
 		&p.AvatarSet,
 		&p.ContactInfoSet,
 		&p.IsRegistered,
+		&profileFetchedAt,
 		&customMXID,
 		&p.AccessToken,
 		&firstActivityTs,
 		&lastActivityTs,
 	)
 	if err != nil {
-		return nil, nil
+		return nil, err
 	}
 	p.Number = number.String
 	p.CustomMXID = id.UserID(customMXID.String)
+	if profileFetchedAt.Valid {
+		p.ProfileFetchedAt = time.UnixMilli(profileFetchedAt.Int64)
+	}
 	p.FirstActivityTs = firstActivityTs.Int64
 	p.LastActivityTs = lastActivityTs.Int64
 	return p, nil
@@ -148,6 +154,7 @@ func (p *Puppet) sqlVariables() []any {
 		p.AvatarSet,
 		p.ContactInfoSet,
 		p.IsRegistered,
+		dbutil.UnixMilliPtr(p.ProfileFetchedAt),
 		dbutil.StrPtr(p.CustomMXID),
 		p.AccessToken,
 	}

database/upgrades/00-latest.sql

@@ -1,4 +1,4 @@
--- v0 -> v20 (compatible with v18+): Latest revision
+-- v0 -> v21 (compatible with v18+): Latest revision
 
 CREATE TABLE portal (
     chat_id     TEXT    NOT NULL,
@@ -33,8 +33,9 @@ CREATE TABLE puppet (
     name_set     BOOLEAN NOT NULL DEFAULT false,
     avatar_set   BOOLEAN NOT NULL DEFAULT false,
 
-    is_registered    BOOLEAN NOT NULL DEFAULT false,
-    contact_info_set BOOLEAN NOT NULL DEFAULT false,
+    is_registered      BOOLEAN NOT NULL DEFAULT false,
+    contact_info_set   BOOLEAN NOT NULL DEFAULT false,
+    profile_fetched_at BIGINT,
 
     custom_mxid  TEXT,
     access_token TEXT NOT NULL,

database/upgrades/21-puppet-profile-fetch-ts.sql

@@ -0,0 +1,2 @@
+-- v21 (compatible with v18+): Add profile fetch timestamp for puppets
+ALTER TABLE puppet ADD profile_fetched_at BIGINT;

example-config.yaml

@@ -106,6 +106,8 @@ bridge:
     private_chat_portal_meta: default
     # Should avatars from the user's contact list be used? This is not safe on multi-user instances.
     use_contact_avatars: false
+    # Should the bridge sync ghost user info even if profile fetching fails? This is not safe on multi-user instances.
+    use_outdated_profiles: false
     # Should the Signal user's phone number be included in the room topic in private chat portal rooms?
     number_in_topic: true
     # Avatar image for the Note to Self room.

go.mod

@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	github.com/beeper/libserv v0.0.0-20231231202820-c7303abfc32c
-	github.com/element-hq/mautrix-go v0.18.0-beta.1-mod-1
+	github.com/element-hq/mautrix-go v0.18.0-beta.1-mod-2.0.20240305190819-c6ec91b05c3c
 	github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/mux v1.8.0
@@ -16,7 +16,7 @@ require (
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/stretchr/testify v1.8.4
 	github.com/tidwall/gjson v1.17.1
-	go.mau.fi/util v0.4.0
+	go.mau.fi/util v0.4.1-0.20240222202553-953608f657a3
 	golang.org/x/crypto v0.19.0
 	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a
 	golang.org/x/net v0.21.0

go.sum

@@ -11,8 +11,8 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/element-hq/mautrix-go v0.18.0-beta.1-mod-1 h1:9ICQ0yZYIkYibOKmzUG2Vy8nJSkbv/qLQaGghn+fqcY=
-github.com/element-hq/mautrix-go v0.18.0-beta.1-mod-1/go.mod h1:eJu6JOtGbObkSyDpiBL58nuDyjLrIBbrrLa9aDbpsaI=
+github.com/element-hq/mautrix-go v0.18.0-beta.1-mod-2.0.20240305190819-c6ec91b05c3c h1:xA1shDsjlmCuE1YQnXMCr9++Rm6U8nL6RleuOUmx/80=
+github.com/element-hq/mautrix-go v0.18.0-beta.1-mod-2.0.20240305190819-c6ec91b05c3c/go.mod h1:eJu6JOtGbObkSyDpiBL58nuDyjLrIBbrrLa9aDbpsaI=
 github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 h1:ATgqloALX6cHCranzkLb8/zjivwQ9DWWDCQRnxTPfaA=
 github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -73,8 +73,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/yuin/goldmark v1.7.0 h1:EfOIvIMZIzHdB/R/zVrikYLPPwJlfMcNczJFMs1m6sA=
 github.com/yuin/goldmark v1.7.0/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
-go.mau.fi/util v0.4.0 h1:S2X3qU4pUcb/vxBRfAuZjbrR9xVMAXSjQojNBLPBbhs=
-go.mau.fi/util v0.4.0/go.mod h1:leeiHtgVBuN+W9aDii3deAXnfC563iN3WK6BF8/AjNw=
+go.mau.fi/util v0.4.1-0.20240222202553-953608f657a3 h1:NcRrdzORHKab5bP1Z8BpH0nxsxsvH0iPPZLpOUN+UIc=
+go.mau.fi/util v0.4.1-0.20240222202553-953608f657a3/go.mod h1:leeiHtgVBuN+W9aDii3deAXnfC563iN3WK6BF8/AjNw=
 go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto=
 go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70=
 golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=

pkg/libsignalgo/profilekey.go

@@ -35,19 +35,25 @@ type ProfileKeyCommitment [C.SignalPROFILE_KEY_COMMITMENT_LEN]byte
 type ProfileKeyVersion [C.SignalPROFILE_KEY_VERSION_ENCODED_LEN]byte
 type AccessKey [C.SignalACCESS_KEY_LEN]byte
 
+var blankProfileKey ProfileKey
+
+func (pk *ProfileKey) IsEmpty() bool {
+	return pk == nil || *pk == blankProfileKey
+}
+
 func (ak *AccessKey) String() string {
-	return string((*ak)[:])
+	return string(ak[:])
 }
 
 func (pv *ProfileKeyVersion) String() string {
-	return string((*pv)[:])
+	return string(pv[:])
 }
 
 func (pk *ProfileKey) Slice() []byte {
 	if pk == nil {
 		return nil
 	}
-	return (*pk)[:]
+	return pk[:]
 }
 
 func (pk *ProfileKey) GetCommitment(u uuid.UUID) (*ProfileKeyCommitment, error) {

pkg/signalmeow/contact.go

@@ -22,6 +22,7 @@ import (
 	"crypto/sha256"
 	"encoding/binary"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"net/http"
 	"strings"
@@ -46,27 +47,23 @@ func (cli *Client) StoreContactDetailsAsContact(ctx context.Context, contactDeta
 		Logger()
 	existingContact, err := cli.Store.ContactStore.LoadContact(ctx, parsedUUID)
 	if err != nil {
-		log.Err(err).Msg("error loading contact")
+		log.Err(err).Msg("Failed to load contact from database")
 		return nil, err
 	}
 	if existingContact == nil {
-		log.Debug().Msg("creating new contact")
 		existingContact = &types.Contact{
 			UUID: parsedUUID,
 		}
-	} else {
-		log.Debug().Msg("updating existing contact")
 	}
 
 	existingContact.E164 = contactDetails.GetNumber()
 	existingContact.ContactName = contactDetails.GetName()
 	if profileKeyString := contactDetails.GetProfileKey(); profileKeyString != nil {
 		profileKey := libsignalgo.ProfileKey(profileKeyString)
-		existingContact.Profile.Key = &profileKey
+		existingContact.Profile.Key = profileKey
 		err = cli.Store.ProfileKeyStore.StoreProfileKey(ctx, existingContact.UUID, profileKey)
 		if err != nil {
-			log.Err(err).Msg("storing profile key")
-			//return *existingContact, nil, err
+			log.Err(err).Msg("Failed to store profile key from contact")
 		}
 	}
 
@@ -86,26 +83,25 @@ func (cli *Client) StoreContactDetailsAsContact(ctx context.Context, contactDeta
 		}
 	}
 
-	log.Debug().Msg("storing contact")
 	storeErr := cli.Store.ContactStore.StoreContact(ctx, *existingContact)
 	if storeErr != nil {
-		log.Err(storeErr).Msg("error storing contact")
+		log.Err(storeErr).Msg("Failed to save contact")
 		return existingContact, storeErr
 	}
 	return existingContact, nil
 }
 
-func (cli *Client) fetchContactThenTryAndUpdateWithProfile(ctx context.Context, profileUUID uuid.UUID) (existingContact *types.Contact, otherSourceUUID uuid.UUID, err error) {
+func (cli *Client) fetchContactThenTryAndUpdateWithProfile(ctx context.Context, profileUUID uuid.UUID) (*types.Contact, error) {
 	log := zerolog.Ctx(ctx).With().
 		Str("action", "fetch contact then try and update with profile").
 		Stringer("profile_uuid", profileUUID).
 		Logger()
 	contactChanged := false
 
-	existingContact, err = cli.Store.ContactStore.LoadContact(ctx, profileUUID)
+	existingContact, err := cli.Store.ContactStore.LoadContact(ctx, profileUUID)
 	if err != nil {
-		log.Err(err).Msg("error loading contact")
-		return
+		log.Err(err).Msg("Failed to load contact from database")
+		return nil, err
 	}
 	if existingContact == nil {
 		log.Debug().Msg("creating new contact")
@@ -116,49 +112,32 @@ func (cli *Client) fetchContactThenTryAndUpdateWithProfile(ctx context.Context,
 	} else {
 		log.Debug().Msg("updating existing contact")
 	}
-	profile, lastFetched, fetchErr := cli.RetrieveProfileByID(ctx, profileUUID)
-	if fetchErr != nil {
-		log.Err(fetchErr).Msg("error retrieving profile")
-		// Don't return here, we still want to return what we have
-	} else if profile != nil {
-		if existingContact.Profile.Name != profile.Name {
-			existingContact.Profile.Name = profile.Name
-			contactChanged = true
-		}
-		if existingContact.Profile.About != profile.About {
-			existingContact.Profile.About = profile.About
-			contactChanged = true
-		}
-		if existingContact.Profile.AboutEmoji != profile.AboutEmoji {
-			existingContact.Profile.AboutEmoji = profile.AboutEmoji
-			contactChanged = true
-		}
-		if existingContact.Profile.AvatarPath != profile.AvatarPath {
-			existingContact.Profile.AvatarPath = profile.AvatarPath
-			contactChanged = true
+	profile, err := cli.RetrieveProfileByID(ctx, profileUUID)
+	if err != nil {
+		logLevel := zerolog.ErrorLevel
+		if errors.Is(err, errProfileKeyNotFound) {
+			logLevel = zerolog.DebugLevel
 		}
-		if existingContact.Profile.Key == nil || *existingContact.Profile.Key != profile.Key {
-			existingContact.Profile.Key = &profile.Key
+		log.WithLevel(logLevel).Err(err).Msg("Failed to fetch profile")
+		// Continue to return contact without profile
+	}
+
+	if profile != nil {
+		// Don't bother saving every fetched timestamp to the database, but save if anything else changed
+		if !existingContact.Profile.Equals(profile) || existingContact.Profile.FetchedAt.IsZero() {
 			contactChanged = true
 		}
+		existingContact.Profile = *profile
 	}
 
 	if contactChanged {
-		existingContact.ProfileFetchTs = lastFetched.UnixMilli()
 		err = cli.Store.ContactStore.StoreContact(ctx, *existingContact)
 		if err != nil {
-			log.Err(err).Msg("error storing contact")
-			return
-		}
-	}
-
-	if fetchErr != nil {
-		otherSourceUUID, fetchErr = cli.Store.ContactStore.UpdateContactWithLatestProfile(ctx, existingContact)
-		if fetchErr != nil {
-			log.Err(fetchErr).Msg("error retrieving latest profile for contact from other users")
+			log.Err(err).Msg("Failed to save contact")
+			return nil, err
 		}
 	}
-	return
+	return existingContact, nil
 }
 
 func (cli *Client) UpdateContactE164(ctx context.Context, uuid uuid.UUID, e164 string) error {
@@ -169,26 +148,23 @@ func (cli *Client) UpdateContactE164(ctx context.Context, uuid uuid.UUID, e164 s
 		Logger()
 	existingContact, err := cli.Store.ContactStore.LoadContact(ctx, uuid)
 	if err != nil {
-		log.Err(err).Msg("error loading contact")
+		log.Err(err).Msg("Failed to load contact from database")
 		return err
 	}
 	if existingContact == nil {
-		log.Debug().Msg("creating new contact")
 		existingContact = &types.Contact{
 			UUID: uuid,
 		}
-	} else {
-		log.Debug().Msg("found existing contact")
 	}
 	if existingContact.E164 == e164 {
 		return nil
 	}
-	log.Debug().Msg("e164 changed for contact")
+	log.Debug().Msg("Contact phone number changed")
 	existingContact.E164 = e164
 	return cli.Store.ContactStore.StoreContact(ctx, *existingContact)
 }
 
-func (cli *Client) ContactByID(ctx context.Context, uuid uuid.UUID) (contact *types.Contact, otherSourceUUID uuid.UUID, err error) {
+func (cli *Client) ContactByID(ctx context.Context, uuid uuid.UUID) (*types.Contact, error) {
 	return cli.fetchContactThenTryAndUpdateWithProfile(ctx, uuid)
 }
 
@@ -201,7 +177,7 @@ func (cli *Client) ContactByE164(ctx context.Context, e164 string) (*types.Conta
 	if contact == nil {
 		return nil, nil
 	}
-	contact, _, err = cli.fetchContactThenTryAndUpdateWithProfile(ctx, contact.UUID)
+	contact, err = cli.fetchContactThenTryAndUpdateWithProfile(ctx, contact.UUID)
 	return contact, err
 }