Add partial support for outgoing WhatsApp media and replies

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/zyedidia/clipboard v1.0.4
 	go.mau.fi/libsignal v0.1.0
 	go.mau.fi/util v0.3.1-0.20240208085450-32294da153ab
-	go.mau.fi/whatsmeow v0.0.0-20240208095108-62a74a5394ef
+	go.mau.fi/whatsmeow v0.0.0-20240208121856-0ef58f1cef30
 	golang.org/x/crypto v0.19.0
 	golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3
 	golang.org/x/image v0.15.0

go.sum

@@ -55,8 +55,8 @@ go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c=
 go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I=
 go.mau.fi/util v0.3.1-0.20240208085450-32294da153ab h1:XZ8W5vHWlXSGmHn1U+Fvbh+xZr9wuHTvbY+qV7aybDY=
 go.mau.fi/util v0.3.1-0.20240208085450-32294da153ab/go.mod h1:rRypwgXVEPILomtFPyQcnbOeuRqf+nRN84vh/CICq4w=
-go.mau.fi/whatsmeow v0.0.0-20240208095108-62a74a5394ef h1:tu9klnNOeOGSbLRIlmUlVt62cKlGmwLlb1pXFr3NYkg=
-go.mau.fi/whatsmeow v0.0.0-20240208095108-62a74a5394ef/go.mod h1:lQHbhaG/fI+6hfGqz5Vzn2OBJBEZ05H0kCP6iJXriN4=
+go.mau.fi/whatsmeow v0.0.0-20240208121856-0ef58f1cef30 h1:gwmgU+l0OSww0Me3N022hLdZk7lnEoK8XeaWGlESG1I=
+go.mau.fi/whatsmeow v0.0.0-20240208121856-0ef58f1cef30/go.mod h1:lQHbhaG/fI+6hfGqz5Vzn2OBJBEZ05H0kCP6iJXriN4=
 go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto=
 go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70=
 golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=

messagix/socket/threads.go

@@ -41,6 +41,7 @@ type ReplyMetaData struct {
 	ReplyMessageId  string `json:"reply_source_id"`
 	ReplySourceType int64  `json:"reply_source_type"` // 1 ?
 	ReplyType       int64  `json:"reply_type"`        // ?
+	ReplySender     int64  `json:"-"`
 }
 
 type MentionData struct {

msgconv/from-matrix.go

@@ -100,32 +100,42 @@ func (mc *MessageConverter) ToMeta(ctx context.Context, evt *event.Event, conten
 	return []socket.Task{task, readTask}, task.Otid, nil
 }
 
-func (mc *MessageConverter) reuploadFileToMeta(ctx context.Context, evt *event.Event, content *event.MessageEventContent) (*types.MercuryUploadResponse, error) {
+func (mc *MessageConverter) downloadMatrixMedia(ctx context.Context, content *event.MessageEventContent) (data []byte, mimeType, fileName string, err error) {
 	mxc := content.URL
 	if content.File != nil {
 		mxc = content.File.URL
 	}
-	data, err := mc.DownloadMatrixMedia(ctx, mxc)
+	data, err = mc.DownloadMatrixMedia(ctx, mxc)
 	if err != nil {
-		return nil, exerrors.NewDualError(ErrMediaDownloadFailed, err)
+		err = exerrors.NewDualError(ErrMediaDownloadFailed, err)
+		return
 	}
 	if content.File != nil {
 		err = content.File.DecryptInPlace(data)
 		if err != nil {
-			return nil, exerrors.NewDualError(ErrMediaDecryptFailed, err)
+			err = exerrors.NewDualError(ErrMediaDecryptFailed, err)
+			return
 		}
 	}
-	mimeType := content.GetInfo().MimeType
+	mimeType = content.GetInfo().MimeType
 	if mimeType == "" {
 		mimeType = http.DetectContentType(data)
 	}
-	fileName := content.FileName
+	fileName = content.FileName
 	if fileName == "" {
 		fileName = content.Body
 		if fileName == "" {
 			fileName = string(content.MsgType)[2:] + exmime.ExtensionFromMimetype(mimeType)
 		}
 	}
+	return
+}
+
+func (mc *MessageConverter) reuploadFileToMeta(ctx context.Context, evt *event.Event, content *event.MessageEventContent) (*types.MercuryUploadResponse, error) {
+	data, mimeType, fileName, err := mc.downloadMatrixMedia(ctx, content)
+	if err != nil {
+		return nil, err
+	}
 	_, isVoice := evt.Content.Raw["org.matrix.msc3245.voice"]
 	if isVoice {
 		data, err = ffmpeg.ConvertBytes(ctx, data, ".wav", []string{}, []string{"-c:a", "pcm_u8", "-ar", "48000"}, mimeType)

msgconv/to-whatsapp.go

@@ -19,14 +19,30 @@ package msgconv
 import (
 	"context"
 	"fmt"
+	"strconv"
+	"strings"
+	"time"
 
+	"go.mau.fi/util/ffmpeg"
+	"go.mau.fi/whatsmeow"
+	"go.mau.fi/whatsmeow/binary/armadillo/waMediaTransport"
+	"go.mau.fi/whatsmeow/binary/armadillo/waMsgApplication"
+	"go.mau.fi/whatsmeow/types"
 	"maunium.net/go/mautrix/event"
 
 	"go.mau.fi/whatsmeow/binary/armadillo/waCommon"
 	"go.mau.fi/whatsmeow/binary/armadillo/waConsumerApplication"
 )
 
-func (mc *MessageConverter) ToWhatsApp(ctx context.Context, evt *event.Event, content *event.MessageEventContent, relaybotFormatted bool) (*waConsumerApplication.ConsumerApplication, error) {
+func (mc *MessageConverter) ToWhatsApp(
+	ctx context.Context,
+	evt *event.Event,
+	content *event.MessageEventContent,
+	relaybotFormatted bool,
+) (*waConsumerApplication.ConsumerApplication, *waMsgApplication.MessageApplication_Metadata, error) {
+	if evt.Type == event.EventSticker {
+		content.MsgType = event.MessageType(event.EventSticker.Type)
+	}
 	if content.MsgType == event.MsgEmote && !relaybotFormatted {
 		content.Body = "/me " + content.Body
 		if content.FormattedBody != "" {
@@ -42,12 +58,49 @@ func (mc *MessageConverter) ToWhatsApp(ctx context.Context, evt *event.Event, co
 				Text: content.Body,
 			},
 		}
-	case event.MsgImage, event.MsgVideo, event.MsgAudio, event.MsgFile:
-		fallthrough
+	case event.MsgImage, event.MsgVideo, event.MsgAudio, event.MsgFile, event.MessageType(event.EventSticker.Type):
+		reuploaded, fileName, err := mc.reuploadMediaToWhatsApp(ctx, evt, content)
+		if err != nil {
+			return nil, nil, err
+		}
+		var caption waCommon.MessageText
+		if content.FileName != "" && content.Body != content.FileName {
+			// TODO also mentions here
+			caption.Text = content.Body
+		}
+		waContent.Content, err = mc.wrapWhatsAppMedia(evt, content, reuploaded, &caption, fileName)
+		if err != nil {
+			return nil, nil, err
+		}
 	case event.MsgLocation:
-		fallthrough
+		lat, long, err := parseGeoURI(content.GeoURI)
+		if err != nil {
+			return nil, nil, err
+		}
+		// TODO does this actually work with any of the messenger clients?
+		waContent.Content = &waConsumerApplication.ConsumerApplication_Content_LocationMessage{
+			LocationMessage: &waConsumerApplication.ConsumerApplication_LocationMessage{
+				Location: &waConsumerApplication.ConsumerApplication_Location{
+					DegreesLatitude:  lat,
+					DegreesLongitude: long,
+					Name:             content.Body,
+				},
+				Address: "Earth",
+			},
+		}
 	default:
-		return nil, fmt.Errorf("%w %s", ErrUnsupportedMsgType, content.MsgType)
+		return nil, nil, fmt.Errorf("%w %s", ErrUnsupportedMsgType, content.MsgType)
+	}
+	var meta waMsgApplication.MessageApplication_Metadata
+	if replyTo := mc.GetMetaReply(ctx, content); replyTo != nil {
+		meta.QuotedMessage = &waMsgApplication.MessageApplication_Metadata_QuotedMessage{
+			StanzaID:  replyTo.ReplyMessageId,
+			RemoteJID: mc.GetData(ctx).JID().String(),
+			// TODO: this is hacky since it hardcodes the server
+			// TODO 2: should this be included for DMs?
+			Participant: types.JID{User: strconv.FormatInt(replyTo.ReplySender, 10), Server: types.MessengerServer}.String(),
+			Payload:     nil,
+		}
 	}
 	return &waConsumerApplication.ConsumerApplication{
 		Payload: &waConsumerApplication.ConsumerApplication_Payload{
@@ -56,5 +109,174 @@ func (mc *MessageConverter) ToWhatsApp(ctx context.Context, evt *event.Event, co
 			},
 		},
 		Metadata: nil,
-	}, nil
+	}, &meta, nil
+}
+
+func parseGeoURI(uri string) (lat, long float64, err error) {
+	if !strings.HasPrefix(uri, "geo:") {
+		err = fmt.Errorf("uri doesn't have geo: prefix")
+		return
+	}
+	// Remove geo: prefix and anything after ;
+	coordinates := strings.Split(strings.TrimPrefix(uri, "geo:"), ";")[0]
+
+	if splitCoordinates := strings.Split(coordinates, ","); len(splitCoordinates) != 2 {
+		err = fmt.Errorf("didn't find exactly two numbers separated by a comma")
+	} else if lat, err = strconv.ParseFloat(splitCoordinates[0], 64); err != nil {
+		err = fmt.Errorf("latitude is not a number: %w", err)
+	} else if long, err = strconv.ParseFloat(splitCoordinates[1], 64); err != nil {
+		err = fmt.Errorf("longitude is not a number: %w", err)
+	}
+	return
+}
+
+func (mc *MessageConverter) reuploadMediaToWhatsApp(ctx context.Context, evt *event.Event, content *event.MessageEventContent) (*waMediaTransport.WAMediaTransport, string, error) {
+	data, mimeType, fileName, err := mc.downloadMatrixMedia(ctx, content)
+	if err != nil {
+		return nil, "", err
+	}
+	_, isVoice := evt.Content.Raw["org.matrix.msc3245.voice"]
+	if isVoice {
+		data, err = ffmpeg.ConvertBytes(ctx, data, ".m4a", []string{}, []string{"-c:a", "aac"}, mimeType)
+		if err != nil {
+			return nil, "", fmt.Errorf("%w voice message to m4a: %w", ErrMediaConvertFailed, err)
+		}
+		mimeType = "audio/mp4"
+		fileName += ".m4a"
+	} else if mimeType == "image/gif" && content.MsgType == event.MsgImage {
+		data, err = ffmpeg.ConvertBytes(ctx, data, ".mp4", []string{"-f", "gif"}, []string{
+			"-pix_fmt", "yuv420p", "-c:v", "libx264", "-movflags", "+faststart",
+			"-filter:v", "crop='floor(in_w/2)*2:floor(in_h/2)*2'",
+		}, mimeType)
+		if err != nil {
+			return nil, "", fmt.Errorf("%w gif to mp4: %w", ErrMediaConvertFailed, err)
+		}
+		mimeType = "video/mp4"
+		fileName += ".mp4"
+		content.MsgType = event.MsgVideo
+		customInfo, ok := evt.Content.Raw["info"].(map[string]any)
+		if !ok {
+			customInfo = make(map[string]any)
+			evt.Content.Raw["info"] = customInfo
+		}
+		customInfo["fi.mau.gif"] = true
+	}
+	mediaType := msgToMediaType(content.MsgType)
+	uploaded, err := mc.GetE2EEClient(ctx).Upload(ctx, data, mediaType)
+	if err != nil {
+		return nil, "", err
+	}
+	mediaTransport := &waMediaTransport.WAMediaTransport{
+		Integral: &waMediaTransport.WAMediaTransport_Integral{
+			FileSHA256:        uploaded.FileSHA256,
+			MediaKey:          uploaded.MediaKey,
+			FileEncSHA256:     uploaded.FileEncSHA256,
+			DirectPath:        uploaded.DirectPath,
+			MediaKeyTimestamp: time.Now().UnixMilli(),
+		},
+		Ancillary: &waMediaTransport.WAMediaTransport_Ancillary{
+			FileLength: uint64(len(data)),
+			Mimetype:   mimeType,
+			Thumbnail:  nil,
+			ObjectID:   uploaded.ObjectID,
+		},
+	}
+	return mediaTransport, fileName, nil
+}
+
+func (mc *MessageConverter) wrapWhatsAppMedia(
+	evt *event.Event,
+	content *event.MessageEventContent,
+	reuploaded *waMediaTransport.WAMediaTransport,
+	caption *waCommon.MessageText,
+	fileName string,
+) (output waConsumerApplication.ConsumerApplication_Content_Content, err error) {
+	switch content.MsgType {
+	case event.MsgImage:
+		imageMsg := &waConsumerApplication.ConsumerApplication_ImageMessage{
+			Caption: caption,
+		}
+		err = imageMsg.Set(&waMediaTransport.ImageTransport{
+			Integral: &waMediaTransport.ImageTransport_Integral{
+				Transport: reuploaded,
+			},
+			Ancillary: &waMediaTransport.ImageTransport_Ancillary{
+				Height: uint32(content.Info.Height),
+				Width:  uint32(content.Info.Width),
+			},
+		})
+		output = &waConsumerApplication.ConsumerApplication_Content_ImageMessage{ImageMessage: imageMsg}
+	case event.MessageType(event.EventSticker.Type):
+		stickerMsg := &waConsumerApplication.ConsumerApplication_StickerMessage{}
+		err = stickerMsg.Set(&waMediaTransport.StickerTransport{
+			Integral: &waMediaTransport.StickerTransport_Integral{
+				Transport: reuploaded,
+			},
+			Ancillary: &waMediaTransport.StickerTransport_Ancillary{
+				Height: uint32(content.Info.Height),
+				Width:  uint32(content.Info.Width),
+			},
+		})
+		output = &waConsumerApplication.ConsumerApplication_Content_StickerMessage{StickerMessage: stickerMsg}
+	case event.MsgVideo:
+		videoMsg := &waConsumerApplication.ConsumerApplication_VideoMessage{
+			Caption: caption,
+		}
+		customInfo, _ := evt.Content.Raw["info"].(map[string]any)
+		isGif, _ := customInfo["fi.mau.gif"].(bool)
+
+		err = videoMsg.Set(&waMediaTransport.VideoTransport{
+			Integral: &waMediaTransport.VideoTransport_Integral{
+				Transport: reuploaded,
+			},
+			Ancillary: &waMediaTransport.VideoTransport_Ancillary{
+				Height:      uint32(content.Info.Height),
+				Width:       uint32(content.Info.Width),
+				Seconds:     uint32(content.Info.Duration / 1000),
+				GifPlayback: isGif,
+			},
+		})
+		output = &waConsumerApplication.ConsumerApplication_Content_VideoMessage{VideoMessage: videoMsg}
+	case event.MsgAudio:
+		_, isVoice := evt.Content.Raw["org.matrix.msc3245.voice"]
+		audioMsg := &waConsumerApplication.ConsumerApplication_AudioMessage{
+			PTT: isVoice,
+		}
+		err = audioMsg.Set(&waMediaTransport.AudioTransport{
+			Integral: &waMediaTransport.AudioTransport_Integral{
+				Transport: reuploaded,
+			},
+			Ancillary: &waMediaTransport.AudioTransport_Ancillary{
+				Seconds: uint32(content.Info.Duration / 1000),
+			},
+		})
+		output = &waConsumerApplication.ConsumerApplication_Content_AudioMessage{AudioMessage: audioMsg}
+	case event.MsgFile:
+		documentMsg := &waConsumerApplication.ConsumerApplication_DocumentMessage{
+			FileName: fileName,
+		}
+		err = documentMsg.Set(&waMediaTransport.DocumentTransport{
+			Integral: &waMediaTransport.DocumentTransport_Integral{
+				Transport: reuploaded,
+			},
+			Ancillary: &waMediaTransport.DocumentTransport_Ancillary{},
+		})
+		output = &waConsumerApplication.ConsumerApplication_Content_DocumentMessage{DocumentMessage: documentMsg}
+	}
+	return
+}
+
+func msgToMediaType(msgType event.MessageType) whatsmeow.MediaType {
+	switch msgType {
+	case event.MsgImage, event.MessageType(event.EventSticker.Type):
+		return whatsmeow.MediaImage
+	case event.MsgVideo:
+		return whatsmeow.MediaVideo
+	case event.MsgAudio:
+		return whatsmeow.MediaAudio
+	case event.MsgFile:
+		fallthrough
+	default:
+		return whatsmeow.MediaDocument
+	}
 }

portal.go

@@ -31,6 +31,7 @@ import (
 	"go.mau.fi/util/variationselector"
 	"go.mau.fi/whatsmeow"
 	"go.mau.fi/whatsmeow/binary/armadillo/waConsumerApplication"
+	"go.mau.fi/whatsmeow/binary/armadillo/waMsgApplication"
 	"go.mau.fi/whatsmeow/types"
 	"go.mau.fi/whatsmeow/types/events"
 	"maunium.net/go/mautrix"
@@ -513,10 +514,11 @@ func (portal *Portal) handleMatrixMessage(ctx context.Context, sender *User, evt
 	var otid int64
 	var tasks []socket.Task
 	var waMsg *waConsumerApplication.ConsumerApplication
+	var waMeta *waMsgApplication.MessageApplication_Metadata
 	var err error
 	if portal.ThreadType.IsWhatsApp() {
 		ctx = context.WithValue(ctx, msgconvContextKeyE2EEClient, sender.E2EEClient)
-		waMsg, err = portal.MsgConv.ToWhatsApp(ctx, evt, content, relaybotFormatted)
+		waMsg, waMeta, err = portal.MsgConv.ToWhatsApp(ctx, evt, content, relaybotFormatted)
 	} else {
 		ctx = context.WithValue(ctx, msgconvContextKeyClient, sender.Client)
 		tasks, otid, err = portal.MsgConv.ToMeta(ctx, evt, content, relaybotFormatted)
@@ -533,7 +535,7 @@ func (portal *Portal) handleMatrixMessage(ctx context.Context, sender *User, evt
 	if waMsg != nil {
 		messageID := sender.E2EEClient.GenerateMessageID()
 		var resp whatsmeow.SendResponse
-		resp, err = sender.E2EEClient.SendFBMessage(ctx, portal.JID(), waMsg, nil, whatsmeow.SendRequestExtra{
+		resp, err = sender.E2EEClient.SendFBMessage(ctx, portal.JID(), waMsg, waMeta, whatsmeow.SendRequestExtra{
 			ID: messageID,
 		})
 		// TODO save message in db before sending and only update timestamp later
@@ -922,6 +924,7 @@ func (portal *Portal) GetMetaReply(ctx context.Context, content *event.MessageEv
 			ReplyMessageId:  replyToMsg.ID,
 			ReplySourceType: 1,
 			ReplyType:       0,
+			ReplySender:     replyToMsg.Sender,
 		}
 	}
 	return nil