feat(send queue): allow sending attachments with the send queue

bindings/matrix-sdk-ffi/src/error.rs

@@ -183,6 +183,12 @@ pub enum QueueWedgeError {
     /// session before sending.
     CrossVerificationRequired,
 
+    /// Some media content to be sent has disappeared from the cache.
+    MissingMediaContent,
+
+    /// Some mime type couldn't be parsed.
+    InvalidMimeType { mime_type: String },
+
     /// Other errors.
     GenericApiError { msg: String },
 }
@@ -201,10 +207,17 @@ impl Display for QueueWedgeError {
             QueueWedgeError::CrossVerificationRequired => {
                 f.write_str("Own verification is required")
             }
+            QueueWedgeError::MissingMediaContent => {
+                f.write_str("Media to be sent disappeared from local storage")
+            }
+            QueueWedgeError::InvalidMimeType { mime_type } => {
+                write!(f, "Invalid mime type '{mime_type}' for media upload")
+            }
             QueueWedgeError::GenericApiError { msg } => f.write_str(msg),
         }
     }
 }
+
 impl From<SdkQueueWedgeError> for QueueWedgeError {
     fn from(value: SdkQueueWedgeError) -> Self {
         match value {
@@ -223,6 +236,10 @@ impl From<SdkQueueWedgeError> for QueueWedgeError {
                 users: users.iter().map(ruma::OwnedUserId::to_string).collect(),
             },
             SdkQueueWedgeError::CrossVerificationRequired => Self::CrossVerificationRequired,
+            SdkQueueWedgeError::MissingMediaContent => Self::MissingMediaContent,
+            SdkQueueWedgeError::InvalidMimeType { mime_type } => {
+                Self::InvalidMimeType { mime_type }
+            }
             SdkQueueWedgeError::GenericApiError { msg } => Self::GenericApiError { msg },
         }
     }

crates/matrix-sdk-base/src/media.rs

@@ -14,6 +14,7 @@ use ruma::{
     },
     MxcUri, UInt,
 };
+use serde::{Deserialize, Serialize};
 
 const UNIQUE_SEPARATOR: &str = "_";
 
@@ -25,7 +26,7 @@ pub trait UniqueKey {
 }
 
 /// The requested format of a media file.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub enum MediaFormat {
     /// The file that was uploaded.
     File,
@@ -44,7 +45,7 @@ impl UniqueKey for MediaFormat {
 }
 
 /// The requested size of a media thumbnail.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct MediaThumbnailSize {
     /// The desired resizing method.
     pub method: Method,
@@ -65,7 +66,7 @@ impl UniqueKey for MediaThumbnailSize {
 }
 
 /// The desired settings of a media thumbnail.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct MediaThumbnailSettings {
     /// The desired size of the thumbnail.
     pub size: MediaThumbnailSize,
@@ -110,7 +111,7 @@ impl UniqueKey for MediaSource {
 }
 
 /// A request for media data.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct MediaRequest {
     /// The source of the media file.
     pub source: MediaSource,

crates/matrix-sdk-base/src/store/integration_tests.rs

@@ -1406,7 +1406,9 @@ impl StateStoreIntegrationTests for DynStateStore {
         assert_eq!(dependents.len(), 1);
         assert_eq!(dependents[0].parent_transaction_id, txn0);
         assert_eq!(dependents[0].own_transaction_id, child_txn);
-        assert_eq!(dependents[0].parent_key.as_ref(), Some(&SentRequestKey::Event(event_id)));
+        assert_matches!(dependents[0].parent_key.as_ref(), Some(&SentRequestKey::Event(ref eid)) => {
+            assert_eq!(*eid, event_id);
+        });
         assert_matches!(dependents[0].kind, DependentQueuedRequestKind::RedactEvent);
 
         // Now remove it.

crates/matrix-sdk-base/src/store/mod.rs

@@ -75,8 +75,9 @@ pub use self::integration_tests::StateStoreIntegrationTests;
 pub use self::{
     memory_store::MemoryStore,
     send_queue::{
-        ChildTransactionId, DependentQueuedRequest, DependentQueuedRequestKind, QueueWedgeError,
-        QueuedRequest, QueuedRequestKind, SentRequestKey, SerializableEventContent,
+        ChildTransactionId, DependentQueuedRequest, DependentQueuedRequestKind,
+        FinishUploadThumbnailInfo, QueueWedgeError, QueuedRequest, QueuedRequestKind,
+        SentRequestKey, SerializableEventContent,
     },
     traits::{
         ComposerDraft, ComposerDraftType, DynStateStore, IntoStateStore, ServerCapabilities,

crates/matrix-sdk-base/src/store/send_queue.rs

@@ -18,12 +18,17 @@ use std::{collections::BTreeMap, fmt, ops::Deref};
 
 use as_variant::as_variant;
 use ruma::{
-    events::{AnyMessageLikeEventContent, EventContent as _, RawExt as _},
+    events::{
+        room::{message::RoomMessageEventContent, MediaSource},
+        AnyMessageLikeEventContent, EventContent as _, RawExt as _,
+    },
     serde::Raw,
-    OwnedDeviceId, OwnedEventId, OwnedTransactionId, OwnedUserId, TransactionId,
+    OwnedDeviceId, OwnedEventId, OwnedTransactionId, OwnedUserId, TransactionId, UInt,
 };
 use serde::{Deserialize, Serialize};
 
+use crate::media::MediaRequest;
+
 /// A thin wrapper to serialize a `AnyMessageLikeEventContent`.
 #[derive(Clone, Serialize, Deserialize)]
 pub struct SerializableEventContent {
@@ -76,6 +81,28 @@ pub enum QueuedRequestKind {
         /// The content of the message-like event we'd like to send.
         content: SerializableEventContent,
     },
+
+    /// Content to upload on the media server.
+    ///
+    /// The bytes must be stored in the media cache, and are identified by the
+    /// cache key.
+    Upload {
+        /// Content type of the media to be uploaded.
+        ///
+        /// Stored as a `String` because `Mime` which we'd really want to use
+        /// here, is not serializable. Oh well.
+        content_type: String,
+
+        /// The cache key used to retrieve the media's bytes in the event cache
+        /// store.
+        cache_key: MediaRequest,
+
+        /// An optional media source for a thumbnail already uploadd.
+        thumbnail_source: Option<MediaSource>,
+
+        /// To which media event transaction does this upload relate?
+        related_to: OwnedTransactionId,
+    },
 }
 
 impl From<SerializableEventContent> for QueuedRequestKind {
@@ -143,6 +170,18 @@ pub enum QueueWedgeError {
     #[error("Own verification is required")]
     CrossVerificationRequired,
 
+    /// Media content was cached in the media store, but has disappeared before
+    /// we could upload it.
+    #[error("Media content disappeared")]
+    MissingMediaContent,
+
+    /// We tried to upload some media content with an unknown mime type.
+    #[error("Invalid mime type '{mime_type}' for media")]
+    InvalidMimeType {
+        /// The observed mime type that's expected to be invalid.
+        mime_type: String,
+    },
+
     /// Other errors.
     #[error("Other unrecoverable error: {msg}")]
     GenericApiError {
@@ -169,6 +208,43 @@ pub enum DependentQueuedRequestKind {
         /// Key used for the reaction.
         key: String,
     },
+
+    /// Upload a file that had a thumbnail.
+    UploadFileWithThumbnail {
+        /// Content type for the file itself (not the thumbnail).
+        content_type: String,
+
+        /// Media request necessary to retrieve the file itself (not the
+        /// thumbnail).
+        cache_key: MediaRequest,
+
+        /// To which media transaction id does this upload relate to?
+        related_to: OwnedTransactionId,
+    },
+
+    /// Finish an upload by updating references to the media cache and sending
+    /// the final media event with the remote MXC URIs.
+    FinishUpload {
+        /// Local echo for the event (containing the local MXC URIs).
+        local_echo: RoomMessageEventContent,
+
+        /// Transaction id for the file upload.
+        file_upload: OwnedTransactionId,
+
+        /// Information about the thumbnail, if present.
+        thumbnail_info: Option<FinishUploadThumbnailInfo>,
+    },
+}
+
+/// Detailed record about a thumbnail used when finishing a media upload.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct FinishUploadThumbnailInfo {
+    /// Transaction id for the thumbnail upload.
+    pub txn: OwnedTransactionId,
+    /// Thumbnail's width.
+    pub width: UInt,
+    /// Thumbnail's height.
+    pub height: UInt,
 }
 
 /// A transaction id identifying a [`DependentQueuedRequest`] rather than its
@@ -210,14 +286,34 @@ impl From<ChildTransactionId> for OwnedTransactionId {
     }
 }
 
+impl From<OwnedTransactionId> for ChildTransactionId {
+    fn from(val: OwnedTransactionId) -> Self {
+        Self(val)
+    }
+}
+
 /// A unique key (identifier) indicating that a transaction has been
 /// successfully sent to the server.
 ///
 /// The owning child transactions can now be resolved.
-#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub enum SentRequestKey {
     /// The parent transaction returned an event when it succeeded.
     Event(OwnedEventId),
+
+    /// The parent transaction returned an uploaded resource URL.
+    Media {
+        /// File that was uploaded by this request.
+        ///
+        /// If the request related to a thumbnail upload, this contains the
+        /// thumbnail media source.
+        file: MediaSource,
+
+        /// Optional thumbnail previously uploaded, when uploading a file.
+        ///
+        /// When uploading a thumbnail, this is set to `None`.
+        thumbnail: Option<MediaSource>,
+    },
 }
 
 impl SentRequestKey {

crates/matrix-sdk-ui/src/timeline/controller/mod.rs

@@ -1359,6 +1359,11 @@ impl<P: RoomDataProvider> TimelineController<P> {
                 self.update_event_send_state(&transaction_id, EventSendState::Sent { event_id })
                     .await;
             }
+
+            RoomSendQueueUpdate::UploadedMedia { related_to } => {
+                // TODO(bnjbvr): Do something else?
+                info!(txn_id = %related_to, "some media for a media event has been uploaded");
+            }
         }
     }
 }

crates/matrix-sdk/src/room/mod.rs

@@ -2045,7 +2045,7 @@ impl Room {
     /// Creates the inner [`MessageType`] for an already-uploaded media file
     /// provided by its source.
     #[allow(clippy::too_many_arguments)]
-    fn make_attachment_type(
+    pub(crate) fn make_attachment_type(
         &self,
         content_type: &Mime,
         filename: &str,
@@ -2131,7 +2131,7 @@ impl Room {
 
     /// Creates the [`RoomMessageEventContent`] based on the message type and
     /// mentions.
-    fn make_attachment_event(
+    pub(crate) fn make_attachment_event(
         msg_type: MessageType,
         mentions: Option<Mentions>,
     ) -> RoomMessageEventContent {