Dima/matrix-rust-sdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(widget): Support for avatars in calls via msc4039 (#6354)

Browse Source 
Allows to get the avatars in Element Call widget !
Using
[MSC4039](https://github.com/nordeck/matrix-spec-proposals/blob/nic/feat/widgetapi-upload-files/proposals/4039-widget-api-media.md)

Counter part of web support:
- https://github.com/element-hq/element-web/pull/32755
- https://github.com/element-hq/element-call/pull/3780

Needs a EC PR to support passing data as base64 via the widget json API
- https://github.com/element-hq/element-call/pull/3818
This commit is contained in:
Valere Fedronic
2026-04-02 12:35:22 +02:00
committed by GitHub
parent 7cea5be9e2
commit fcce02fc11
11 changed files with 301 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bindings/matrix-sdk-ffi/src/widget.rs
+8
View File
@@ -266,6 +266,7 @@ pub fn get_element_call_required_permissions(
requires_client: true,
update_delayed_event: true,
send_delayed_event: true,
download_files: true,
}
}
@@ -331,6 +332,8 @@ pub struct WidgetCapabilities {
pub update_delayed_event: bool,
/// This allows the widget to send events with a delay.
pub send_delayed_event: bool,
/// This allows the widget to download files (avatars)
pub download_files: bool,
}
impl From<WidgetCapabilities> for matrix_sdk::widget::Capabilities {
@@ -341,6 +344,7 @@ impl From<WidgetCapabilities> for matrix_sdk::widget::Capabilities {
requires_client: value.requires_client,
update_delayed_event: value.update_delayed_event,
send_delayed_event: value.send_delayed_event,
download_file: value.download_files,
}
}
}
@@ -353,6 +357,7 @@ impl From<matrix_sdk::widget::Capabilities> for WidgetCapabilities {
requires_client: value.requires_client,
update_delayed_event: value.update_delayed_event,
send_delayed_event: value.send_delayed_event,
download_files: value.download_file,
}
}
}
@@ -553,5 +558,8 @@ mod tests {
// RTC decline
cap_assert("org.matrix.msc2762.receive.event:org.matrix.msc4310.rtc.decline");
cap_assert("org.matrix.msc2762.send.event:org.matrix.msc4310.rtc.decline");
// Download avatars
cap_assert("org.matrix.msc4039.download_file");
}
}
crates/matrix-sdk/CHANGELOG.md
+3
View File
@@ -89,6 +89,9 @@ All notable changes to this project will be documented in this file.
to true will now trigger a download of all historical keys for the room in
question from the client's key backup.
([#6017](https://github.com/matrix-org/matrix-rust-sdk/pull/6017))
- Add widget partial support for MSC4039. Allows widgets to download non-encrypted files from the
content repository (like avatars).
([#6354](https://github.com/matrix-org/matrix-rust-sdk/pull/6354))
### Bugfix
crates/matrix-sdk/src/widget/capabilities.rs
+17 -1
View File
@@ -57,6 +57,9 @@ pub struct Capabilities {
pub update_delayed_event: bool,
/// This allows the widget to send events with a delay.
pub send_delayed_event: bool,
/// This allows the widget to download files as per MSC4039.
pub download_file: bool,
}
impl Capabilities {
@@ -114,6 +117,8 @@ pub(super) const REQUIRES_CLIENT: &str = "io.element.requires_client";
pub(super) const SEND_DELAYED_EVENT: &str = "org.matrix.msc4157.send.delayed_event";
pub(super) const UPDATE_DELAYED_EVENT: &str = "org.matrix.msc4157.update_delayed_event";
pub(super) const DOWNLOAD_FILE: &str = "org.matrix.msc4039.download_file";
impl Serialize for Capabilities {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
@@ -174,6 +179,9 @@ impl Serialize for Capabilities {
if self.send_delayed_event {
seq.serialize_element(SEND_DELAYED_EVENT)?;
}
if self.download_file {
seq.serialize_element(DOWNLOAD_FILE)?;
}
for filter in &self.read {
let name = match filter {
Filter::MessageLike(_) => READ_EVENT,
@@ -204,6 +212,7 @@ impl<'de> Deserialize<'de> for Capabilities {
RequiresClient,
UpdateDelayedEvent,
SendDelayedEvent,
DownloadFile,
Read(Filter),
Send(Filter),
Unknown,
@@ -224,6 +233,9 @@ impl<'de> Deserialize<'de> for Capabilities {
if s == SEND_DELAYED_EVENT {
return Ok(Self::SendDelayedEvent);
}
if s == DOWNLOAD_FILE {
return Ok(Self::DownloadFile);
}
match s.split_once(':') {
Some((READ_EVENT, filter_s)) => Ok(Permission::Read(Filter::MessageLike(
@@ -284,6 +296,7 @@ impl<'de> Deserialize<'de> for Capabilities {
Permission::Unknown => {}
Permission::UpdateDelayedEvent => capabilities.update_delayed_event = true,
Permission::SendDelayedEvent => capabilities.send_delayed_event = true,
Permission::DownloadFile => capabilities.download_file = true,
}
}
@@ -321,7 +334,8 @@ mod tests {
"org.matrix.msc2762.send.state_event:org.matrix.msc3401.call.member#@user:matrix.server",
"org.matrix.msc3819.send.to_device:io.element.call.encryption_keys",
"org.matrix.msc4157.send.delayed_event",
"org.matrix.msc4157.update_delayed_event"
"org.matrix.msc4157.update_delayed_event",
"org.matrix.msc4039.download_file"
]"#;
let parsed = serde_json::from_str::<Capabilities>(capabilities_str).unwrap();
@@ -351,6 +365,7 @@ mod tests {
requires_client: true,
update_delayed_event: true,
send_delayed_event: true,
download_file: true,
};
assert_eq!(parsed, expected);
@@ -381,6 +396,7 @@ mod tests {
requires_client: true,
update_delayed_event: false,
send_delayed_event: false,
download_file: false,
};
let capabilities_str = serde_json::to_string(&capabilities).unwrap();
crates/matrix-sdk/src/widget/machine/driver_req.rs
+21 -2
View File
@@ -17,7 +17,7 @@
use std::{collections::BTreeMap, marker::PhantomData};
use ruma::{
OwnedUserId,
OwnedMxcUri, OwnedUserId,
api::client::{account::request_openid_token, delayed_events::update_delayed_event},
events::{AnyStateEvent, AnyTimelineEvent, AnyToDeviceEventContent},
serde::Raw,
@@ -31,7 +31,7 @@ use super::{
Action, MatrixDriverRequestMeta, SendToDeviceEventResponse, WidgetMachine,
from_widget::SendEventResponse, incoming::MatrixDriverResponse,
};
use crate::widget::{Capabilities, StateKeySelector};
use crate::widget::{Capabilities, StateKeySelector, machine::from_widget::DownloadFileResponse};
#[derive(Clone, Debug)]
pub(crate) enum MatrixDriverRequestData {
@@ -59,6 +59,9 @@ pub(crate) enum MatrixDriverRequestData {
/// Data for sending a UpdateDelayedEvent client server api request.
UpdateDelayedEvent(UpdateDelayedEventRequest),
/// Request a download of a file.
DownloadFile(DownloadFileRequest),
}
/// A handle to a pending `toWidget` request.
@@ -339,3 +342,19 @@ impl FromMatrixDriverResponse for update_delayed_event::unstable::Response {
}
}
}
#[derive(Deserialize, Debug, Clone)]
pub(crate) struct DownloadFileRequest {
// The MXC url of the file to download.
pub(crate) content_uri: OwnedMxcUri,
}
impl MatrixDriverRequest for DownloadFileRequest {
type Response = DownloadFileResponse;
}
impl From<DownloadFileRequest> for MatrixDriverRequestData {
fn from(req: DownloadFileRequest) -> Self {
MatrixDriverRequestData::DownloadFile(req)
}
}
crates/matrix-sdk/src/widget/machine/from_widget.rs
+35 -2
View File
@@ -22,7 +22,7 @@ use ruma::{
error::{ErrorBody, StandardErrorBody},
},
events::AnyTimelineEvent,
serde::Raw,
serde::{Base64, Raw},
};
use serde::{Deserialize, Serialize};
use tracing::error;
@@ -33,7 +33,10 @@ use super::{
};
use crate::{
Error, HttpError, RumaApiError,
widget::{StateKeySelector, machine::driver_req::FromMatrixDriverResponse},
widget::{
StateKeySelector,
machine::driver_req::{DownloadFileRequest, FromMatrixDriverResponse},
},
};
#[derive(Deserialize, Debug)]
@@ -49,6 +52,8 @@ pub(super) enum FromWidgetRequest {
SendToDevice(SendToDeviceRequest),
#[serde(rename = "org.matrix.msc4157.update_delayed_event")]
DelayedEventUpdate(UpdateDelayedEventRequest),
#[serde(rename = "org.matrix.msc4039.download_file")]
DownloadFile(DownloadFileRequest),
}
/// The full response a client sends to a [`FromWidgetRequest`] in case of an
@@ -145,6 +150,7 @@ impl SupportedApiVersionsResponse {
ApiVersion::MSC2762UpdateState,
ApiVersion::MSC2871,
ApiVersion::MSC3819,
ApiVersion::MSC4039,
],
}
}
@@ -192,6 +198,9 @@ pub(super) enum ApiVersion {
/// Supports access to the TURN servers.
#[serde(rename = "town.robin.msc3846")]
MSC3846,
#[serde(rename = "org.matrix.msc4039")]
MSC4039,
}
#[derive(Deserialize, Debug)]
@@ -276,3 +285,27 @@ impl FromMatrixDriverResponse for SendToDeviceEventResponse {
}
}
}
/// Response for a download file request.
/// https://github.com/matrix-org/matrix-spec-proposals/pull/4039
#[derive(Serialize, Debug)]
pub(crate) struct DownloadFileResponse {
// The binary file content in a format that can cross the
// widget-driver-api boundary.
#[serde(rename = "file")]
pub(crate) file_data_base64: Base64,
}
impl FromMatrixDriverResponse for DownloadFileResponse {
fn from_response(matrix_driver_response: MatrixDriverResponse) -> Option<Self> {
match matrix_driver_response {
MatrixDriverResponse::FileDownloaded(resp) => {
Some(Self { file_data_base64: resp.file_data_base64 })
}
_ => {
error!("bug in MatrixDriver, received wrong event response");
None
}
}
}
}
crates/matrix-sdk/src/widget/machine/incoming.rs
+58 -1
View File
@@ -28,7 +28,7 @@ use super::{
from_widget::{FromWidgetRequest, SendEventResponse},
to_widget::ToWidgetResponse,
};
use crate::widget::Capabilities;
use crate::widget::{Capabilities, machine::from_widget::DownloadFileResponse};
/// Incoming message for the widget client side module that it must process.
pub(crate) enum IncomingMessage {
@@ -87,6 +87,8 @@ pub(crate) enum MatrixDriverResponse {
/// Client updated a delayed event.
/// A response to a [`MatrixDriverRequestData::UpdateDelayedEvent`] command.
DelayedEventUpdated(delayed_events::update_delayed_event::unstable::Response),
/// The client successfully downloaded a file from a widget action.
FileDownloaded(DownloadFileResponse),
}
pub(super) struct IncomingWidgetMessage {
@@ -136,3 +138,58 @@ impl<'de> Deserialize<'de> for IncomingWidgetMessage {
Ok(Self { widget_id, request_id, kind })
}
}
#[cfg(test)]
mod tests {
use assert_matches2::assert_let;
use crate::widget::machine::{
from_widget::FromWidgetRequest,
incoming::{IncomingWidgetMessage, IncomingWidgetMessageKind},
};
#[test]
fn parse_download_file_widget_action() {
let raw = r#"
{
"api": "fromWidget",
"widgetId": "aGNStSuL3hhIISSCXgpt15j2",
"requestId": "generated-id-1234",
"action": "org.matrix.msc4039.download_file",
"data": {
"content_uri": "mxc://server/id"
}
}
"#;
assert_let!(
IncomingWidgetMessageKind::Request(incoming_request) =
serde_json::from_str::<IncomingWidgetMessage>(raw).unwrap().kind
);
assert_let!(FromWidgetRequest::DownloadFile(req) = incoming_request.deserialize().unwrap());
assert_eq!(req.content_uri, "mxc://server/id");
}
#[test]
fn parse_download_file_request_with_non_mxc_url() {
let raw = r#"
{
"api": "fromWidget",
"widgetId": "aGNStSuL3hhIISSCXgpt15j2",
"requestId": "generated-id-1234",
"action": "org.matrix.msc4039.download_file",
"data": {
"content_uri": "https://server/id"
}
}
"#;
assert_let!(
IncomingWidgetMessageKind::Request(incoming_request) =
serde_json::from_str::<IncomingWidgetMessage>(raw).unwrap().kind
);
assert_let!(FromWidgetRequest::DownloadFile(req) = incoming_request.deserialize().unwrap());
assert!(!req.content_uri.is_valid());
}
}
crates/matrix-sdk/src/widget/machine/mod.rs
+46 -2
View File
@@ -53,7 +53,10 @@ use super::{
capabilities::{SEND_DELAYED_EVENT, UPDATE_DELAYED_EVENT},
filter::FilterInput,
};
use crate::{Error, Result, widget::Filter};
use crate::{
Error, Result,
widget::{Filter, capabilities::DOWNLOAD_FILE, machine::driver_req::DownloadFileRequest},
};
mod driver_req;
mod from_widget;
@@ -66,7 +69,7 @@ mod to_widget;
pub(crate) use self::{
driver_req::{MatrixDriverRequestData, SendEventRequest, SendToDeviceRequest},
from_widget::{SendEventResponse, SendToDeviceEventResponse},
from_widget::{DownloadFileResponse, SendEventResponse, SendToDeviceEventResponse},
incoming::{IncomingMessage, MatrixDriverResponse},
};
@@ -381,6 +384,10 @@ impl WidgetMachine {
})
.unwrap_or_default()
}
FromWidgetRequest::DownloadFile(req) => self
.process_download_file_request(req, raw_request)
.map(|a| vec![a])
.unwrap_or_default(),
}
}
@@ -884,6 +891,43 @@ impl WidgetMachine {
actions
}
fn process_download_file_request(
&mut self,
req: DownloadFileRequest,
raw_request: Raw<FromWidgetRequest>,
) -> Option<Action> {
let CapabilitiesState::Negotiated(capabilities) = &self.capabilities else {
return Some(Self::send_from_widget_error_string_response(
raw_request,
"Received download file request before capabilities negotiation",
));
};
if !capabilities.download_file {
return Some(Self::send_from_widget_error_string_response(
raw_request,
format!("Not allowed: missing the {DOWNLOAD_FILE} capability."),
));
}
if !req.content_uri.is_valid() {
return Some(Self::send_from_widget_error_string_response(
raw_request,
"Invalid content URI",
));
}
let (request, action) = self.send_matrix_driver_request(req)?;
request.add_response_handler(|result, _| {
vec![Self::send_from_widget_response(
raw_request,
result.map_err(FromWidgetErrorResponse::from_error),
)]
});
Some(action)
}
}
type ToWidgetResponseFn =
crates/matrix-sdk/src/widget/machine/tests/api_versions.rs
+1
View File
@@ -51,6 +51,7 @@ fn test_get_supported_api_versions() {
"org.matrix.msc2762_update_state",
"org.matrix.msc2871",
"org.matrix.msc3819",
"org.matrix.msc4039",
]
},
}),
crates/matrix-sdk/src/widget/matrix.rs
+13 -2
View File
@@ -21,10 +21,11 @@ use as_variant::as_variant;
use matrix_sdk_base::{
crypto::CollectStrategy,
deserialized_responses::{EncryptionInfo, RawAnySyncOrStrippedState},
media::{MediaFormat, MediaRequestParameters},
sync::State,
};
use ruma::{
EventId, OwnedDeviceId, OwnedUserId, RoomId, TransactionId,
EventId, OwnedDeviceId, OwnedMxcUri, OwnedUserId, RoomId, TransactionId,
api::client::{
account::request_openid_token::v3::{Request as OpenIdRequest, Response as OpenIdResponse},
delayed_events::{self, update_delayed_event::unstable::UpdateAction},
@@ -36,8 +37,9 @@ use ruma::{
AnyMessageLikeEventContent, AnyStateEvent, AnyStateEventContent, AnySyncStateEvent,
AnySyncTimelineEvent, AnyTimelineEvent, AnyToDeviceEvent, AnyToDeviceEventContent,
MessageLikeEventType, StateEventType, TimelineEventType, ToDeviceEventType,
room::MediaSource,
},
serde::{Raw, from_raw_json_value},
serde::{Base64, Raw, from_raw_json_value},
to_device::DeviceIdOrAllDevices,
};
use serde::{Deserialize, Serialize};
@@ -144,6 +146,15 @@ impl MatrixDriver {
Ok(events)
}
pub(crate) async fn download_attachment(&self, mxc_uri: OwnedMxcUri) -> Result<Base64> {
let req = MediaRequestParameters {
source: MediaSource::Plain(mxc_uri.to_owned()),
format: MediaFormat::File,
};
let response = self.room.client().media().get_media_content(&req, true).await?;
Ok(Base64::new(response))
}
/// Sends the given `event` to the room.
///
/// This method allows the widget machine to handle widget requests by
crates/matrix-sdk/src/widget/mod.rs
+9 -1
View File
@@ -33,7 +33,7 @@ use self::{
},
matrix::MatrixDriver,
};
use crate::{Result, room::Room};
use crate::{Result, room::Room, widget::machine::DownloadFileResponse};
mod capabilities;
mod filter;
@@ -252,6 +252,14 @@ impl WidgetDriver {
.await
.map(MatrixDriverResponse::ToDeviceSent)
}
MatrixDriverRequestData::DownloadFile(req) => matrix_driver
.download_attachment(req.content_uri)
.await
.map(|file_data_base64| {
MatrixDriverResponse::FileDownloaded(DownloadFileResponse {
file_data_base64,
})
}),
};
// Forward the Matrix driver response to the incoming message stream.
crates/matrix-sdk/tests/integration/widget.rs
+90 -1
View File
@@ -45,7 +45,7 @@ use ruma::{
room::{member::MembershipState, message::RoomMessageEventContent},
},
owned_room_id, room_id,
serde::{JsonObject, Raw},
serde::{Base64, JsonObject, Raw},
to_device::DeviceIdOrAllDevices,
user_id,
};
@@ -1842,6 +1842,95 @@ async fn test_send_encrypted_to_device_different_content() {
}
}
#[async_test]
async fn test_try_download_file_without_permission() {
let (_, _mock_server, driver_handle) = run_test_driver(false, false).await;
negotiate_capabilities(&driver_handle, json!([])).await;
send_request(
&driver_handle,
"000",
"org.matrix.msc4039.download_file",
json!({
"content_uri":"mxc://example.org/profile.jpg",
}),
)
.await;
let response = recv_message(&driver_handle).await;
if response["api"] == "fromWidget" && response["action"] == "org.matrix.msc4039.download_file" {
let error_response = response["response"]["error"]["message"].clone();
assert_eq!(
error_response.as_str().unwrap(),
"Not allowed: missing the org.matrix.msc4039.download_file capability."
);
}
}
#[async_test]
async fn test_download_non_mxc_uri_should_fail() {
let (_, _mock_server, driver_handle) = run_test_driver(false, false).await;
negotiate_capabilities(&driver_handle, json!(["org.matrix.msc4039.download_file"])).await;
send_request(
&driver_handle,
"000",
"org.matrix.msc4039.download_file",
json!({
"content_uri":"https://example.org/profile.jpg",
}),
)
.await;
let response = recv_message(&driver_handle).await;
if response["api"] == "fromWidget" && response["action"] == "org.matrix.msc4039.download_file" {
let error_response = response["response"]["error"]["message"].clone();
assert_eq!(error_response.as_str().unwrap(), "Invalid content URI");
}
}
#[async_test]
async fn test_try_download_file() {
let (_, mock_server, driver_handle) = run_test_driver(false, false).await;
negotiate_capabilities(&driver_handle, json!(["org.matrix.msc4039.download_file"])).await;
send_request(
&driver_handle,
"000",
"org.matrix.msc4039.download_file",
json!({
"content_uri":"mxc://example.org/xJbofAzMprfEWsmWWqGsMuEY",
}),
)
.await;
let bundle = vec![1, 2, 3, 4, 5];
mock_server
.mock_authed_media_download()
.expect_any_access_token()
.ok_bytes(bundle.clone())
.mock_once()
.named("media_download")
.mount()
.await;
let response = recv_message(&driver_handle).await;
print!("{response:?}");
assert_eq!(response["api"], "fromWidget");
assert_eq!(response["action"], "org.matrix.msc4039.download_file");
assert_eq!(response["requestId"], "000");
let response = response["response"].as_object().expect("response is an object");
let base64 = response.get("file").unwrap().as_str().unwrap();
let decoded: Base64 = Base64::parse(base64).expect("valid base64 in widget response");
assert_eq!(decoded.as_bytes(), bundle.as_slice());
}
async fn negotiate_capabilities(driver_handle: &WidgetDriverHandle, caps: JsonValue) {
{
// Receive toWidget capabilities request