docs: Adding release notes for 0.5.0

Merge pull request #657 from gnunicorn/ben-preping-for-release

.cargo/config.toml

@@ -1,2 +1,25 @@
+[alias]
+xtask = "run --package xtask --"
+
 [doc.extern-map.registries]
 crates-io = "https://docs.rs/"
+
+[target.'cfg(all())']
+rustflags = [
+    "-Wrust_2018_idioms",
+    "-Wsemicolon_in_expressions_from_macros",
+    "-Wunused_extern_crates",
+    "-Wunused_import_braces",
+    "-Wunused_qualifications",
+    "-Wtrivial_casts",
+    "-Wtrivial_numeric_casts",
+    "-Wclippy::cloned_instead_of_copied",
+    "-Wclippy::dbg_macro",
+    "-Wclippy::inefficient_to_string",
+    "-Wclippy::macro_use_imports",
+    "-Wclippy::mut_mut",
+    "-Wclippy::needless_borrow",
+    "-Wclippy::nonstandard_macro_braces",
+    "-Wclippy::str_to_string",
+    "-Wclippy::todo",
+]

.github/workflows/appservice.yml

@@ -0,0 +1,46 @@
+name: Appservice
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  test-appservice:
+    if: github.event_name == 'push' || !github.event.pull_request.draft
+    name: ${{ matrix.os }} / appservice / stable
+
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu, macOS]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          override: true
+
+      - name: Load cache
+        uses: Swatinem/rust-cache@v1
+
+      - name: Run checks
+        uses: actions-rs/cargo@v1
+        with:
+          command: run
+          args: -p xtask -- ci test-appservice

.github/workflows/benchmarks.yml

@@ -0,0 +1,56 @@
+name: Benchmarks
+
+on:
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    name: Run Benchmarks
+    runs-on: ubuntu-latest
+    environment: matrix-rust-bot
+    if: github.event_name == 'push' || !github.event.pull_request.draft
+
+    steps:
+    - name: Checkout the repo
+      uses: actions/checkout@v2
+
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: nightly
+        components: rustfmt
+        profile: minimal
+        override: true
+
+    - name: Run Benchmarks
+      run: cargo bench | tee benchmark-output.txt
+
+    - name: Check benchmark result for PR
+      if: github.event_name == 'pull_request'
+      uses: benchmark-action/github-action-benchmark@v1
+      with:
+        name: Rust Benchmark
+        tool: 'cargo'
+        output-file-path: benchmark-output.txt
+        auto-push: false
+        # comment to alert the user this has gone bad
+        github-token: ${{ secrets.MRB_ACCESS_TOKEN }}
+        alert-threshold: '120%'
+        comment-on-alert: true
+        fail-threshold: '150%'
+        fail-on-alert: true
+
+    - name: Store benchmark result
+      if: github.event_name != 'pull_request'
+      uses: benchmark-action/github-action-benchmark@v1
+      with:
+        name: Rust Benchmark
+        tool: 'cargo'
+        output-file-path: benchmark-output.txt
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        auto-push: true
+        # Show alert with commit comment on detecting possible performance regression
+        alert-threshold: '150%'
+        comment-on-alert: true
+        fail-on-alert: true
+        alert-comment-cc-users: '@gnunicornBen,@jplatte,@poljar'

.github/workflows/ci.yml

@@ -1,216 +1,91 @@
 name: CI
 
 on:
+  workflow_dispatch:
   push:
+    branches: [main]
   pull_request:
-    branches: [ master ]
+    branches: [main]
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  style:
-    name: Check style
+  test-features:
+    name: linux / features-${{ matrix.name }}
+    if: github.event_name == 'push' || !github.event.pull_request.draft
+
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        name:
+          - no-encryption
+          - no-sled
+          - no-encryption-and-sled
+          - sled-cryptostore
+          - rustls-tls
+          - markdown
+          - socks
+          - sso-login
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          override: true
+
+      - name: Load cache
+        uses: Swatinem/rust-cache@v1
+
+      - name: Test
+        uses: actions-rs/cargo@v1
+        with:
+          command: run
+          args: -p xtask -- ci test-features ${{ matrix.name }}
+
+  test-crypto-features:
+    name: linux / crypto-crate features
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || !github.event.pull_request.draft
 
     steps:
     - name: Checkout the repo
       uses: actions/checkout@v2
 
-    - name: Install rust
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: nightly
-        components: rustfmt
-        profile: minimal
-        override: true
-
-    - name: Cargo fmt
-      uses: actions-rs/cargo@v1
-      with:
-        command: fmt
-        args: --all -- --check
-
-  clippy:
-    name: Run clippy
-    needs: [style]
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout the repo
-      uses: actions/checkout@v2
-
-    - name: Install rust
+    - name: Install Rust
       uses: actions-rs/toolchain@v1
       with:
         toolchain: stable
-        components: clippy
         profile: minimal
         override: true
 
+    - name: Load cache
+      uses: Swatinem/rust-cache@v1
+
     - name: Clippy
       uses: actions-rs/cargo@v1
       with:
-        command: clippy
-        args: --all-targets -- -D warnings
-
-    - name: Clippy without default features
-      uses: actions-rs/cargo@v1
-      with:
-        command: clippy
-        # TODO: add `--all-targets` once all warnings in examples are resolved
-        args: --no-default-features --features native-tls,warp -- -D warnings
-
-  check-wasm:
-    name: linux / WASM
-    needs: [clippy]
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout the repo
-      uses: actions/checkout@v2
-
-    - name: Install rust
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        target: wasm32-unknown-unknown
-        profile: minimal
-        override: true
-
-    - name: Install emscripten
-      uses: mymindstorm/setup-emsdk@v7
-
-    - name: Check
-      run: |
-        cd matrix_sdk/examples/wasm_command_bot
-        cargo check --target wasm32-unknown-unknown
-
-  test-appservice:
-    name: ${{ matrix.name }}
-    needs: [clippy]
-
-    runs-on: ${{ matrix.os || 'ubuntu-latest' }}
-    strategy:
-      matrix:
-        name:
-          - linux / appservice / stable / warp
-          - macOS / appservice / stable / warp
-
-        include:
-          - name: linux / appservice / stable / warp
-            cargo_args: --features warp
-
-          - name: macOS / appservice / stable / warp
-            os: macOS-latest
-            cargo_args: --features warp
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v1
-
-      - name: Install rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: ${{ matrix.rust || 'stable' }}
-          target: ${{ matrix.target }}
-          profile: minimal
-          override: true
-
-      - name: Clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --manifest-path matrix_sdk_appservice/Cargo.toml ${{ matrix.cargo_args }} -- -D warnings
-
-      - name: Build
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --manifest-path matrix_sdk_appservice/Cargo.toml ${{ matrix.cargo_args }}
-
-      - name: Test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --manifest-path matrix_sdk_appservice/Cargo.toml ${{ matrix.cargo_args }}
-
-  test-features:
-    name: ${{ matrix.name }}
-    needs: [clippy]
-
-    runs-on: ${{ matrix.os || 'ubuntu-latest' }}
-    strategy:
-      matrix:
-        name:
-          - linux / features-no-encryption
-          - linux / features-no-sled
-          - linux / features-no-encryption-and-sled
-          - linux / features-sled_cryptostore
-          - linux / features-rustls-tls
-          - linux / features-markdown
-          - linux / features-socks
-          - linux / features-sso_login
-          - linux / features-require_auth_for_profile_requests
-
-        include:
-          - name: linux / features-no-encryption
-            cargo_args: --no-default-features --features "sled_state_store, native-tls"
-
-          - name: linux / features-no-sled
-            cargo_args: --no-default-features --features "encryption, native-tls"
-
-          - name: linux / features-no-encryption-and-sled
-            cargo_args: --no-default-features --features "native-tls"
-
-          - name: linux / features-sled_cryptostore
-            cargo_args: --no-default-features --features "encryption, sled_cryptostore, native-tls"
-
-          - name: linux / features-rustls-tls
-            cargo_args: --no-default-features --features rustls-tls
-
-          - name: linux / features-require_auth_for_profile_requests
-            cargo_args: --no-default-features --features "require_auth_for_profile_requests, native-tls"
-
-          - name: linux / features-markdown
-            cargo_args: --features markdown
-
-          - name: linux / features-socks
-            cargo_args: --features socks
-
-          - name: linux / features-sso_login
-            cargo_args: --features sso_login
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v1
-
-      - name: Install rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: ${{ matrix.rust || 'stable' }}
-          target: ${{ matrix.target }}
-          profile: minimal
-          override: true
-
-      - name: Check
-        uses: actions-rs/cargo@v1
-        with:
-          command: check
-          args: --manifest-path matrix_sdk/Cargo.toml ${{ matrix.cargo_args }}
-
-      - name: Test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --manifest-path matrix_sdk/Cargo.toml ${{ matrix.cargo_args }}
+        command: run
+        args: -p xtask -- ci test-crypto
 
   test:
     name: ${{ matrix.name }}
-    needs: [clippy]
+    if: github.event_name == 'push' || !github.event.pull_request.draft
 
     runs-on: ${{ matrix.os || 'ubuntu-latest' }}
     strategy:
+      fail-fast: true
       matrix:
         name:
           - linux / stable
@@ -230,18 +105,15 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v1
 
-      - name: Install rust
+      - name: Install Rust
         uses: actions-rs/toolchain@v1
         with:
           toolchain: ${{ matrix.rust || 'stable' }}
-          target: ${{ matrix.target }}
           profile: minimal
           override: true
 
-      - name: Build
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
+      - name: Load cache
+        uses: Swatinem/rust-cache@v1
 
       - name: Test
         uses: actions-rs/cargo@v1

.github/workflows/coverage.yml

@@ -2,7 +2,9 @@ name: Code coverage
 
 on:
   push:
-    branches: [ master ]
+    branches: [main]
+  pull_request:
+    branches: [main]
 
 env:
   CARGO_TERM_COLOR: always
@@ -16,24 +18,27 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v2
 
-    - name: Install stable toolchain
+    - name: Install Rust
       uses: actions-rs/toolchain@v1
       with:
         toolchain: stable
         profile: minimal
         override: true
 
+    - name: Load cache
+      uses: Swatinem/rust-cache@v1
+
     - name: Install tarpaulin
       uses: actions-rs/cargo@v1
       with:
         command: install
-        args: cargo-tarpaulin -f
+        args: cargo-tarpaulin
 
     - name: Run tarpaulin
       uses: actions-rs/cargo@v1
       with:
         command: tarpaulin
-        args: --ignore-config --exclude-files "matrix_sdk/examples/*,matrix_sdk_common,matrix_sdk_test" --out Xml
+        args: --ignore-config --exclude-files "crates/matrix-sdk/examples/*,crates/matrix-sdk-common,crates/matrix-sdk-test" --out Xml
 
     - name: Upload to codecov.io
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v3

.github/workflows/docs.yml

@@ -1,36 +1,41 @@
-name: docs
+name: Docs
 
 on:
   push:
-    branches: [master]
+    branches: [main]
   pull_request:
 
 jobs:
   docs:
-    name: docs
+    name: Docs
     runs-on: ubuntu-latest
+    if: github.event_name == 'push' || !github.event.pull_request.draft
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
 
-      - name: Install stable toolchain
+      - name: Install Rust
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
           toolchain: nightly
           override: true
 
+      - name: Load cache
+        uses: Swatinem/rust-cache@v1
+
+      # Keep in sync with xtask docs
       - name: Build docs
         uses: actions-rs/cargo@v1
         env:
-          RUSTDOCFLAGS: "--enable-index-page -Zunstable-options"
+          RUSTDOCFLAGS: "--enable-index-page -Zunstable-options --cfg docsrs -Dwarnings"
         with:
           command: doc
-          args: --no-deps --workspace --features docs -Zrustdoc-map
+          args: --no-deps --workspace --features docsrs -Zrustdoc-map
 
       - name: Deploy docs
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/style.yml

@@ -0,0 +1,105 @@
+name: Style
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  style:
+    name: Check Formatting
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || !github.event.pull_request.draft
+
+    steps:
+    - name: Checkout the repo
+      uses: actions/checkout@v2
+
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: nightly
+        components: rustfmt
+        profile: minimal
+        override: true
+
+    - name: Cargo fmt
+      uses: actions-rs/cargo@v1
+      with:
+        command: fmt
+        args: -- --check
+
+  pre-commit-styles:
+    name: Check Style
+    if: github.event_name == 'push' || !github.event.pull_request.draft
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Actions Repository
+      uses: actions/checkout@v2
+
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: nightly
+        components: clippy
+        profile: minimal
+        override: true
+
+    - name: Setup Python Env
+      uses: actions/setup-python@v2
+
+    - name: Check Styles with pre-commit
+      uses: pre-commit/action@v2.0.3
+      with:
+        # only run `commit`-hooks, excludes typos, clippy, etc
+        extra_args: --show-diff-on-failure --hook-stage commit
+
+  typos:
+    name: Spell Check with Typos
+    needs: [style]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || !github.event.pull_request.draft
+
+    steps:
+    - name: Checkout Actions Repository
+      uses: actions/checkout@v2
+
+    - name: Check the spelling of the files in our repo
+      uses: crate-ci/typos@master
+
+  clippy:
+    name: Run clippy
+    needs: [style]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || !github.event.pull_request.draft
+
+    steps:
+    - name: Checkout the repo
+      uses: actions/checkout@v2
+
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: nightly
+        components: clippy
+        profile: minimal
+        override: true
+
+    - name: Load cache
+      uses: Swatinem/rust-cache@v1
+
+    - name: Clippy
+      uses: actions-rs/cargo@v1
+      with:
+        command: run
+        args: -p xtask -- ci clippy

.github/workflows/wasm.yml

@@ -0,0 +1,75 @@
+name: WASM
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  check-wasm:
+    name: Build test / ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || !github.event.pull_request.draft
+
+    strategy:
+      fail-fast: true
+      matrix:
+        name:
+          - matrix-sdk-qrcode
+          - matrix-sdk-base
+          - matrix-sdk-common
+          - matrix-sdk-crypto
+          - indexeddb-no-crypto
+          - indexeddb-with-crypto
+
+        include:
+          - name: matrix-sdk (no-default, wasm-flags)
+            cmd: matrix-sdk-no-default
+          - name: matrix-sdk / indexeddb_stores
+            cmd: matrix-sdk-indexeddb-stores
+          - name: matrix-sdk / indexeddb_stores / no crypto
+            cmd: matrix-sdk-indexeddb-stores-no-crypto
+          - name: matrix-sdk / wasm-example
+            cmd: matrix-sdk-command-bot
+
+    steps:
+    - name: Checkout the repo
+      uses: actions/checkout@v2
+
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        target: wasm32-unknown-unknown
+        components: clippy
+        profile: minimal
+        override: true
+
+    - name: Install WasmPack
+      uses: jetli/wasm-pack-action@v0.3.0
+      with:
+        version: 'latest'
+
+    - name: Load cache
+      uses: Swatinem/rust-cache@v1
+
+    - name: Rust Check
+      uses: actions-rs/cargo@v1
+      with:
+        command: run
+        args: -p xtask -- ci wasm ${{ matrix.cmd || matrix.name }}
+
+    - name: Wasm-Pack test
+      uses: actions-rs/cargo@v1
+      with:
+        command: run
+        args: -p xtask -- ci wasm-pack ${{ matrix.cmd || matrix.name }}

.gitignore

@@ -1,4 +1,6 @@
 Cargo.lock
 target
 master.zip
-emsdk-*
+emsdk-*
+.idea/
+.vscode/

.pre-commit-config.yaml

@@ -1,10 +1,13 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.2.3
+    rev: v4.2.0
     hooks:
     -   id: check-yaml
+    -   id: check-toml
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
+    -   id: check-merge-conflict
+    -   id: mixed-line-ending
 
 -   repo: local
     hooks:
@@ -16,6 +19,7 @@ repos:
 
       - id: clippy
         name: clippy
+        stages: [push]
         language: system
         types: [file, rust]
         entry: cargo clippy --all-targets --all
@@ -23,7 +27,15 @@ repos:
 
       - id: test
         name: test
+        stages: [push]
         language: system
         files: '\.rs$'
         entry: cargo test --lib
         pass_filenames: false
+
+      - id: typos
+        name: typos
+        stages: [push]
+        language: system
+        entry: typos
+        pass_filenames: false

.rustfmt.toml

@@ -1,3 +1,4 @@
+edition = "2018"
 max_width = 100
 comment_width = 80
 wrap_comments = true

.typos.toml

@@ -0,0 +1,11 @@
+[default.extend-words]
+# Remove this once base64 gets correctly ignored by typos
+# Or if we're able to ignore certain lines.
+Fo = "Fo"
+BA = "BA"
+UE = "UE"
+
+[files]
+# Our json files contain a bunch of base64 encoded ed25519 keys which aren't
+# automatically ignored, we ignore them here.
+extend-exclude = ["*.json"]

Cargo.toml

@@ -1,11 +1,4 @@
 [workspace]
-members = [
-    "matrix_sdk",
-    "matrix_qrcode",
-    "matrix_sdk_base",
-    "matrix_sdk_test",
-    "matrix_sdk_test_macros",
-    "matrix_sdk_crypto",
-    "matrix_sdk_common",
-    "matrix_sdk_appservice"
-]
+members = ["benchmarks", "crates/*", "labs/*", "xtask"]
+# xtask and labs should only be compiled when invoked explicitly
+default-members = ["benchmarks", "crates/*"]

Makefile

@@ -1,17 +0,0 @@
-all: build
-
-build:
-	cargo build
-test:
-	cargo test
-
-coverage:
-	cargo tarpaulin -v
-
-clean:
-	cargo clean
-
-format:
-	cargo fmt
-
-.PHONY: clean test coverage

README.md

@@ -1,8 +1,8 @@
 ![Build Status](https://img.shields.io/github/workflow/status/matrix-org/matrix-rust-sdk/CI?style=flat-square)
-[![codecov](https://img.shields.io/codecov/c/github/matrix-org/matrix-rust-sdk/master.svg?style=flat-square)](https://codecov.io/gh/matrix-org/matrix-rust-sdk)
+[![codecov](https://img.shields.io/codecov/c/github/matrix-org/matrix-rust-sdk/main.svg?style=flat-square)](https://codecov.io/gh/matrix-org/matrix-rust-sdk)
 [![License](https://img.shields.io/badge/License-Apache%202.0-yellowgreen.svg?style=flat-square)](https://opensource.org/licenses/Apache-2.0)
 [![#matrix-rust-sdk](https://img.shields.io/badge/matrix-%23matrix--rust--sdk-blue?style=flat-square)](https://matrix.to/#/#matrix-rust-sdk:matrix.org)
-[![Docs - Master](https://img.shields.io/badge/docs-master-blue.svg?style=flat-square)](https://matrix-org.github.io/matrix-rust-sdk/)
+[![Docs - Main](https://img.shields.io/badge/docs-main-blue.svg?style=flat-square)](https://matrix-org.github.io/matrix-rust-sdk/)
 [![Docs - Stable](https://img.shields.io/crates/v/matrix-sdk?color=blue&label=docs&style=flat-square)](https://docs.rs/matrix-sdk)
 
 # matrix-rust-sdk
@@ -24,6 +24,10 @@ The rust-sdk consists of multiple crates that can be picked at your convenience:
 - **matrix-sdk-crypto** - No (network) IO encryption state machine that can be
   used to add Matrix E2EE support to your client or client library.
 
+## Minimum Supported Rust Version (MSRV)
+
+These crates are built with the Rust language version 2021 and require a minimum compiler version of `1.60`
+
 ## Status
 
 The library is in an alpha state, things that are implemented generally work but

RELEASE-NOTES-0.5.md

@@ -0,0 +1,43 @@
+# SDK 0.5 - stores and native crypto
+
+The @matrix-org/rust team is happy to present to the wider public version 0.5 of the [`matrix-rust-sdk`][sdk], now available on crates.io for your convenience. This marks an important milestone after over half a year of work since the latest release (0.4) and with many new team members working on it. It comes with many bug fixes, improvements and updates, but a few we'd like to highlight here:
+
+## Better, safer, native-er crypto
+
+One of the biggest improvements under the hood is the replacement of the former crypto core written in C, libolm, with the fully rustic, fully audited [Vodozemac][vodozemac]. [Vodozemac][vodozemac] is a much more robust new implementation of the olm crypto primitives and engines with all the learning included and pitfalls from the previous C implementation avoided in a freshly baked, very robust library. With this release both matrix-sdk-crypto and matrix-sdk (when the `e2e-encryption` is enabled, which it is by default) use vodozemac to handle all crypto needs. 
+
+## Stores, stores, stores
+
+This release has also seen a major refactoring around the state and crypto store primitives and implementations. From this release forward, the implementation of the storage layer is truly modular and pluggable, with the default implementation (based on `sled`) even being a separate crate. Furthermore this release has also additional support for the [`indexeddb`-storage][indexeddb] layer for in-browser WASM needs. As before, the base still ships with an in-memory store, but if none of them fits your needs, you are very much invited to build your own - we recommend looking at the implementation of the existing [`matrix-sdk-sled`][sled] to understand what is needed.
+
+We've further extended and cleaned up the storage API and hardened the two existing implementations: both `sled` and `indexeddb` will now encrypt _all metadata_, including all keys, if a passphrase is given. Even a core dump of a database won't show any strings or other identifiable data. Both implementation also hold a database version string now, allowing future migrations of the database schema while we move forward.
+
+## WebAssembly 
+
+It already came through in the previous paragraph: we now fully support `wasm` as a primary target for the main sdk and its dependencies (browser for now). While it was already possible to build the SDK for wasm before, if you were lucky and had the specific emscripten setup, even crypto didn't always fail to compile, our move to vodozemac frees us from these problems and our CI now makes sure all our PRs will continue to build on WebAssembly, too. And with the `indexeddb`-store, we also have a fully persistent storage layer implementation for all your in-browser matrix needs built in.
+
+## More features
+
+Of course a lot more has happened over the last few months as we've ramped up our work on the SDK. Most notably, you can now check whether a room has been created as a `space`, and we offer nice automatic thumbnailing and resizing support if you enable `image`. Additionally, there is a very early event-timeline API behind the `experimental-timeline`-feature-flag. Just to name a few. We really recommend you migrate from the older version to this one. 
+
+## Process and Workflow
+
+We've also ramped up our CI, parallelized it a lot, while adding more tests. In general we don't merge anything breaking the tests, thus our `main` can also be considered `stable` to build again, though things might be changed without prior notice. We recommend keeping an eye out in our [team chat][chat] to stay current.
+
+### Versions
+
+This and all future releases of matrix-sdk adhere to [semver][semver] and we do our best to stick to it for the default-feature-set. As you will see only the main crate `matrix-sdk` is actually versioned at `0.5`, while other crates - especially newer ones - have different versions attached. We recommend to sticking to the high-level matrix-sdk-crate wherever reasonable as lower level crates might change more often, also in breaking fashion. 
+
+With this release all our crates are build with `rust v2021` and need a rust compiler of at least `1.60` (due to our need for weak-dependencies).
+
+### Conventional Commit
+
+We also want to make it easier for external parties to follow changes and stay up to date with what's-what, so, with this release, we are implementing the [conventional commits][cc] standard for our commits and will start putting in automatic changelog generation. Which makes it easier for us to create comprehensive changelogs for releases, but also for anyone following `main` to figure out what changed.
+
+[sdk]: https://crates.io/crates/matrix-sdk/
+[vodozemac]: https://github.com/matrix-org/vodozemac
+[sled]: https://crates.io/crates/matrix-sdk-sled/
+[indexeddb]: https://crates.io/crates/matrix-sdk-indexeddb/
+[chat]: https://matrix.to/#/#matrix-rust-sdk:matrix.org
+[semver]: https://semver.org/
+[cc]: http://conventionalcommits.org/ 

benchmarks/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "benchmarks"
+description = "Matrix SDK benchmarks"
+edition = "2021"
+license = "Apache-2.0"
+rust-version = "1.56"
+version = "1.0.0"
+publish = false
+
+[dependencies]
+criterion = { version = "0.3.5", features = ["async", "async_tokio", "html_reports"] }
+matrix-sdk-crypto = { path = "../crates/matrix-sdk-crypto", version = "0.5.0" }
+matrix-sdk-sled = { path = "../crates/matrix-sdk-sled", version = "0.1.0", default-features = false, features = ["crypto-store"] }
+matrix-sdk-test = { path = "../crates/matrix-sdk-test", version = "0.5.0" }
+ruma = "0.6.1"
+serde_json = "1.0.79"
+tempfile = "3.3.0"
+tokio = { version = "1.17.0", default-features = false, features = ["rt-multi-thread"] }
+
+[target.'cfg(target_os = "linux")'.dependencies]
+pprof = { version = "0.8.0", features = ["flamegraph", "criterion"] }
+
+[[bench]]
+name = "crypto_bench"
+harness = false

benchmarks/README.md

@@ -57,7 +57,7 @@ $ cargo bench --bench crypto_bench -- --baseline libolm
 The benchmarks support profiling and generating [Flame Graphs] while they run in
 profiling mode using [pprof].
 
-Profiling usually requieres root permissions, to avoid the need for root
+Profiling usually requires root permissions, to avoid the need for root
 permissions you can adjust the value of `perf_event_paranoid`, e.g. the most
 permisive value is `-1`:
 

benchmarks/benches/crypto_bench.rs

@@ -1,57 +1,59 @@
-use std::sync::Arc;
+use std::ops::Deref;
 
 use criterion::*;
-use matrix_sdk_common::uuid::Uuid;
 use matrix_sdk_crypto::{EncryptionSettings, OlmMachine};
+use matrix_sdk_sled::CryptoStore as SledCryptoStore;
 use matrix_sdk_test::response_from_file;
 use ruma::{
     api::{
-        client::r0::{
+        client::{
             keys::{claim_keys, get_keys},
-            to_device::send_event_to_device::Response as ToDeviceResponse,
+            to_device::send_event_to_device::v3::Response as ToDeviceResponse,
         },
         IncomingResponse,
     },
-    room_id, user_id, DeviceIdBox, UserId,
+    device_id, room_id, user_id, DeviceId, OwnedUserId, TransactionId, UserId,
 };
 use serde_json::Value;
 use tokio::runtime::Builder;
 
-fn alice_id() -> UserId {
+fn alice_id() -> &'static UserId {
     user_id!("@alice:example.org")
 }
 
-fn alice_device_id() -> DeviceIdBox {
-    "JLAFKJWSCS".into()
+fn alice_device_id() -> &'static DeviceId {
+    device_id!("JLAFKJWSCS")
 }
 
-fn keys_query_response() -> get_keys::Response {
-    let data = include_bytes!("./keys_query.json");
+fn keys_query_response() -> get_keys::v3::Response {
+    let data = include_bytes!("crypto_bench/keys_query.json");
     let data: Value = serde_json::from_slice(data).unwrap();
     let data = response_from_file(&data);
-    get_keys::Response::try_from_http_response(data).expect("Can't parse the keys upload response")
-}
-
-fn keys_claim_response() -> claim_keys::Response {
-    let data = include_bytes!("./keys_claim.json");
-    let data: Value = serde_json::from_slice(data).unwrap();
-    let data = response_from_file(&data);
-    claim_keys::Response::try_from_http_response(data)
+    get_keys::v3::Response::try_from_http_response(data)
         .expect("Can't parse the keys upload response")
 }
 
-fn huge_keys_query_resopnse() -> get_keys::Response {
-    let data = include_bytes!("./keys_query_2000_members.json");
+fn keys_claim_response() -> claim_keys::v3::Response {
+    let data = include_bytes!("crypto_bench/keys_claim.json");
     let data: Value = serde_json::from_slice(data).unwrap();
     let data = response_from_file(&data);
-    get_keys::Response::try_from_http_response(data).expect("Can't parse the keys query response")
+    claim_keys::v3::Response::try_from_http_response(data)
+        .expect("Can't parse the keys upload response")
+}
+
+fn huge_keys_query_response() -> get_keys::v3::Response {
+    let data = include_bytes!("crypto_bench/keys_query_2000_members.json");
+    let data: Value = serde_json::from_slice(data).unwrap();
+    let data = response_from_file(&data);
+    get_keys::v3::Response::try_from_http_response(data)
+        .expect("Can't parse the keys query response")
 }
 
 pub fn keys_query(c: &mut Criterion) {
     let runtime = Builder::new_multi_thread().build().expect("Can't create runtime");
-    let machine = OlmMachine::new(&alice_id(), &alice_device_id());
+    let machine = runtime.block_on(OlmMachine::new(alice_id(), alice_device_id()));
     let response = keys_query_response();
-    let uuid = Uuid::new_v4();
+    let txn_id = TransactionId::new();
 
     let count = response.device_keys.values().fold(0, |acc, d| acc + d.len())
         + response.master_keys.len()
@@ -65,32 +67,27 @@ pub fn keys_query(c: &mut Criterion) {
 
     group.bench_with_input(BenchmarkId::new("memory store", &name), &response, |b, response| {
         b.to_async(&runtime)
-            .iter(|| async { machine.mark_request_as_sent(&uuid, response).await.unwrap() })
+            .iter(|| async { machine.mark_request_as_sent(&txn_id, response).await.unwrap() })
     });
 
     let dir = tempfile::tempdir().unwrap();
-    let machine = runtime
-        .block_on(OlmMachine::new_with_default_store(
-            &alice_id(),
-            &alice_device_id(),
-            dir.path(),
-            None,
-        ))
-        .unwrap();
+    let store = Box::new(SledCryptoStore::open_with_passphrase(dir.path(), None).unwrap());
+    let machine =
+        runtime.block_on(OlmMachine::with_store(alice_id(), alice_device_id(), store)).unwrap();
 
     group.bench_with_input(BenchmarkId::new("sled store", &name), &response, |b, response| {
         b.to_async(&runtime)
-            .iter(|| async { machine.mark_request_as_sent(&uuid, response).await.unwrap() })
+            .iter(|| async { machine.mark_request_as_sent(&txn_id, response).await.unwrap() })
     });
 
     group.finish()
 }
 
 pub fn keys_claiming(c: &mut Criterion) {
-    let runtime = Arc::new(Builder::new_multi_thread().build().expect("Can't create runtime"));
+    let runtime = Builder::new_multi_thread().build().expect("Can't create runtime");
 
     let keys_query_response = keys_query_response();
-    let uuid = Uuid::new_v4();
+    let txn_id = TransactionId::new();
 
     let response = keys_claim_response();
 
@@ -104,14 +101,14 @@ pub fn keys_claiming(c: &mut Criterion) {
     group.bench_with_input(BenchmarkId::new("memory store", &name), &response, |b, response| {
         b.iter_batched(
             || {
-                let machine = OlmMachine::new(&alice_id(), &alice_device_id());
+                let machine = runtime.block_on(OlmMachine::new(alice_id(), alice_device_id()));
                 runtime
-                    .block_on(machine.mark_request_as_sent(&uuid, &keys_query_response))
+                    .block_on(machine.mark_request_as_sent(&txn_id, &keys_query_response))
                     .unwrap();
-                (machine, runtime.clone())
+                (machine, &runtime, &txn_id)
             },
-            move |(machine, runtime)| {
-                runtime.block_on(machine.mark_request_as_sent(&uuid, response)).unwrap()
+            move |(machine, runtime, txn_id)| {
+                runtime.block_on(machine.mark_request_as_sent(txn_id, response)).unwrap()
             },
             BatchSize::SmallInput,
         )
@@ -121,21 +118,19 @@ pub fn keys_claiming(c: &mut Criterion) {
         b.iter_batched(
             || {
                 let dir = tempfile::tempdir().unwrap();
+                let store =
+                    Box::new(SledCryptoStore::open_with_passphrase(dir.path(), None).unwrap());
+
                 let machine = runtime
-                    .block_on(OlmMachine::new_with_default_store(
-                        &alice_id(),
-                        &alice_device_id(),
-                        dir.path(),
-                        None,
-                    ))
+                    .block_on(OlmMachine::with_store(alice_id(), alice_device_id(), store))
                     .unwrap();
                 runtime
-                    .block_on(machine.mark_request_as_sent(&uuid, &keys_query_response))
+                    .block_on(machine.mark_request_as_sent(&txn_id, &keys_query_response))
                     .unwrap();
-                (machine, runtime.clone())
+                (machine, &runtime, &txn_id)
             },
-            move |(machine, runtime)| {
-                runtime.block_on(machine.mark_request_as_sent(&uuid, response)).unwrap()
+            move |(machine, runtime, txn_id)| {
+                runtime.block_on(machine.mark_request_as_sent(txn_id, response)).unwrap()
             },
             BatchSize::SmallInput,
         )
@@ -148,18 +143,18 @@ pub fn room_key_sharing(c: &mut Criterion) {
     let runtime = Builder::new_multi_thread().build().expect("Can't create runtime");
 
     let keys_query_response = keys_query_response();
-    let uuid = Uuid::new_v4();
+    let txn_id = TransactionId::new();
     let response = keys_claim_response();
     let room_id = room_id!("!test:localhost");
 
     let to_device_response = ToDeviceResponse::new();
-    let users: Vec<UserId> = keys_query_response.device_keys.keys().cloned().collect();
+    let users: Vec<OwnedUserId> = keys_query_response.device_keys.keys().cloned().collect();
 
     let count = response.one_time_keys.values().fold(0, |acc, d| acc + d.len());
 
-    let machine = OlmMachine::new(&alice_id(), &alice_device_id());
-    runtime.block_on(machine.mark_request_as_sent(&uuid, &keys_query_response)).unwrap();
-    runtime.block_on(machine.mark_request_as_sent(&uuid, &response)).unwrap();
+    let machine = runtime.block_on(OlmMachine::new(alice_id(), alice_device_id()));
+    runtime.block_on(machine.mark_request_as_sent(&txn_id, &keys_query_response)).unwrap();
+    runtime.block_on(machine.mark_request_as_sent(&txn_id, &response)).unwrap();
 
     let mut group = c.benchmark_group("Room key sharing");
     group.throughput(Throughput::Elements(count as u64));
@@ -168,7 +163,11 @@ pub fn room_key_sharing(c: &mut Criterion) {
     group.bench_function(BenchmarkId::new("memory store", &name), |b| {
         b.to_async(&runtime).iter(|| async {
             let requests = machine
-                .share_group_session(&room_id, users.iter(), EncryptionSettings::default())
+                .share_group_session(
+                    room_id,
+                    users.iter().map(Deref::deref),
+                    EncryptionSettings::default(),
+                )
                 .await
                 .unwrap();
 
@@ -178,26 +177,25 @@ pub fn room_key_sharing(c: &mut Criterion) {
                 machine.mark_request_as_sent(&request.txn_id, &to_device_response).await.unwrap();
             }
 
-            machine.invalidate_group_session(&room_id).await.unwrap();
+            machine.invalidate_group_session(room_id).await.unwrap();
         })
     });
-
     let dir = tempfile::tempdir().unwrap();
-    let machine = runtime
-        .block_on(OlmMachine::new_with_default_store(
-            &alice_id(),
-            &alice_device_id(),
-            dir.path(),
-            None,
-        ))
-        .unwrap();
-    runtime.block_on(machine.mark_request_as_sent(&uuid, &keys_query_response)).unwrap();
-    runtime.block_on(machine.mark_request_as_sent(&uuid, &response)).unwrap();
+    let store = Box::new(SledCryptoStore::open_with_passphrase(dir.path(), None).unwrap());
+
+    let machine =
+        runtime.block_on(OlmMachine::with_store(alice_id(), alice_device_id(), store)).unwrap();
+    runtime.block_on(machine.mark_request_as_sent(&txn_id, &keys_query_response)).unwrap();
+    runtime.block_on(machine.mark_request_as_sent(&txn_id, &response)).unwrap();
 
     group.bench_function(BenchmarkId::new("sled store", &name), |b| {
         b.to_async(&runtime).iter(|| async {
             let requests = machine
-                .share_group_session(&room_id, users.iter(), EncryptionSettings::default())
+                .share_group_session(
+                    room_id,
+                    users.iter().map(Deref::deref),
+                    EncryptionSettings::default(),
+                )
                 .await
                 .unwrap();
 
@@ -207,7 +205,7 @@ pub fn room_key_sharing(c: &mut Criterion) {
                 machine.mark_request_as_sent(&request.txn_id, &to_device_response).await.unwrap();
             }
 
-            machine.invalidate_group_session(&room_id).await.unwrap();
+            machine.invalidate_group_session(room_id).await.unwrap();
         })
     });
 
@@ -217,10 +215,10 @@ pub fn room_key_sharing(c: &mut Criterion) {
 pub fn devices_missing_sessions_collecting(c: &mut Criterion) {
     let runtime = Builder::new_multi_thread().build().expect("Can't create runtime");
 
-    let machine = OlmMachine::new(&alice_id(), &alice_device_id());
-    let response = huge_keys_query_resopnse();
-    let uuid = Uuid::new_v4();
-    let users: Vec<UserId> = response.device_keys.keys().cloned().collect();
+    let machine = runtime.block_on(OlmMachine::new(alice_id(), alice_device_id()));
+    let response = huge_keys_query_response();
+    let txn_id = TransactionId::new();
+    let users: Vec<OwnedUserId> = response.device_keys.keys().cloned().collect();
 
     let count = response.device_keys.values().fold(0, |acc, d| acc + d.len());
 
@@ -229,29 +227,26 @@ pub fn devices_missing_sessions_collecting(c: &mut Criterion) {
 
     let name = format!("{} devices", count);
 
-    runtime.block_on(machine.mark_request_as_sent(&uuid, &response)).unwrap();
+    runtime.block_on(machine.mark_request_as_sent(&txn_id, &response)).unwrap();
 
     group.bench_function(BenchmarkId::new("memory store", &name), |b| {
         b.to_async(&runtime).iter_with_large_drop(|| async {
-            machine.get_missing_sessions(users.iter()).await.unwrap()
+            machine.get_missing_sessions(users.iter().map(Deref::deref)).await.unwrap()
         })
     });
 
     let dir = tempfile::tempdir().unwrap();
-    let machine = runtime
-        .block_on(OlmMachine::new_with_default_store(
-            &alice_id(),
-            &alice_device_id(),
-            dir.path(),
-            None,
-        ))
-        .unwrap();
+    let store = Box::new(SledCryptoStore::open_with_passphrase(dir.path(), None).unwrap());
 
-    runtime.block_on(machine.mark_request_as_sent(&uuid, &response)).unwrap();
+    let machine =
+        runtime.block_on(OlmMachine::with_store(alice_id(), alice_device_id(), store)).unwrap();
+
+    runtime.block_on(machine.mark_request_as_sent(&txn_id, &response)).unwrap();
 
     group.bench_function(BenchmarkId::new("sled store", &name), |b| {
-        b.to_async(&runtime)
-            .iter(|| async { machine.get_missing_sessions(users.iter()).await.unwrap() })
+        b.to_async(&runtime).iter(|| async {
+            machine.get_missing_sessions(users.iter().map(Deref::deref)).await.unwrap()
+        })
     });
 
     group.finish()

codecov.yaml

@@ -0,0 +1,16 @@
+coverage:
+  status:
+    project:
+      default:
+        # Commits pushed to master should not make the overall
+        # project coverage decrease by more than 1%:
+        target: auto
+        threshold: 1%
+    patch:
+      default:
+        # Be tolerant on slight code coverage diff on PRs to limit
+        # noisy red coverage status on github PRs.
+        # Note: The coverage stats are still uploaded
+        # to codecov so that PR reviewers can see uncovered lines
+        target: auto
+        threshold: 1%

contrib/key-sharing-algorithm/Makefile

@@ -0,0 +1,15 @@
+.PHONY: default clean png pdf
+
+default: model.png model.pdf
+
+clean:
+	rm -f model.png model.pdf
+
+pdf: model.pdf
+png: model.png
+
+model.pdf: model.dot
+	./setdotlabelwidth 45 <model.dot | dot -Tpdf -o model.pdf
+
+model.png: model.dot
+	./setdotlabelwidth 45 <model.dot | dot -Tpng -o model.png

contrib/key-sharing-algorithm/README.md

@@ -0,0 +1,11 @@
+This models the room key sharing algorithm as a decision tree and provides
+tooling to render it as a PDF or PNG.
+
+# Usage
+
+    make  # Render the decision tree both as a PDF and PNG
+
+    make pdf  # Renders the decision tree as a PDF
+    make png  # Renders the decision tree as a PNG
+
+    make clean  # Remove rendered artifacts

contrib/key-sharing-algorithm/model.dot

@@ -0,0 +1,45 @@
+digraph {
+    label="Matrix room key sharing algorithm"
+    fontname="Fira Sans"
+    ratio=0.5
+
+    node [shape=box, colorscheme=paired6, style=filled, fillcolor=white, fontname="Fira Sans"]
+    edge [fontname="Fira Sans"]
+
+    /* Non-end states, additional checks needed. */
+    verified_device_check [label="START\n\nIs this our own, verified device?", labelfontname="Fira Sans"]
+    outbound_session_check [label="Outbound session exists?"]
+    outbound_exists [label="Session previously previously_shared with this user ID/device ID pair?"]
+    previously_shared [label="Requesting device sender (Curve25519) key matches the key we originally shared with?"]
+    own_device_check [label="Is this our own device?"]
+
+    /* End states */
+    allow_verified [label="Share the entire session from the earliest known index.\n\nOk(None)", color=4, fillcolor=3]
+    allow_limited [label="Share a limited session starting from index i, which is the index we previously shared at.\n\nOk(Some(i))", color=4, fillcolor=3]
+    refuse_device_key_changed [label="Sender key changed, refuse to share.\n\nErr(KeyForwardDecision::ChangedSenderKey)", color=6, fillcolor=5]
+    refuse_not_shared [label="Session was never shared with this device, refuse to share.\n\nErr(KeyForwardDecision::OutboundSessionNotShared)", color=6, fillcolor=5]
+    refuse_untrusted_own_device [label="Our own device, but it is untrusted and we haven't previously shared with it. Refuse to share.\n\nErr(KeyForwardDecision::UntrustedDevice)", color=6, fillcolor=5]
+    refuse_missing_outbound_session [label="Not our device and haven't previously shared with it. Refuse to share.\n\nErr(KeyForwardDecision::MissingOutboundSession)", color=6, fillcolor=5]
+
+    /* Checks */
+
+    /* Is this our own verified device? */
+    verified_device_check -> allow_verified [label="Yes"]
+    verified_device_check -> outbound_session_check [label="No"]
+
+    /* Does the outbound session exist? */
+    outbound_session_check -> outbound_exists [label="Yes"]
+    outbound_session_check -> own_device_check [label="No"]
+
+    /* Previously shared? */
+    outbound_exists -> previously_shared [label="Yes"]
+    outbound_exists -> refuse_not_shared [label="No"]
+
+    /* Requesting device sender key matches key it was shared with? */
+    previously_shared -> allow_limited [label="Yes"]
+    previously_shared -> refuse_device_key_changed [label="No"]
+
+    /* Is this our own device? */
+    own_device_check -> refuse_untrusted_own_device [label="Yes"]
+    own_device_check -> refuse_missing_outbound_session [label="No"]
+}

contrib/key-sharing-algorithm/setdotlabelwidth

@@ -0,0 +1,37 @@
+#!/usr/bin/perl
+#
+# Adapted from https://stackoverflow.com/a/68057031
+
+use strict;
+
+my $usage = "setdotlabelwidth [char-width] < [dotfile]";
+my $width = shift() or die("Usage: $usage $!");
+
+while(<STDIN>)
+{
+  if (m/label="(.*?)"/)
+  {
+    my $labeltext = $1;
+    my @words = split(/ +|(?=\\n)/, $labeltext);
+    my @newtext = ();
+    my $newline = "";
+    foreach my $word(@words)
+    {
+      if (length($newline) > 0 and
+          length($newline) + length($word) > $width)
+      {
+        push(@newtext, $newline);
+        $newline = "";
+      }
+
+      $newline .= " " if (length($newline) > 0);
+      $newline .= $word;
+    }
+
+    push(@newtext, $newline) if (length($newline) > 0);
+    my $newlabel = join("\\n", @newtext);
+    s/label=".*?"/label="$newlabel"/;
+  }
+
+  print;
+}

crates/matrix-sdk-appservice/Cargo.toml

@@ -0,0 +1,50 @@
+[package]
+authors = ["Johannes Becker <j.becker@famedly.com>"]
+edition = "2018"
+homepage = "https://github.com/matrix-org/matrix-rust-sdk"
+repository = "https://github.com/matrix-org/matrix-rust-sdk"
+description = "Appservice SDK based on the matrix-sdk"
+keywords = ["matrix", "chat", "messaging", "ruma", "nio", "appservice"]
+license = "Apache-2.0"
+name = "matrix-sdk-appservice"
+version = "0.1.0"
+rust-version = "1.60"
+
+[features]
+default = ["native-tls"]
+
+anyhow = ["matrix-sdk/anyhow"]
+e2e-encryption = [
+    "matrix-sdk/e2e-encryption"
+]
+eyre = ["matrix-sdk/eyre"]
+sled = ["matrix-sdk/sled"]
+
+markdown = ["matrix-sdk/markdown"]
+native-tls = ["matrix-sdk/native-tls"]
+rustls-tls = ["matrix-sdk/rustls-tls"]
+socks = ["matrix-sdk/socks"]
+sso-login = ["matrix-sdk/sso-login"]
+
+docs = []
+
+[dependencies]
+dashmap = "5.2.0"
+http = "0.2.6"
+matrix-sdk = { version = "0.5.0", path = "../matrix-sdk", default-features = false, features = ["appservice"] }
+percent-encoding = "2.1.0"
+regex = "1.5.5"
+ruma = { version = "0.6.1", features = ["client-api-c", "appservice-api-s"] }
+serde = "1.0.136"
+serde_json = "1.0.79"
+serde_yaml = "0.8.23"
+thiserror = "1.0.30"
+tracing = "0.1.34"
+url = "2.2.2"
+warp = { version = "0.3.2", default-features = false }
+
+[dev-dependencies]
+matrix-sdk-test = { version = "0.5.0", path = "../matrix-sdk-test", features = ["appservice"] }
+mockito = "0.31.0"
+tokio = { version = "1.17.0", default-features = false, features = ["rt-multi-thread", "macros"] }
+tracing-subscriber = "0.3.11"

crates/matrix-sdk-appservice/examples/appservice_autojoin.rs

@@ -1,13 +1,11 @@
-use std::{convert::TryFrom, env};
+use std::env;
 
 use matrix_sdk_appservice::{
     matrix_sdk::{
+        event_handler::Ctx,
         room::Room,
         ruma::{
-            events::{
-                room::member::{MemberEventContent, MembershipState},
-                SyncStateEvent,
-            },
+            events::room::member::{MembershipState, OriginalSyncRoomMemberEvent},
             UserId,
         },
     },
@@ -18,12 +16,12 @@ use tracing::trace;
 pub async fn handle_room_member(
     appservice: AppService,
     room: Room,
-    event: SyncStateEvent<MemberEventContent>,
+    event: OriginalSyncRoomMemberEvent,
 ) -> Result<()> {
     if !appservice.user_id_is_in_namespace(&event.state_key)? {
         trace!("not an appservice user: {}", event.state_key);
     } else if let MembershipState::Invite = event.content.membership {
-        let user_id = UserId::try_from(event.state_key.as_str())?;
+        let user_id = UserId::parse(event.state_key.as_str())?;
         appservice.register_virtual_user(user_id.localpart()).await?;
 
         let client = appservice.virtual_user_client(user_id.localpart()).await?;
@@ -42,14 +40,16 @@ pub async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let server_name = "localhost";
     let registration = AppServiceRegistration::try_from_yaml_file("./tests/registration.yaml")?;
 
-    let mut appservice = AppService::new(homeserver_url, server_name, registration).await?;
+    let appservice = AppService::new(homeserver_url, server_name, registration).await?;
     appservice
-        .register_event_handler({
-            let appservice = appservice.clone();
-            move |event: SyncStateEvent<MemberEventContent>, room: Room| {
-                handle_room_member(appservice.clone(), room, event)
-            }
-        })
+        .register_event_handler_context(appservice.clone())?
+        .register_event_handler(
+            move |event: OriginalSyncRoomMemberEvent,
+                  room: Room,
+                  Ctx(appservice): Ctx<AppService>| {
+                handle_room_member(appservice, room, event)
+            },
+        )
         .await?;
 
     let (host, port) = appservice.registration().get_host_and_port()?;

crates/matrix-sdk-appservice/src/error.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use ruma::api::client::uiaa::UiaaInfo;
 use thiserror::Error;
 
 #[derive(Error, Debug)]
@@ -44,7 +45,7 @@ pub enum Error {
     HttpRequest(#[from] ruma::api::error::FromHttpRequestError),
 
     #[error(transparent)]
-    Identifier(#[from] ruma::identifiers::Error),
+    Identifier(#[from] ruma::IdParseError),
 
     #[error(transparent)]
     Http(#[from] http::Error),
@@ -73,15 +74,35 @@ pub enum Error {
     #[error(transparent)]
     SerdeJson(#[from] serde_json::Error),
 
-    #[cfg(feature = "warp")]
+    #[error(transparent)]
+    Utf8Error(#[from] std::str::Utf8Error),
+
     #[error("warp rejection: {0}")]
     WarpRejection(String),
 }
 
-#[cfg(feature = "warp")]
+impl Error {
+    /// Try to destructure the error into an universal interactive auth info.
+    ///
+    /// Some requests require universal interactive auth, doing such a request
+    /// will always fail the first time with a 401 status code, the response
+    /// body will contain info how the client can authenticate.
+    ///
+    /// The request will need to be retried, this time containing additional
+    /// authentication data.
+    ///
+    /// This method is an convenience method to get to the info the server
+    /// returned on the first, failed request.
+    pub fn uiaa_response(&self) -> Option<&UiaaInfo> {
+        match self {
+            Error::Matrix(matrix) => matrix.uiaa_response(),
+            _ => None,
+        }
+    }
+}
+
 impl warp::reject::Reject for Error {}
 
-#[cfg(feature = "warp")]
 impl From<warp::Rejection> for Error {
     fn from(rejection: warp::Rejection) -> Self {
         Self::WarpRejection(format!("{:?}", rejection))
@@ -93,3 +114,9 @@ impl From<matrix_sdk::HttpError> for Error {
         matrix_sdk::Error::from(e).into()
     }
 }
+
+impl From<matrix_sdk::StoreError> for Error {
+    fn from(e: matrix_sdk::StoreError) -> Self {
+        matrix_sdk::Error::from(e).into()
+    }
+}

crates/matrix-sdk-appservice/src/event_handler.rs

@@ -0,0 +1,35 @@
+use std::{future::Future, pin::Pin, sync::Arc};
+
+use matrix_sdk::locks::Mutex;
+
+use crate::{
+    ruma::api::appservice::query::{
+        query_room_alias::v1 as query_room, query_user_id::v1 as query_user,
+    },
+    AppService,
+};
+
+pub(crate) type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
+pub(crate) type AppserviceFn<A, R> =
+    Box<dyn FnMut(AppService, A) -> BoxFuture<'static, R> + Send + Sync + 'static>;
+
+#[derive(Default, Clone)]
+pub struct EventHandler {
+    pub users: Arc<Mutex<Option<AppserviceFn<query_user::IncomingRequest, bool>>>>,
+    pub rooms: Arc<Mutex<Option<AppserviceFn<query_room::IncomingRequest, bool>>>>,
+}
+
+impl std::fmt::Debug for EventHandler {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut debug = f.debug_struct("EventHandler");
+        match self.users.try_lock() {
+            Ok(lock) => debug.field("users", &lock.is_some()),
+            Err(_) => debug.field("users", &format_args!("<locked>")),
+        };
+        match self.rooms.try_lock() {
+            Ok(lock) => debug.field("rooms", &lock.is_some()),
+            Err(_) => debug.field("rooms", &format_args!("<locked>")),
+        };
+        debug.finish()
+    }
+}

crates/matrix-sdk-appservice/src/lib.rs

@@ -35,7 +35,7 @@
 //! # async {
 //! #
 //! use matrix_sdk_appservice::{
-//!     ruma::events::{SyncStateEvent, room::member::MemberEventContent},
+//!     ruma::events::room::member::SyncRoomMemberEvent,
 //!     AppService, AppServiceRegistration
 //! };
 //!
@@ -55,7 +55,7 @@
 //!     ")?;
 //!
 //! let mut appservice = AppService::new(homeserver_url, server_name, registration).await?;
-//! appservice.register_event_handler(|_ev: SyncStateEvent<MemberEventContent>| async {
+//! appservice.register_event_handler(|_ev: SyncRoomMemberEvent| async {
 //!     // do stuff
 //! });
 //!
@@ -70,10 +70,7 @@
 //!
 //! [Application Service]: https://matrix.org/docs/spec/application_service/r0.1.2
 //! [matrix-org/matrix-rust-sdk#228]: https://github.com/matrix-org/matrix-rust-sdk/issues/228
-//! [examples directory]: https://github.com/matrix-org/matrix-rust-sdk/tree/master/matrix_sdk_appservice/examples
-
-#[cfg(not(any(feature = "warp")))]
-compile_error!("one webserver feature must be enabled. available ones: `warp`");
+//! [examples directory]: https://github.com/matrix-org/matrix-rust-sdk/tree/main/crates/matrix-sdk-appservice/examples
 
 use std::{
     convert::{TryFrom, TryInto},
@@ -86,6 +83,7 @@ use std::{
 
 use dashmap::DashMap;
 pub use error::Error;
+use event_handler::AppserviceFn;
 use http::Uri;
 pub use matrix_sdk;
 #[doc(no_inline)]
@@ -94,30 +92,32 @@ use matrix_sdk::{
     bytes::Bytes,
     event_handler::{EventHandler, EventHandlerResult, SyncEvent},
     reqwest::Url,
-    Client, ClientConfig, Session,
+    Client, ClientBuildError, ClientBuilder, Session,
 };
 use regex::Regex;
 use ruma::{
     api::{
-        appservice::Registration,
-        client::{
-            error::ErrorKind,
-            r0::{account::register, uiaa::UiaaResponse},
+        appservice::{
+            query::{query_room_alias::v1 as query_room, query_user_id::v1 as query_user},
+            Registration,
         },
-        error::{FromHttpResponseError, ServerError},
+        client::account::register,
     },
-    assign, identifiers, DeviceId, ServerNameBox, UserId,
+    assign, DeviceId, IdParseError, OwnedServerName, UserId,
 };
 use serde::de::DeserializeOwned;
-use tracing::{info, warn};
+use tracing::info;
 
 mod error;
+pub mod event_handler;
 mod webserver;
 
 pub type Result<T, E = Error> = std::result::Result<T, E>;
 pub type Host = String;
 pub type Port = u16;
 
+const USER_KEY: &[u8] = b"appservice.users.";
+
 /// AppService Registration
 ///
 /// Wrapper around [`Registration`]
@@ -199,17 +199,18 @@ pub type VirtualUser = ();
 #[derive(Debug, Clone)]
 pub struct AppService {
     homeserver_url: Url,
-    server_name: ServerNameBox,
+    server_name: OwnedServerName,
     registration: Arc<AppServiceRegistration>,
     clients: Arc<DashMap<Localpart, Client>>,
+    event_handler: event_handler::EventHandler,
 }
 
 impl AppService {
     /// Create new AppService
     ///
     /// Also creates and caches a [`Client`] for the [`MainUser`].
-    /// The default [`ClientConfig`] is used, if you want to customize it
-    /// use [`Self::new_with_config()`] instead.
+    /// A default [`ClientBuilder`] is used, if you want to customize it
+    /// use [`with_client_builder()`][Self::with_client_builder] instead.
     ///
     /// # Arguments
     ///
@@ -222,38 +223,36 @@ impl AppService {
     /// [AppService Registration]: https://matrix.org/docs/spec/application_service/r0.1.2#registration
     pub async fn new(
         homeserver_url: impl TryInto<Url, Error = url::ParseError>,
-        server_name: impl TryInto<ServerNameBox, Error = identifiers::Error>,
+        server_name: impl TryInto<OwnedServerName, Error = IdParseError>,
         registration: AppServiceRegistration,
     ) -> Result<Self> {
-        let appservice = Self::new_with_config(
-            homeserver_url,
-            server_name,
-            registration,
-            ClientConfig::default(),
-        )
-        .await?;
+        let appservice =
+            Self::with_client_builder(homeserver_url, server_name, registration, Client::builder())
+                .await?;
 
         Ok(appservice)
     }
 
-    /// Same as [`Self::new()`] but lets you provide a [`ClientConfig`] for the
-    /// [`Client`]
-    pub async fn new_with_config(
+    /// Same as [`new()`][Self::new] but lets you provide a [`ClientBuilder`]
+    /// for the [`Client`]
+    pub async fn with_client_builder(
         homeserver_url: impl TryInto<Url, Error = url::ParseError>,
-        server_name: impl TryInto<ServerNameBox, Error = identifiers::Error>,
+        server_name: impl TryInto<OwnedServerName, Error = IdParseError>,
         registration: AppServiceRegistration,
-        client_config: ClientConfig,
+        builder: ClientBuilder,
     ) -> Result<Self> {
         let homeserver_url = homeserver_url.try_into()?;
         let server_name = server_name.try_into()?;
         let registration = Arc::new(registration);
         let clients = Arc::new(DashMap::new());
         let sender_localpart = registration.sender_localpart.clone();
+        let event_handler = event_handler::EventHandler::default();
 
-        let appservice = AppService { homeserver_url, server_name, registration, clients };
+        let appservice =
+            AppService { homeserver_url, server_name, registration, clients, event_handler };
 
         // we create and cache the [`MainUser`] by default
-        appservice.create_and_cache_client(&sender_localpart, client_config).await?;
+        appservice.create_and_cache_client(&sender_localpart, builder).await?;
 
         Ok(appservice)
     }
@@ -266,8 +265,9 @@ impl AppService {
     ///
     /// This method is a singleton that saves the client internally for re-use
     /// based on the `localpart`. The cached [`Client`] can be retrieved either
-    /// by calling this method again or by calling [`Self::get_cached_client()`]
-    /// which is non-async convenience wrapper.
+    /// by calling this method again or by calling
+    /// [`get_cached_client()`][Self::get_cached_client] which is non-async
+    /// convenience wrapper.
     ///
     /// Note that if you want to do actions like joining rooms with a virtual
     /// user it needs to be registered first. `Self::register_virtual_user()`
@@ -281,20 +281,20 @@ impl AppService {
     /// [assert the identity]: https://matrix.org/docs/spec/application_service/r0.1.2#identity-assertion
     pub async fn virtual_user_client(&self, localpart: impl AsRef<str>) -> Result<Client> {
         let client =
-            self.virtual_user_client_with_config(localpart, ClientConfig::default()).await?;
+            self.virtual_user_client_with_client_builder(localpart, Client::builder()).await?;
 
         Ok(client)
     }
 
-    /// Same as [`Self::virtual_user_client()`] but with the ability to pass in
-    /// a [`ClientConfig`]
+    /// Same as [`virtual_user_client()`][Self::virtual_user_client] but with
+    /// the ability to pass in a [`ClientBuilder`]
     ///
     /// Since this method is a singleton follow-up calls with different
-    /// [`ClientConfig`]s will be ignored.
-    pub async fn virtual_user_client_with_config(
+    /// [`ClientBuilder`]s will be ignored.
+    pub async fn virtual_user_client_with_client_builder(
         &self,
         localpart: impl AsRef<str>,
-        config: ClientConfig,
+        builder: ClientBuilder,
     ) -> Result<Client> {
         // TODO: check if localpart is covered by namespace?
         let localpart = localpart.as_ref();
@@ -302,7 +302,7 @@ impl AppService {
         let client = if let Some(client) = self.clients.get(localpart) {
             client.clone()
         } else {
-            self.create_and_cache_client(localpart, config).await?
+            self.create_and_cache_client(localpart, builder).await?
         };
 
         Ok(client)
@@ -311,22 +311,23 @@ impl AppService {
     async fn create_and_cache_client(
         &self,
         localpart: &str,
-        config: ClientConfig,
+        mut builder: ClientBuilder,
     ) -> Result<Client> {
         let user_id = UserId::parse_with_server_name(localpart, &self.server_name)?;
 
         // The `as_token` in the `Session` maps to the [`MainUser`]
         // (`sender_localpart`) by default, so we don't need to assert identity
         // in that case
-        let config = if localpart != self.registration.sender_localpart {
-            let request_config = config.get_request_config().assert_identity();
-            config.request_config(request_config)
-        } else {
-            config
-        };
+        if localpart != self.registration.sender_localpart {
+            builder = builder.assert_identity();
+        }
 
-        let client =
-            Client::new_with_config(self.homeserver_url.clone(), config.appservice_mode())?;
+        let client = builder
+            .homeserver_url(self.homeserver_url.clone())
+            .appservice_mode()
+            .build()
+            .await
+            .map_err(ClientBuildError::assert_valid_builder_args)?;
 
         let session = Session {
             access_token: self.registration.as_token.clone(),
@@ -344,8 +345,9 @@ impl AppService {
     /// Get cached [`Client`]
     ///
     /// Will return the client for the given `localpart` if previously
-    /// constructed with [`Self::virtual_user_client()`] or
-    /// [`Self::virtual_user_client_with_config()`].
+    /// constructed with [`virtual_user_client()`][Self::virtual_user_client] or
+    /// [`virtual_user_client_with_config()`][Self::
+    /// virtual_user_client_with_client_builder].
     ///
     /// If no `localpart` is given it assumes the [`MainUser`]'s `localpart`. If
     /// no client for `localpart` is found it will return an Error.
@@ -373,7 +375,7 @@ impl AppService {
     ///
     /// [`join` room `timeline` events]: https://spec.matrix.org/unstable/client-server-api/#get_matrixclientr0sync
     /// [MSC2409]: https://github.com/matrix-org/matrix-doc/pull/2409
-    pub async fn register_event_handler<Ev, Ctx, H>(&mut self, handler: H) -> Result<()>
+    pub async fn register_event_handler<Ev, Ctx, H>(&self, handler: H) -> Result<&Self>
     where
         Ev: SyncEvent + DeserializeOwned + Send + 'static,
         H: EventHandler<Ev, Ctx>,
@@ -382,44 +384,115 @@ impl AppService {
         let client = self.get_cached_client(None)?;
         client.register_event_handler(handler).await;
 
-        Ok(())
+        Ok(self)
     }
 
-    /// Register a virtual user by sending a [`register::Request`] to the
+    /// Convenience wrapper around [`Client::register_event_handler_context`]
+    /// attaches the event handler context to the [`MainUser`]'s [`Client`].
+    pub fn register_event_handler_context<T>(&self, ctx: T) -> Result<&Self>
+    where
+        T: Clone + Send + Sync + 'static,
+    {
+        let client = self.get_cached_client(None)?;
+        client.register_event_handler_context(ctx);
+
+        Ok(self)
+    }
+
+    /// Register a responder for queries about the existence of a user with a
+    /// given mxid.
+    ///
+    /// See [GET /_matrix/app/v1/users/{userId}](https://matrix.org/docs/spec/application_service/r0.1.2#get-matrix-app-v1-users-userid).
+    ///
+    /// # Example
+    /// ```no_run
+    /// # use matrix_sdk_appservice::AppService;
+    /// # fn run(appservice: AppService) {
+    /// appservice.register_user_query(Box::new(|appservice, req| Box::pin(async move {
+    ///     println!("Got request for {}", req.user_id);
+    ///     true
+    /// })));
+    /// # }
+    /// ```
+    pub async fn register_user_query(
+        &self,
+        handler: AppserviceFn<query_user::IncomingRequest, bool>,
+    ) {
+        *self.event_handler.users.lock().await = Some(handler);
+    }
+
+    /// Register a responder for queries about the existence of a room with the
+    /// given alias.
+    ///
+    /// See [GET /_matrix/app/v1/rooms/{roomAlias}](https://matrix.org/docs/spec/application_service/r0.1.2#get-matrix-app-v1-rooms-roomalias).
+    ///
+    /// # Example
+    /// ```no_run
+    /// # use matrix_sdk_appservice::AppService;
+    /// # fn run(appservice: AppService) {
+    /// appservice.register_room_query(Box::new(|appservice, req| Box::pin(async move {
+    ///     println!("Got request for {}", req.room_alias);
+    ///     true
+    /// })));
+    /// # }
+    /// ```
+    pub async fn register_room_query(
+        &self,
+        handler: AppserviceFn<query_room::IncomingRequest, bool>,
+    ) {
+        *self.event_handler.rooms.lock().await = Some(handler);
+    }
+
+    /// Register a virtual user by sending a [`register::v3::Request`] to the
     /// homeserver
     ///
     /// # Arguments
     ///
     /// * `localpart` - The localpart of the user to register. Must be covered
     ///   by the namespaces in the [`Registration`] in order to succeed.
+    ///
+    /// # Returns
+    /// This function may return a UIAA response, which should be checked for
+    /// with [`Error::uiaa_response()`].
     pub async fn register_virtual_user(&self, localpart: impl AsRef<str>) -> Result<()> {
-        let request = assign!(register::Request::new(), {
+        if self.is_user_registered(localpart.as_ref()).await? {
+            return Ok(());
+        }
+        let request = assign!(register::v3::Request::new(), {
             username: Some(localpart.as_ref()),
             login_type: Some(&register::LoginType::ApplicationService),
         });
 
         let client = self.get_cached_client(None)?;
-        match client.register(request).await {
-            Ok(_) => (),
-            Err(error) => match error {
-                matrix_sdk::HttpError::UiaaError(FromHttpResponseError::Http(
-                    ServerError::Known(UiaaResponse::MatrixError(ref matrix_error)),
-                )) => {
-                    match matrix_error.kind {
-                        ErrorKind::UserInUse => {
-                            // TODO: persist the fact that we registered that user
-                            warn!("{}", matrix_error.message);
-                        }
-                        _ => return Err(error.into()),
-                    }
-                }
-                _ => return Err(error.into()),
-            },
-        }
+        client.register(request).await?;
+        self.set_user_registered(localpart.as_ref()).await?;
 
         Ok(())
     }
 
+    /// Add the given localpart to the database of registered localparts.
+    async fn set_user_registered(&self, localpart: impl AsRef<str>) -> Result<()> {
+        let client = self.get_cached_client(None)?;
+        client
+            .store()
+            .set_custom_value(
+                &[USER_KEY, localpart.as_ref().as_bytes()].concat(),
+                vec![u8::from(true)],
+            )
+            .await?;
+        Ok(())
+    }
+
+    /// Get whether a localpart is listed in the database as registered.
+    async fn is_user_registered(&self, localpart: impl AsRef<str>) -> Result<bool> {
+        let client = self.get_cached_client(None)?;
+        let key = [USER_KEY, localpart.as_ref().as_bytes()].concat();
+        let store = client.store().get_custom_value(&key).await?;
+        let registered =
+            store.and_then(|vec| vec.get(0).copied()).map_or(false, |b| b == u8::from(true));
+        Ok(registered)
+    }
+
     /// Get the AppService [registration]
     ///
     /// [registration]: https://matrix.org/docs/spec/application_service/r0.1.2#registration
@@ -455,10 +528,8 @@ impl AppService {
     /// functionality.
     ///
     /// [application-service-specific routes]: https://spec.matrix.org/unstable/application-service-api/#legacy-routes
-    #[cfg(feature = "warp")]
-    #[cfg_attr(docs, doc(cfg(feature = "warp")))]
     pub fn warp_filter(&self) -> warp::filters::BoxedFilter<(impl warp::Reply,)> {
-        webserver::warp::warp_filter(self.clone())
+        webserver::warp_filter(self.clone())
     }
 
     /// Convenience method that runs an http server depending on the selected
@@ -471,14 +542,8 @@ impl AppService {
         let port = port.into();
         info!("Starting AppService on {}:{}", &host, &port);
 
-        #[cfg(feature = "warp")]
-        {
-            webserver::warp::run_server(self.clone(), host, port).await?;
-            Ok(())
-        }
-
-        #[cfg(not(any(feature = "warp",)))]
-        unreachable!()
+        webserver::run_server(self.clone(), host, port).await?;
+        Ok(())
     }
 }
 
@@ -493,7 +558,7 @@ pub(crate) fn transform_request_path(
 ) -> Result<http::Request<Bytes>> {
     let uri = request.uri();
     // remove trailing slash from path
-    let path = uri.path().trim_end_matches('/').to_string();
+    let path = uri.path().trim_end_matches('/').to_owned();
 
     if !path.starts_with("/_matrix/app/v1/") {
         let path = match path {
@@ -528,7 +593,8 @@ pub(crate) fn transform_request_path(
                 };
 
                 if let Some(query) = uri.query() {
-                    path.push_str(&format!("?{}", query));
+                    path.push('?');
+                    path.push_str(query);
                 }
 
                 path

crates/matrix-sdk-appservice/src/webserver.rs

@@ -12,10 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::{net::ToSocketAddrs, result::Result as StdResult};
+use std::net::ToSocketAddrs;
 
-use futures::TryFutureExt;
-use matrix_sdk::{bytes::Bytes, ruma};
+use matrix_sdk::{
+    bytes::Bytes,
+    ruma::{
+        self,
+        api::{
+            appservice::query::{
+                query_room_alias::v1 as query_room, query_user_id::v1 as query_user,
+            },
+            IncomingRequest,
+        },
+    },
+};
 use serde::Serialize;
 use warp::{filters::BoxedFilter, path::FullPath, Filter, Rejection, Reply};
 
@@ -95,7 +105,7 @@ mod filters {
 
     fn common(appservice: AppService) -> BoxedFilter<(AppService, http::Request<Bytes>)> {
         warp::any()
-            .and(filters::valid_access_token(appservice.registration().hs_token.clone()))
+            .and(valid_access_token(appservice.registration().hs_token.clone()))
             .map(move || appservice.clone())
             .and(http_request().and_then(|request| async move {
                 let request = crate::transform_request_path(request).map_err(Error::from)?;
@@ -152,34 +162,57 @@ mod filters {
 }
 
 mod handlers {
+    use percent_encoding::percent_decode_str;
+
     use super::*;
 
     pub async fn user(
-        _user_id: String,
-        _appservice: AppService,
-        _request: http::Request<Bytes>,
-    ) -> StdResult<impl warp::Reply, Rejection> {
+        user_id: String,
+        appservice: AppService,
+        request: http::Request<Bytes>,
+    ) -> Result<impl Reply, Rejection> {
+        if let Some(user_exists) = appservice.event_handler.users.lock().await.as_mut() {
+            let user_id = percent_decode_str(&user_id).decode_utf8().map_err(Error::from)?;
+            let request = query_user::IncomingRequest::try_from_http_request(request, &[user_id])
+                .map_err(Error::from)?;
+            return if user_exists(appservice.clone(), request).await {
+                Ok(warp::reply::json(&String::from("{}")))
+            } else {
+                Err(warp::reject::not_found())
+            };
+        }
         Ok(warp::reply::json(&String::from("{}")))
     }
 
     pub async fn room(
-        _room_id: String,
-        _appservice: AppService,
-        _request: http::Request<Bytes>,
-    ) -> StdResult<impl warp::Reply, Rejection> {
+        room_id: String,
+        appservice: AppService,
+        request: http::Request<Bytes>,
+    ) -> Result<impl Reply, Rejection> {
+        if let Some(room_exists) = appservice.event_handler.rooms.lock().await.as_mut() {
+            let room_id = percent_decode_str(&room_id).decode_utf8().map_err(Error::from)?;
+            let request = query_room::IncomingRequest::try_from_http_request(request, &[room_id])
+                .map_err(Error::from)?;
+            return if room_exists(appservice.clone(), request).await {
+                Ok(warp::reply::json(&String::from("{}")))
+            } else {
+                Err(warp::reject::not_found())
+            };
+        }
         Ok(warp::reply::json(&String::from("{}")))
     }
 
     pub async fn transaction(
-        _txn_id: String,
+        txn_id: String,
         appservice: AppService,
         request: http::Request<Bytes>,
-    ) -> StdResult<impl warp::Reply, Rejection> {
+    ) -> Result<impl Reply, Rejection> {
         let incoming_transaction: ruma::api::appservice::event::push_events::v1::IncomingRequest =
-            ruma::api::IncomingRequest::try_from_http_request(request).map_err(Error::from)?;
+            ruma::api::IncomingRequest::try_from_http_request(request, &[txn_id])
+                .map_err(Error::from)?;
 
         let client = appservice.get_cached_client(None)?;
-        client.receive_transaction(incoming_transaction).map_err(Error::from).await?;
+        client.receive_transaction(incoming_transaction).await.map_err(Error::from)?;
 
         Ok(warp::reply::json(&String::from("{}")))
     }
@@ -196,7 +229,7 @@ struct ErrorMessage {
     message: String,
 }
 
-pub async fn handle_rejection(err: Rejection) -> std::result::Result<impl Reply, Rejection> {
+pub async fn handle_rejection(err: Rejection) -> Result<impl Reply, Rejection> {
     if err.find::<Unauthorized>().is_some() || err.find::<warp::reject::InvalidQuery>().is_some() {
         let code = http::StatusCode::UNAUTHORIZED;
         let message = "UNAUTHORIZED";

crates/matrix-sdk-appservice/tests/tests.rs

@@ -4,17 +4,14 @@ use std::{
 };
 
 use matrix_sdk::{
-    ruma::{
-        api::appservice::Registration,
-        events::{room::member::MemberEventContent, SyncStateEvent},
-    },
-    ClientConfig, RequestConfig,
+    config::RequestConfig,
+    ruma::{api::appservice::Registration, events::room::member::OriginalSyncRoomMemberEvent},
+    Client,
 };
 use matrix_sdk_appservice::*;
 use matrix_sdk_test::{appservice::TransactionBuilder, async_test, EventsJson};
-use ruma::room_id;
+use ruma::{api::MatrixVersion, room_id};
 use serde_json::json;
-#[cfg(feature = "warp")]
 use warp::{Filter, Reply};
 
 fn registration_string() -> String {
@@ -36,16 +33,17 @@ async fn appservice(registration: Option<Registration>) -> Result<AppService> {
     let homeserver_url = mockito::server_url();
     let server_name = "localhost";
 
-    let client_config =
-        ClientConfig::default().request_config(RequestConfig::default().disable_retry());
+    let client_builder = Client::builder()
+        .request_config(RequestConfig::default().disable_retry())
+        .server_versions([MatrixVersion::V1_0]);
 
-    Ok(AppService::new_with_config(
+    AppService::with_client_builder(
         homeserver_url.as_ref(),
         server_name,
         registration,
-        client_config,
+        client_builder,
     )
-    .await?)
+    .await
 }
 
 #[async_test]
@@ -88,7 +86,6 @@ async fn test_put_transaction() -> Result<()> {
 
     let appservice = appservice(None).await?;
 
-    #[cfg(feature = "warp")]
     let status = warp::test::request()
         .method("PUT")
         .path(uri)
@@ -107,10 +104,10 @@ async fn test_put_transaction() -> Result<()> {
 #[async_test]
 async fn test_get_user() -> Result<()> {
     let appservice = appservice(None).await?;
+    appservice.register_user_query(Box::new(|_, _| Box::pin(async move { true }))).await;
 
     let uri = "/_matrix/app/v1/users/%40_botty_1%3Adev.famedly.local?access_token=hs_token";
 
-    #[cfg(feature = "warp")]
     let status = warp::test::request()
         .method("GET")
         .path(uri)
@@ -128,10 +125,10 @@ async fn test_get_user() -> Result<()> {
 #[async_test]
 async fn test_get_room() -> Result<()> {
     let appservice = appservice(None).await?;
+    appservice.register_room_query(Box::new(|_, _| Box::pin(async move { true }))).await;
 
     let uri = "/_matrix/app/v1/rooms/%23magicforest%3Aexample.com?access_token=hs_token";
 
-    #[cfg(feature = "warp")]
     let status = warp::test::request()
         .method("GET")
         .path(uri)
@@ -156,7 +153,6 @@ async fn test_invalid_access_token() -> Result<()> {
 
     let appservice = appservice(None).await?;
 
-    #[cfg(feature = "warp")]
     let status = warp::test::request()
         .method("PUT")
         .path(uri)
@@ -182,7 +178,6 @@ async fn test_no_access_token() -> Result<()> {
 
     let appservice = appservice(None).await?;
 
-    #[cfg(feature = "warp")]
     {
         let status = warp::test::request()
             .method("PUT")
@@ -202,14 +197,14 @@ async fn test_no_access_token() -> Result<()> {
 
 #[async_test]
 async fn test_event_handler() -> Result<()> {
-    let mut appservice = appservice(None).await?;
+    let appservice = appservice(None).await?;
 
     #[allow(clippy::mutex_atomic)]
     let on_state_member = Arc::new(Mutex::new(false));
     appservice
         .register_event_handler({
             let on_state_member = on_state_member.clone();
-            move |_ev: SyncStateEvent<MemberEventContent>| {
+            move |_ev: OriginalSyncRoomMemberEvent| {
                 *on_state_member.lock().unwrap() = true;
                 future::ready(())
             }
@@ -222,7 +217,6 @@ async fn test_event_handler() -> Result<()> {
     transaction_builder.add_room_event(EventsJson::Member);
     let transaction = transaction_builder.build_json_transaction();
 
-    #[cfg(feature = "warp")]
     warp::test::request()
         .method("PUT")
         .path(uri)
@@ -241,7 +235,6 @@ async fn test_event_handler() -> Result<()> {
 async fn test_unrelated_path() -> Result<()> {
     let appservice = appservice(None).await?;
 
-    #[cfg(feature = "warp")]
     let status = {
         let consumer_filter = warp::any()
             .and(appservice.warp_filter())
@@ -278,7 +271,6 @@ async fn test_appservice_on_sub_path() -> Result<()> {
 
     let appservice = appservice(None).await?;
 
-    #[cfg(feature = "warp")]
     {
         warp::test::request()
             .method("PUT")
@@ -297,8 +289,8 @@ async fn test_appservice_on_sub_path() -> Result<()> {
 
     let members = appservice
         .get_cached_client(None)?
-        .get_room(&room_id)
-        .expect("Expected room to be availabe")
+        .get_room(room_id)
+        .expect("Expected room to be available")
         .members_no_sync()
         .await?;
 

crates/matrix-sdk-base/Cargo.toml

@@ -0,0 +1,65 @@
+[package]
+authors = ["Damir Jelić <poljar@termina.org.uk>"]
+description = "The base component to build a Matrix client library."
+edition = "2021"
+homepage = "https://github.com/matrix-org/matrix-rust-sdk"
+keywords = ["matrix", "chat", "messaging", "ruma", "nio"]
+license = "Apache-2.0"
+name = "matrix-sdk-base"
+readme = "README.md"
+repository = "https://github.com/matrix-org/matrix-rust-sdk"
+rust-version = "1.60"
+version = "0.5.0"
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]
+
+[features]
+default = []
+e2e-encryption = ["matrix-sdk-crypto"]
+qrcode = ["matrix-sdk-crypto/qrcode"]
+experimental-timeline = []
+
+# helpers for testing features build upon this
+testing = ["http"]
+
+[dependencies]
+async-stream = "0.3.3"
+async-trait = "0.1.53"
+chacha20poly1305 = { version = "0.9.0", optional = true }
+dashmap = "5.2.0"
+futures-channel = "0.3.21"
+futures-core = "0.3.21"
+futures-util = { version = "0.3.21", default-features = false }
+hmac = { version = "0.12.1", optional = true }
+http = { version = "0.2.6", optional = true }
+lru = "0.7.5"
+matrix-sdk-common = { version = "0.5.0", path = "../matrix-sdk-common" }
+matrix-sdk-crypto = { version = "0.5.0", path = "../matrix-sdk-crypto", optional = true }
+pbkdf2 = { version = "0.11.0", default-features = false, optional = true }
+rand = { version = "0.8.5", optional = true }
+serde = { version = "1.0.136", features = ["rc"] }
+serde_json = "1.0.79"
+sha2 = { version = "0.10.2", optional = true }
+thiserror = "1.0.30"
+tracing = "0.1.34"
+zeroize = { version = "1.3.0", features = ["zeroize_derive"] }
+
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+ruma = { version = "0.6.1", features = ["client-api-c", "js", "signatures"] }
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+ruma = { version = "0.6.2", features = ["client-api-c", "signatures"] }
+
+[dev-dependencies]
+futures = { version = "0.3.21", default-features = false, features = ["executor"] }
+tracing = { version = "0.1.26", features = ["log"] }
+http = "0.2.6"
+assign = "1.1.1"
+env_logger = "0.9.0"
+matrix-sdk-test = { version = "0.5.0", path = "../matrix-sdk-test" }
+tokio = { version = "1.17.0", default-features = false, features = ["rt-multi-thread", "macros"] }
+
+[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
+wasm-bindgen-test = "0.3.30"

crates/matrix-sdk-base/README.md

@@ -0,0 +1,10 @@
+This crate implements the base to build a [Matrix](https://matrix.org/) client
+library.
+
+## Crate Feature Flags
+
+The following crate feature flags are available:
+
+* `encryption`: Enables end-to-end encryption support in the library.
+* `qrcode`: Enables QRcode generation and reading code.
+* `testing`: Provides facilities and functions for tests, in particular for integration testing store implementations. ATTENTION: do not ever use outside of tests, we do not provide any stability warantees on these, these are merely helpers. If you find you _need_ any function provided here outside of tests, please open a Github Issue and inform us about your use case for us to consider.

crates/matrix-sdk-base/src/error.rs

@@ -17,7 +17,7 @@
 
 use std::io::Error as IoError;
 
-#[cfg(feature = "encryption")]
+#[cfg(feature = "e2e-encryption")]
 use matrix_sdk_crypto::{CryptoStoreError, MegolmError, OlmError};
 use serde_json::Error as JsonError;
 use thiserror::Error;
@@ -26,6 +26,7 @@ use thiserror::Error;
 pub type Result<T, E = Error> = std::result::Result<T, E>;
 
 /// Internal representation of errors.
+#[non_exhaustive]
 #[derive(Error, Debug)]
 pub enum Error {
     /// Queried endpoint requires authentication but was called on an anonymous
@@ -33,6 +34,12 @@ pub enum Error {
     #[error("the queried endpoint requires authentication but was called before logging in")]
     AuthenticationRequired,
 
+    /// Attempting to restore a session after the olm-machine has already been
+    /// set up fails
+    #[cfg(feature = "e2e-encryption")]
+    #[error("The olm machine has already been initialized")]
+    BadCryptoStoreState,
+
     /// A generic error returned when the state store fails not due to
     /// IO or (de)serialization.
     #[error(transparent)]
@@ -47,20 +54,17 @@ pub enum Error {
     IoError(#[from] IoError),
 
     /// An error occurred in the crypto store.
-    #[cfg(feature = "encryption")]
-    #[cfg_attr(feature = "docs", doc(cfg(encryption)))]
+    #[cfg(feature = "e2e-encryption")]
     #[error(transparent)]
     CryptoStore(#[from] CryptoStoreError),
 
     /// An error occurred during a E2EE operation.
-    #[cfg(feature = "encryption")]
-    #[cfg_attr(feature = "docs", doc(cfg(encryption)))]
+    #[cfg(feature = "e2e-encryption")]
     #[error(transparent)]
     OlmError(#[from] OlmError),
 
     /// An error occurred during a E2EE group operation.
-    #[cfg(feature = "encryption")]
-    #[cfg_attr(feature = "docs", doc(cfg(encryption)))]
+    #[cfg(feature = "e2e-encryption")]
     #[error(transparent)]
     MegolmError(#[from] MegolmError),
 }

crates/matrix-sdk-base/src/lib.rs

@@ -14,19 +14,13 @@
 // limitations under the License.
 
 #![doc = include_str!("../README.md")]
-#![cfg_attr(feature = "docs", feature(doc_cfg))]
-#![deny(
-    missing_debug_implementations,
-    missing_docs,
-    trivial_casts,
-    trivial_numeric_casts,
-    unused_extern_crates,
-    unused_import_braces,
-    unused_qualifications
-)]
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+#![warn(missing_docs, missing_debug_implementations)]
 
 pub use matrix_sdk_common::*;
 
+#[cfg(feature = "experimental-timeline")]
+pub use crate::timeline_stream::TimelineStreamError;
 pub use crate::{
     error::{Error, Result},
     session::Session,
@@ -37,11 +31,18 @@ mod error;
 pub mod media;
 mod rooms;
 mod session;
-mod store;
+pub mod store;
+#[cfg(feature = "experimental-timeline")]
+mod timeline_stream;
+mod utils;
 
-pub use client::{BaseClient, BaseClientConfig};
-#[cfg(feature = "encryption")]
-#[cfg_attr(feature = "docs", doc(cfg(encryption)))]
+pub use client::BaseClient;
+#[cfg(any(test, feature = "testing"))]
+pub use http;
+#[cfg(feature = "e2e-encryption")]
 pub use matrix_sdk_crypto as crypto;
-pub use rooms::{Room, RoomInfo, RoomMember, RoomType};
+pub use rooms::{DisplayName, Room, RoomInfo, RoomMember, RoomType};
 pub use store::{StateChanges, StateStore, Store, StoreError};
+pub use utils::{
+    MinimalRoomMemberEvent, MinimalStateEvent, OriginalMinimalStateEvent, RedactedMinimalStateEvent,
+};

crates/matrix-sdk-base/src/media.rs

@@ -0,0 +1,168 @@
+//! Common types for [media content](https://matrix.org/docs/spec/client_server/r0.6.1#id66).
+
+use ruma::{
+    api::client::media::get_content_thumbnail::v3::Method,
+    events::{
+        room::{
+            message::{
+                AudioMessageEventContent, FileMessageEventContent, ImageMessageEventContent,
+                LocationMessageEventContent, VideoMessageEventContent,
+            },
+            MediaSource,
+        },
+        sticker::StickerEventContent,
+    },
+    UInt,
+};
+
+const UNIQUE_SEPARATOR: &str = "_";
+
+/// A trait to uniquely identify values of the same type.
+pub trait UniqueKey {
+    /// A string that uniquely identifies `Self` compared to other values of
+    /// the same type.
+    fn unique_key(&self) -> String;
+}
+
+/// The requested format of a media file.
+#[derive(Clone, Debug)]
+pub enum MediaFormat {
+    /// The file that was uploaded.
+    File,
+
+    /// A thumbnail of the file that was uploaded.
+    Thumbnail(MediaThumbnailSize),
+}
+
+impl UniqueKey for MediaFormat {
+    fn unique_key(&self) -> String {
+        match self {
+            Self::File => "file".into(),
+            Self::Thumbnail(size) => size.unique_key(),
+        }
+    }
+}
+
+/// The requested size of a media thumbnail.
+#[derive(Clone, Debug)]
+pub struct MediaThumbnailSize {
+    /// The desired resizing method.
+    pub method: Method,
+
+    /// The desired width of the thumbnail. The actual thumbnail may not match
+    /// the size specified.
+    pub width: UInt,
+
+    /// The desired height of the thumbnail. The actual thumbnail may not match
+    /// the size specified.
+    pub height: UInt,
+}
+
+impl UniqueKey for MediaThumbnailSize {
+    fn unique_key(&self) -> String {
+        format!("{}{}{}x{}", self.method, UNIQUE_SEPARATOR, self.width, self.height)
+    }
+}
+
+impl UniqueKey for MediaSource {
+    fn unique_key(&self) -> String {
+        match self {
+            Self::Plain(uri) => uri.to_string(),
+            Self::Encrypted(file) => file.url.to_string(),
+        }
+    }
+}
+
+/// A request for media data.
+#[derive(Clone, Debug)]
+pub struct MediaRequest {
+    /// The source of the media file.
+    pub source: MediaSource,
+
+    /// The requested format of the media data.
+    pub format: MediaFormat,
+}
+
+impl UniqueKey for MediaRequest {
+    fn unique_key(&self) -> String {
+        format!("{}{}{}", self.source.unique_key(), UNIQUE_SEPARATOR, self.format.unique_key())
+    }
+}
+/// Trait for media event content.
+pub trait MediaEventContent {
+    /// Get the source of the file for `Self`.
+    ///
+    /// Returns `None` if `Self` has no file.
+    fn source(&self) -> Option<MediaSource>;
+
+    /// Get the source of the thumbnail for `Self`.
+    ///
+    /// Returns `None` if `Self` has no thumbnail.
+    fn thumbnail_source(&self) -> Option<MediaSource>;
+}
+
+impl MediaEventContent for StickerEventContent {
+    fn source(&self) -> Option<MediaSource> {
+        Some(MediaSource::Plain(self.url.clone()))
+    }
+
+    fn thumbnail_source(&self) -> Option<MediaSource> {
+        None
+    }
+}
+
+impl MediaEventContent for AudioMessageEventContent {
+    fn source(&self) -> Option<MediaSource> {
+        Some(self.source.clone())
+    }
+
+    fn thumbnail_source(&self) -> Option<MediaSource> {
+        None
+    }
+}
+
+impl MediaEventContent for FileMessageEventContent {
+    fn source(&self) -> Option<MediaSource> {
+        Some(self.source.clone())
+    }
+
+    fn thumbnail_source(&self) -> Option<MediaSource> {
+        self.info.as_ref()?.thumbnail_source.clone()
+    }
+}
+
+impl MediaEventContent for ImageMessageEventContent {
+    fn source(&self) -> Option<MediaSource> {
+        Some(self.source.clone())
+    }
+
+    fn thumbnail_source(&self) -> Option<MediaSource> {
+        self.info
+            .as_ref()
+            .and_then(|info| info.thumbnail_source.clone())
+            .or_else(|| Some(self.source.clone()))
+    }
+}
+
+impl MediaEventContent for VideoMessageEventContent {
+    fn source(&self) -> Option<MediaSource> {
+        Some(self.source.clone())
+    }
+
+    fn thumbnail_source(&self) -> Option<MediaSource> {
+        self.info
+            .as_ref()
+            .and_then(|info| info.thumbnail_source.clone())
+            .or_else(|| Some(self.source.clone()))
+    }
+}
+
+impl MediaEventContent for LocationMessageEventContent {
+    fn source(&self) -> Option<MediaSource> {
+        None
+    }
+
+    fn thumbnail_source(&self) -> Option<MediaSource> {
+        self.info.as_ref()?.thumbnail_source.clone()
+    }
+}

crates/matrix-sdk-base/src/rooms/members.rs

@@ -17,21 +17,24 @@ use std::sync::Arc;
 use ruma::{
     events::{
         presence::PresenceEvent,
-        room::{member::MemberEventContent, power_levels::PowerLevelsEventContent},
-        SyncStateEvent,
+        room::{member::MembershipState, power_levels::SyncRoomPowerLevelsEvent},
     },
     MxcUri, UserId,
 };
 
-use crate::deserialized_responses::MemberEvent;
+use crate::{deserialized_responses::MemberEvent, MinimalRoomMemberEvent};
 
 /// A member of a room.
 #[derive(Clone, Debug)]
 pub struct RoomMember {
     pub(crate) event: Arc<MemberEvent>,
-    pub(crate) profile: Arc<Option<MemberEventContent>>,
+    // The latest member event sent by the member themselves.
+    // Stored in addition to the latest member event overall to get displayname
+    // and avatar from, which should be ignored on events sent by others.
+    pub(crate) profile: Arc<Option<MinimalRoomMemberEvent>>,
+    #[allow(dead_code)]
     pub(crate) presence: Arc<Option<PresenceEvent>>,
-    pub(crate) power_levels: Arc<Option<SyncStateEvent<PowerLevelsEventContent>>>,
+    pub(crate) power_levels: Arc<Option<SyncRoomPowerLevelsEvent>>,
     pub(crate) max_power_level: i64,
     pub(crate) is_room_creator: bool,
     pub(crate) display_name_ambiguous: bool,
@@ -40,15 +43,15 @@ pub struct RoomMember {
 impl RoomMember {
     /// Get the unique user id of this member.
     pub fn user_id(&self) -> &UserId {
-        &self.event.state_key
+        self.event.user_id()
     }
 
     /// Get the display name of the member if there is one.
     pub fn display_name(&self) -> Option<&str> {
         if let Some(p) = self.profile.as_ref() {
-            p.displayname.as_deref()
+            p.as_original().and_then(|e| e.content.displayname.as_deref())
         } else {
-            self.event.content.displayname.as_deref()
+            self.event.original_content()?.displayname.as_deref()
         }
     }
 
@@ -66,9 +69,10 @@ impl RoomMember {
 
     /// Get the avatar url of the member, if there is one.
     pub fn avatar_url(&self) -> Option<&MxcUri> {
-        match self.profile.as_ref() {
-            Some(p) => p.avatar_url.as_ref(),
-            None => self.event.content.avatar_url.as_ref(),
+        if let Some(p) = self.profile.as_ref() {
+            p.as_original().and_then(|e| e.content.avatar_url.as_deref())
+        } else {
+            self.event.original_content()?.avatar_url.as_deref()
         }
     }
 
@@ -86,16 +90,9 @@ impl RoomMember {
 
     /// Get the power level of this member.
     pub fn power_level(&self) -> i64 {
-        self.power_levels
+        (*self.power_levels)
             .as_ref()
-            .as_ref()
-            .map(|e| {
-                e.content
-                    .users
-                    .get(self.user_id())
-                    .map(|p| (*p).into())
-                    .unwrap_or_else(|| e.content.users_default.into())
-            })
+            .map(|e| e.power_levels().for_user(self.user_id()).into())
             .unwrap_or_else(|| if self.is_room_creator { 100 } else { 0 })
     }
 
@@ -106,4 +103,13 @@ impl RoomMember {
     pub fn name_ambiguous(&self) -> bool {
         self.display_name_ambiguous
     }
+
+    /// Get the membership state of this member.
+    pub fn membership(&self) -> &MembershipState {
+        if let Some(p) = self.profile.as_ref() {
+            p.membership()
+        } else {
+            self.event.membership()
+        }
+    }
 }

crates/matrix-sdk-base/src/rooms/mod.rs

@@ -0,0 +1,344 @@
+mod members;
+mod normal;
+
+use std::{cmp::max, collections::HashSet, fmt};
+
+pub use members::RoomMember;
+pub use normal::{Room, RoomInfo, RoomType};
+use ruma::{
+    events::{
+        room::{
+            avatar::RoomAvatarEventContent, canonical_alias::RoomCanonicalAliasEventContent,
+            create::RoomCreateEventContent, encryption::RoomEncryptionEventContent,
+            guest_access::RoomGuestAccessEventContent,
+            history_visibility::RoomHistoryVisibilityEventContent,
+            join_rules::RoomJoinRulesEventContent, name::RoomNameEventContent,
+            redaction::OriginalSyncRoomRedactionEvent, tombstone::RoomTombstoneEventContent,
+            topic::RoomTopicEventContent,
+        },
+        AnyStrippedStateEvent, AnySyncStateEvent, RedactContent, RedactedEventContent,
+        StateEventContent, SyncStateEvent,
+    },
+    EventId, OwnedUserId, RoomVersionId,
+};
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+
+use crate::MinimalStateEvent;
+
+/// The name of the room, either from the metadata or calculated
+/// according to [matrix specification](https://matrix.org/docs/spec/client_server/latest#calculating-the-display-name-for-a-room)
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+pub enum DisplayName {
+    /// The room has been named explicitly as
+    Named(String),
+    /// The room has a canonical alias that should be used
+    Aliased(String),
+    /// The room has not given an explicit name but a name could be
+    /// calculated
+    Calculated(String),
+    /// The room doesn't have a name right now, but used to have one
+    /// e.g. because it was a DM and everyone has left the room
+    EmptyWas(String),
+    /// No useful name could be calculated or ever found
+    Empty,
+}
+
+impl fmt::Display for DisplayName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            DisplayName::Named(s) | DisplayName::Calculated(s) | DisplayName::Aliased(s) => {
+                write!(f, "{}", s)
+            }
+            DisplayName::EmptyWas(s) => write!(f, "Empty Room (was {})", s),
+            DisplayName::Empty => write!(f, "Empty Room"),
+        }
+    }
+}
+
+/// A base room info struct that is the backbone of normal as well as stripped
+/// rooms. Holds all the state events that are important to present a room to
+/// users.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct BaseRoomInfo {
+    /// The avatar URL of this room.
+    avatar: Option<MinimalStateEvent<RoomAvatarEventContent>>,
+    /// The canonical alias of this room.
+    canonical_alias: Option<MinimalStateEvent<RoomCanonicalAliasEventContent>>,
+    /// The `m.room.create` event content of this room.
+    create: Option<MinimalStateEvent<RoomCreateEventContent>>,
+    /// A list of user ids this room is considered as direct message, if this
+    /// room is a DM.
+    pub(crate) dm_targets: HashSet<OwnedUserId>,
+    /// The `m.room.encryption` event content that enabled E2EE in this room.
+    pub(crate) encryption: Option<RoomEncryptionEventContent>,
+    /// The guest access policy of this room.
+    guest_access: Option<MinimalStateEvent<RoomGuestAccessEventContent>>,
+    /// The history visibility policy of this room.
+    history_visibility: Option<MinimalStateEvent<RoomHistoryVisibilityEventContent>>,
+    /// The join rule policy of this room.
+    join_rules: Option<MinimalStateEvent<RoomJoinRulesEventContent>>,
+    /// The maximal power level that can be found in this room.
+    pub(crate) max_power_level: i64,
+    /// The `m.room.name` of this room.
+    name: Option<MinimalStateEvent<RoomNameEventContent>>,
+    /// The `m.room.tombstone` event content of this room.
+    tombstone: Option<MinimalStateEvent<RoomTombstoneEventContent>>,
+    /// The topic of this room.
+    topic: Option<MinimalStateEvent<RoomTopicEventContent>>,
+}
+
+impl BaseRoomInfo {
+    /// Create a new, empty base room info.
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub(crate) fn calculate_room_name(
+        &self,
+        joined_member_count: u64,
+        invited_member_count: u64,
+        heroes: Vec<RoomMember>,
+    ) -> DisplayName {
+        calculate_room_name(
+            joined_member_count,
+            invited_member_count,
+            heroes.iter().take(3).map(|mem| mem.name()).collect::<Vec<&str>>(),
+        )
+    }
+
+    /// Handle a state event for this room and update our info accordingly.
+    ///
+    /// Returns true if the event modified the info, false otherwise.
+    pub fn handle_state_event(&mut self, ev: &AnySyncStateEvent) -> bool {
+        match ev {
+            // No redacted branch - enabling encryption cannot be undone.
+            AnySyncStateEvent::RoomEncryption(SyncStateEvent::Original(encryption)) => {
+                self.encryption = Some(encryption.content.clone());
+            }
+            AnySyncStateEvent::RoomAvatar(a) => {
+                self.avatar = Some(a.into());
+            }
+            AnySyncStateEvent::RoomName(n) => {
+                self.name = Some(n.into());
+            }
+            AnySyncStateEvent::RoomCreate(c) if self.create.is_none() => {
+                self.create = Some(c.into());
+            }
+            AnySyncStateEvent::RoomHistoryVisibility(h) => {
+                self.history_visibility = Some(h.into());
+            }
+            AnySyncStateEvent::RoomGuestAccess(g) => {
+                self.guest_access = Some(g.into());
+            }
+            AnySyncStateEvent::RoomJoinRules(c) => {
+                self.join_rules = Some(c.into());
+            }
+            AnySyncStateEvent::RoomCanonicalAlias(a) => {
+                self.canonical_alias = Some(a.into());
+            }
+            AnySyncStateEvent::RoomTopic(t) => {
+                self.topic = Some(t.into());
+            }
+            AnySyncStateEvent::RoomTombstone(t) => {
+                self.tombstone = Some(t.into());
+            }
+            AnySyncStateEvent::RoomPowerLevels(p) => {
+                self.max_power_level = p
+                    .power_levels()
+                    .users
+                    .values()
+                    .fold(self.max_power_level, |acc, &p| max(acc, p.into()));
+            }
+            _ => return false,
+        }
+
+        true
+    }
+
+    /// Handle a stripped state event for this room and update our info
+    /// accordingly.
+    ///
+    /// Returns true if the event modified the info, false otherwise.
+    pub fn handle_stripped_state_event(&mut self, ev: &AnyStrippedStateEvent) -> bool {
+        match ev {
+            AnyStrippedStateEvent::RoomEncryption(encryption) => {
+                self.encryption = Some(encryption.content.clone());
+            }
+            AnyStrippedStateEvent::RoomAvatar(a) => {
+                self.avatar = Some(a.into());
+            }
+            AnyStrippedStateEvent::RoomName(n) => {
+                self.name = Some(n.into());
+            }
+            AnyStrippedStateEvent::RoomCreate(c) if self.create.is_none() => {
+                self.create = Some(c.into());
+            }
+            AnyStrippedStateEvent::RoomHistoryVisibility(h) => {
+                self.history_visibility = Some(h.into());
+            }
+            AnyStrippedStateEvent::RoomGuestAccess(g) => {
+                self.guest_access = Some(g.into());
+            }
+            AnyStrippedStateEvent::RoomJoinRules(c) => {
+                self.join_rules = Some(c.into());
+            }
+            AnyStrippedStateEvent::RoomCanonicalAlias(a) => {
+                self.canonical_alias = Some(a.into());
+            }
+            AnyStrippedStateEvent::RoomTopic(t) => {
+                self.topic = Some(t.into());
+            }
+            AnyStrippedStateEvent::RoomTombstone(t) => {
+                self.tombstone = Some(t.into());
+            }
+            AnyStrippedStateEvent::RoomPowerLevels(p) => {
+                self.max_power_level = p
+                    .content
+                    .users
+                    .values()
+                    .fold(self.max_power_level, |acc, &p| max(acc, p.into()));
+            }
+            _ => return false,
+        }
+
+        true
+    }
+
+    pub fn handle_redaction(&mut self, event: &OriginalSyncRoomRedactionEvent) {
+        let room_version = self
+            .create
+            .as_ref()
+            .and_then(|ev| Some(ev.as_original()?.content.room_version.to_owned()))
+            .unwrap_or(RoomVersionId::V1);
+
+        // FIXME: Use let chains once available to get rid of unwrap()s
+        if self.avatar.has_event_id(&event.redacts) {
+            self.avatar.as_mut().unwrap().redact(&room_version);
+        } else if self.canonical_alias.has_event_id(&event.redacts) {
+            self.canonical_alias.as_mut().unwrap().redact(&room_version);
+        } else if self.create.has_event_id(&event.redacts) {
+            self.create.as_mut().unwrap().redact(&room_version);
+        } else if self.guest_access.has_event_id(&event.redacts) {
+            self.guest_access.as_mut().unwrap().redact(&room_version);
+        } else if self.history_visibility.has_event_id(&event.redacts) {
+            self.history_visibility.as_mut().unwrap().redact(&room_version);
+        } else if self.join_rules.has_event_id(&event.redacts) {
+            self.join_rules.as_mut().unwrap().redact(&room_version);
+        } else if self.name.has_event_id(&event.redacts) {
+            self.name.as_mut().unwrap().redact(&room_version);
+        } else if self.tombstone.has_event_id(&event.redacts) {
+            self.tombstone.as_mut().unwrap().redact(&room_version);
+        } else if self.topic.has_event_id(&event.redacts) {
+            self.topic.as_mut().unwrap().redact(&room_version);
+        }
+    }
+}
+
+trait OptionExt {
+    fn has_event_id(&self, ev_id: &EventId) -> bool;
+}
+
+impl<C> OptionExt for Option<MinimalStateEvent<C>>
+where
+    C: StateEventContent + RedactContent,
+    C::Redacted: StateEventContent + RedactedEventContent + DeserializeOwned,
+{
+    fn has_event_id(&self, ev_id: &EventId) -> bool {
+        self.as_ref().and_then(|ev| ev.event_id()).map_or(false, |id| id == ev_id)
+    }
+}
+
+impl Default for BaseRoomInfo {
+    fn default() -> Self {
+        Self {
+            avatar: None,
+            canonical_alias: None,
+            create: None,
+            dm_targets: Default::default(),
+            encryption: None,
+            guest_access: None,
+            history_visibility: None,
+            join_rules: None,
+            max_power_level: 100,
+            name: None,
+            tombstone: None,
+            topic: None,
+        }
+    }
+}
+
+/// Calculate room name according to step 3 of the [naming algorithm.]
+fn calculate_room_name(
+    joined_member_count: u64,
+    invited_member_count: u64,
+    heroes: Vec<&str>,
+) -> DisplayName {
+    let heroes_count = heroes.len() as u64;
+    let invited_joined = invited_member_count + joined_member_count;
+    let invited_joined_minus_one = invited_joined.saturating_sub(1);
+
+    let names = if heroes_count >= invited_joined_minus_one {
+        let mut names = heroes;
+        // stabilize ordering
+        names.sort_unstable();
+        names.join(", ")
+    } else if heroes_count < invited_joined_minus_one && invited_joined > 1 {
+        let mut names = heroes;
+        names.sort_unstable();
+
+        // TODO: What length does the spec want us to use here and in
+        // the `else`?
+        format!("{}, and {} others", names.join(", "), (invited_joined - heroes_count))
+    } else {
+        "".to_owned()
+    };
+
+    // User is alone.
+    if invited_joined <= 1 {
+        if names.is_empty() {
+            DisplayName::Empty
+        } else {
+            DisplayName::EmptyWas(names)
+        }
+    } else {
+        DisplayName::Calculated(names)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+
+    fn test_calculate_room_name() {
+        let mut actual = calculate_room_name(2, 0, vec!["a"]);
+        assert_eq!(DisplayName::Calculated("a".to_owned()), actual);
+
+        actual = calculate_room_name(3, 0, vec!["a", "b"]);
+        assert_eq!(DisplayName::Calculated("a, b".to_owned()), actual);
+
+        actual = calculate_room_name(4, 0, vec!["a", "b", "c"]);
+        assert_eq!(DisplayName::Calculated("a, b, c".to_owned()), actual);
+
+        actual = calculate_room_name(5, 0, vec!["a", "b", "c"]);
+        assert_eq!(DisplayName::Calculated("a, b, c, and 2 others".to_owned()), actual);
+
+        actual = calculate_room_name(0, 0, vec![]);
+        assert_eq!(DisplayName::Empty, actual);
+
+        actual = calculate_room_name(1, 0, vec![]);
+        assert_eq!(DisplayName::Empty, actual);
+
+        actual = calculate_room_name(0, 1, vec![]);
+        assert_eq!(DisplayName::Empty, actual);
+
+        actual = calculate_room_name(1, 0, vec!["a"]);
+        assert_eq!(DisplayName::EmptyWas("a".to_owned()), actual);
+
+        actual = calculate_room_name(1, 0, vec!["a", "b"]);
+        assert_eq!(DisplayName::EmptyWas("a, b".to_owned()), actual);
+
+        actual = calculate_room_name(1, 0, vec!["a", "b", "c"]);
+        assert_eq!(DisplayName::EmptyWas("a, b, c".to_owned()), actual);
+    }
+}

crates/matrix-sdk-base/src/session.rs

@@ -15,17 +15,42 @@
 
 //! User sessions.
 
-use ruma::{DeviceId, UserId};
+use ruma::{OwnedDeviceId, OwnedUserId};
 use serde::{Deserialize, Serialize};
 
 /// A user session, containing an access token and information about the
 /// associated user account.
+///
+/// # Example
+///
+/// ```
+/// use matrix_sdk_base::Session;
+/// use ruma::{device_id, user_id};
+///
+/// let session = Session {
+///     access_token: "My-Token".to_owned(),
+///     user_id: user_id!("@example:localhost").to_owned(),
+///     device_id: device_id!("MYDEVICEID").to_owned(),
+/// };
+///
+/// assert_eq!(session.device_id.as_str(), "MYDEVICEID");
+/// ```
 #[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)]
 pub struct Session {
     /// The access token used for this session.
     pub access_token: String,
     /// The user the access token was issued for.
-    pub user_id: UserId,
+    pub user_id: OwnedUserId,
     /// The ID of the client device
-    pub device_id: Box<DeviceId>,
+    pub device_id: OwnedDeviceId,
+}
+
+impl From<ruma::api::client::session::login::v3::Response> for Session {
+    fn from(response: ruma::api::client::session::login::v3::Response) -> Self {
+        Self {
+            access_token: response.access_token,
+            user_id: response.user_id,
+            device_id: response.device_id,
+        }
+    }
 }

crates/matrix-sdk-base/src/store/ambiguity_map.rs

@@ -15,7 +15,10 @@
 use std::collections::{BTreeMap, BTreeSet};
 
 use matrix_sdk_common::deserialized_responses::{AmbiguityChange, MemberEvent};
-use ruma::{events::room::member::MembershipState, EventId, RoomId, UserId};
+use ruma::{
+    events::room::member::{MembershipState, SyncRoomMemberEvent},
+    OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UserId,
+};
 use tracing::trace;
 
 use super::{Result, StateChanges};
@@ -24,18 +27,18 @@ use crate::Store;
 #[derive(Clone, Debug)]
 pub struct AmbiguityCache {
     pub store: Store,
-    pub cache: BTreeMap<RoomId, BTreeMap<String, BTreeSet<UserId>>>,
-    pub changes: BTreeMap<RoomId, BTreeMap<EventId, AmbiguityChange>>,
+    pub cache: BTreeMap<OwnedRoomId, BTreeMap<String, BTreeSet<OwnedUserId>>>,
+    pub changes: BTreeMap<OwnedRoomId, BTreeMap<OwnedEventId, AmbiguityChange>>,
 }
 
 #[derive(Clone, Debug)]
 struct AmbiguityMap {
     display_name: String,
-    users: BTreeSet<UserId>,
+    users: BTreeSet<OwnedUserId>,
 }
 
 impl AmbiguityMap {
-    fn remove(&mut self, user_id: &UserId) -> Option<UserId> {
+    fn remove(&mut self, user_id: &UserId) -> Option<OwnedUserId> {
         self.users.remove(user_id);
 
         if self.user_count() == 1 {
@@ -45,7 +48,7 @@ impl AmbiguityMap {
         }
     }
 
-    fn add(&mut self, user_id: UserId) -> Option<UserId> {
+    fn add(&mut self, user_id: OwnedUserId) -> Option<OwnedUserId> {
         let ambiguous_user =
             if self.user_count() == 1 { self.users.iter().next().cloned() } else { None };
 
@@ -72,7 +75,7 @@ impl AmbiguityCache {
         &mut self,
         changes: &StateChanges,
         room_id: &RoomId,
-        member_event: &MemberEvent,
+        member_event: &SyncRoomMemberEvent,
     ) -> Result<()> {
         // Synapse seems to have a bug where it puts the same event into the
         // state and the timeline sometimes.
@@ -86,7 +89,7 @@ impl AmbiguityCache {
         if self
             .changes
             .get(room_id)
-            .map(|c| c.contains_key(&member_event.event_id))
+            .map(|c| c.contains_key(member_event.event_id()))
             .unwrap_or(false)
         {
             return Ok(());
@@ -103,9 +106,10 @@ impl AmbiguityCache {
             return Ok(());
         }
 
-        let disambiguated_member = old_map.as_mut().and_then(|o| o.remove(&member_event.state_key));
+        let disambiguated_member =
+            old_map.as_mut().and_then(|o| o.remove(member_event.state_key()));
         let ambiguated_member =
-            new_map.as_mut().and_then(|n| n.add(member_event.state_key.clone()));
+            new_map.as_mut().and_then(|n| n.add(member_event.state_key().clone()));
         let ambiguous = new_map.as_ref().map(|n| n.is_ambiguous()).unwrap_or(false);
 
         self.update(room_id, old_map, new_map);
@@ -116,9 +120,9 @@ impl AmbiguityCache {
             member_ambiguous: ambiguous,
         };
 
-        trace!("Handling display name ambiguity for {}: {:#?}", member_event.state_key, change);
+        trace!("Handling display name ambiguity for {}: {:#?}", member_event.state_key(), change);
 
-        self.add_change(room_id, member_event.event_id.clone(), change);
+        self.add_change(room_id, member_event.event_id().to_owned(), change);
 
         Ok(())
     }
@@ -129,7 +133,7 @@ impl AmbiguityCache {
         old_map: Option<AmbiguityMap>,
         new_map: Option<AmbiguityMap>,
     ) {
-        let entry = self.cache.entry(room_id.clone()).or_insert_with(BTreeMap::new);
+        let entry = self.cache.entry(room_id.to_owned()).or_default();
 
         if let Some(old) = old_map {
             entry.insert(old.display_name, old.users);
@@ -140,47 +144,49 @@ impl AmbiguityCache {
         }
     }
 
-    fn add_change(&mut self, room_id: &RoomId, event_id: EventId, change: AmbiguityChange) {
-        self.changes.entry(room_id.clone()).or_insert_with(BTreeMap::new).insert(event_id, change);
+    fn add_change(&mut self, room_id: &RoomId, event_id: OwnedEventId, change: AmbiguityChange) {
+        self.changes.entry(room_id.to_owned()).or_default().insert(event_id, change);
     }
 
     async fn get(
         &mut self,
         changes: &StateChanges,
         room_id: &RoomId,
-        member_event: &MemberEvent,
+        member_event: &SyncRoomMemberEvent,
     ) -> Result<(Option<AmbiguityMap>, Option<AmbiguityMap>)> {
         use MembershipState::*;
 
         let old_event = if let Some(m) =
-            changes.members.get(room_id).and_then(|m| m.get(&member_event.state_key))
+            changes.members.get(room_id).and_then(|m| m.get(member_event.state_key()))
         {
-            Some(m.clone())
+            Some(MemberEvent::Sync(m.clone()))
         } else {
-            self.store.get_member_event(room_id, &member_event.state_key).await?
+            self.store.get_member_event(room_id, member_event.state_key()).await?
         };
 
         let old_display_name = if let Some(event) = old_event {
-            if matches!(event.content.membership, Join | Invite) {
+            if matches!(event.membership(), Join | Invite) {
                 let display_name = if let Some(d) = changes
                     .profiles
                     .get(room_id)
-                    .and_then(|p| p.get(&member_event.state_key))
-                    .and_then(|p| p.displayname.as_deref())
+                    .and_then(|p| p.get(member_event.state_key()))
+                    .and_then(|p| p.as_original())
+                    .and_then(|p| p.content.displayname.as_deref())
                 {
-                    Some(d.to_string())
+                    Some(d.to_owned())
                 } else if let Some(d) = self
                     .store
-                    .get_profile(room_id, &member_event.state_key)
+                    .get_profile(room_id, member_event.state_key())
                     .await?
-                    .and_then(|c| c.displayname)
+                    .and_then(|p| p.into_original())
+                    .and_then(|p| p.content.displayname)
                 {
                     Some(d)
                 } else {
-                    event.content.displayname.clone()
+                    event.original_content().and_then(|c| c.displayname.clone())
                 };
 
-                Some(display_name.unwrap_or_else(|| event.state_key.localpart().to_string()))
+                Some(display_name.unwrap_or_else(|| event.user_id().localpart().to_owned()))
             } else {
                 None
             }
@@ -189,30 +195,28 @@ impl AmbiguityCache {
         };
 
         let old_map = if let Some(old_name) = old_display_name.as_deref() {
-            let old_display_name_map = if let Some(u) =
-                self.cache.entry(room_id.clone()).or_insert_with(BTreeMap::new).get(old_name)
-            {
-                u.clone()
-            } else {
-                self.store.get_users_with_display_name(room_id, old_name).await?
-            };
+            let old_display_name_map =
+                if let Some(u) = self.cache.entry(room_id.to_owned()).or_default().get(old_name) {
+                    u.clone()
+                } else {
+                    self.store.get_users_with_display_name(room_id, old_name).await?
+                };
 
-            Some(AmbiguityMap { display_name: old_name.to_string(), users: old_display_name_map })
+            Some(AmbiguityMap { display_name: old_name.to_owned(), users: old_display_name_map })
         } else {
             None
         };
 
-        let new_map = if matches!(member_event.content.membership, Join | Invite) {
+        let new_map = if matches!(member_event.membership(), Join | Invite) {
             let new = member_event
-                .content
-                .displayname
-                .as_deref()
-                .unwrap_or_else(|| member_event.state_key.localpart());
+                .as_original()
+                .and_then(|ev| ev.content.displayname.as_deref())
+                .unwrap_or_else(|| member_event.state_key().localpart());
 
             // We don't allow other users to set the display name, so if we
             // have a more trusted version of the display
             // name use that.
-            let new_display_name = if member_event.sender.as_str() == member_event.state_key {
+            let new_display_name = if member_event.sender().as_str() == member_event.state_key() {
                 new
             } else if let Some(old) = old_display_name.as_deref() {
                 old
@@ -220,11 +224,8 @@ impl AmbiguityCache {
                 new
             };
 
-            let new_display_name_map = if let Some(u) = self
-                .cache
-                .entry(room_id.clone())
-                .or_insert_with(BTreeMap::new)
-                .get(new_display_name)
+            let new_display_name_map = if let Some(u) =
+                self.cache.entry(room_id.to_owned()).or_default().get(new_display_name)
             {
                 u.clone()
             } else {
@@ -232,7 +233,7 @@ impl AmbiguityCache {
             };
 
             Some(AmbiguityMap {
-                display_name: new_display_name.to_string(),
+                display_name: new_display_name.to_owned(),
                 users: new_display_name_map,
             })
         } else {

crates/matrix-sdk-base/src/store/integration_tests.rs

@@ -0,0 +1,775 @@
+//! Macro of integration tests for StateStore implementions.
+
+/// Macro building to allow your StateStore implementation to run the entire
+/// tests suite locally.
+///
+/// You need to provide a `async fn get_store() -> StoreResult<impl StateStore>`
+/// providing a fresh store on the same level you invoke the macro.
+///
+/// ## Usage Example:
+/// ```no_run
+/// # use matrix_sdk_base::store::{
+/// #    StateStore,
+/// #    MemoryStore as MyStore,
+/// #    Result as StoreResult,
+/// # };
+///
+/// #[cfg(test)]
+/// mod tests {
+///
+///    use super::{MyStore, StoreResult, StateStore};
+///
+///    async fn get_store() -> StoreResult<impl StateStore> {
+///        Ok(MyStore::new())
+///    }
+///
+///    statestore_integration_tests! { integration }
+/// }
+/// ```
+#[allow(unused_macros, unused_extern_crates)]
+#[macro_export]
+macro_rules! statestore_integration_tests {
+    ($($name:ident)*) => {
+        $(
+            mod $name {
+
+                #[cfg(feature = "experimental-timeline")]
+                use futures_util::StreamExt;
+                #[cfg(feature = "experimental-timeline")]
+                use ruma::{
+                    api::{
+                        client::{
+                            message::get_message_events::v3::Response as MessageResponse,
+                            sync::sync_events::v3::Response as SyncResponse,
+                        },
+                        IncomingResponse,
+                    }
+                };
+                use matrix_sdk_test::{async_test, test_json};
+                use ruma::{
+                    api::client::media::get_content_thumbnail::v3::Method,
+                    device_id, event_id,
+                    events::{
+                        presence::PresenceEvent,
+                        room::{
+                            member::{
+                                MembershipState, OriginalSyncRoomMemberEvent, SyncRoomMemberEvent,
+                                RoomMemberEventContent, StrippedRoomMemberEvent,
+                            },
+                            power_levels::RoomPowerLevelsEventContent,
+                            MediaSource,
+                        },
+                        AnyEphemeralRoomEventContent, AnySyncEphemeralRoomEvent,
+                        AnyStrippedStateEvent, AnyGlobalAccountDataEvent, AnyRoomAccountDataEvent,
+                        AnySyncStateEvent, GlobalAccountDataEventType, RoomAccountDataEventType,
+                        StateEventType, StateUnsigned,
+                    },
+                    mxc_uri,
+                    receipt::ReceiptType,
+                    room_id,
+                    serde::Raw,
+                    uint, user_id, MilliSecondsSinceUnixEpoch, UserId, EventId, OwnedEventId,
+                    RoomId,
+                };
+                use serde_json::{json, Value as JsonValue};
+
+                use std::collections::{BTreeMap, BTreeSet};
+
+                #[cfg(feature = "experimental-timeline")]
+                use $crate::{
+                    http::Response,
+                    deserialized_responses::{ RoomEvent, SyncRoomEvent, TimelineSlice},
+                };
+                use $crate::{
+                    RoomType, Session,
+                    media::{MediaFormat, MediaRequest, MediaThumbnailSize},
+                    store::{
+                        Store,
+                        StateStore,
+                        Result as StoreResult,
+                        StateChanges
+                    }
+                };
+
+                use super::get_store;
+
+
+                fn user_id() -> &'static UserId {
+                    user_id!("@example:localhost")
+                }
+                pub(crate) fn invited_user_id() -> &'static UserId {
+                    user_id!("@invited:localhost")
+                }
+
+                pub(crate) fn room_id() -> &'static RoomId {
+                    room_id!("!test:localhost")
+                }
+
+                pub(crate) fn stripped_room_id() -> &'static RoomId {
+                    room_id!("!stripped:localhost")
+                }
+
+                pub(crate) fn first_receipt_event_id() -> &'static EventId {
+                    event_id!("$example")
+                }
+
+                /// Populate the given `StateStore`.
+                pub(crate) async fn populated_store(inner: Box<dyn StateStore>) -> StoreResult<Store> {
+                    let mut changes = StateChanges::default();
+                    let store = Store::new(inner);
+
+                    let user_id = user_id();
+                    let invited_user_id = invited_user_id();
+                    let room_id = room_id();
+                    let stripped_room_id = stripped_room_id();
+                    let device_id = device_id!("device");
+
+                    let session = Session {
+                        access_token: "token".to_owned(),
+                        user_id: user_id.to_owned(),
+                        device_id: device_id.to_owned(),
+                    };
+                    store.restore_session(session).await.unwrap();
+
+                    changes.sync_token = Some("t392-516_47314_0_7_1_1_1_11444_1".to_owned());
+
+                    let presence_json: &JsonValue = &test_json::PRESENCE;
+                    let presence_raw =
+                        serde_json::from_value::<Raw<PresenceEvent>>(presence_json.clone()).unwrap();
+                    let presence_event = presence_raw.deserialize().unwrap();
+                    changes.add_presence_event(presence_event, presence_raw);
+
+                    let pushrules_json: &JsonValue = &test_json::PUSH_RULES;
+                    let pushrules_raw =
+                        serde_json::from_value::<Raw<AnyGlobalAccountDataEvent>>(pushrules_json.clone())
+                            .unwrap();
+                    let pushrules_event = pushrules_raw.deserialize().unwrap();
+                    changes.add_account_data(pushrules_event, pushrules_raw);
+
+                    let mut room = store.get_or_create_room(room_id, RoomType::Joined).await.clone_info();
+                    room.mark_as_left();
+
+                    let tag_json: &JsonValue = &test_json::TAG;
+                    let tag_raw =
+                        serde_json::from_value::<Raw<AnyRoomAccountDataEvent>>(tag_json.clone()).unwrap();
+                    let tag_event = tag_raw.deserialize().unwrap();
+                    changes.add_room_account_data(room_id, tag_event, tag_raw);
+
+                    let name_json: &JsonValue = &test_json::NAME;
+                    let name_raw = serde_json::from_value::<Raw<AnySyncStateEvent>>(name_json.clone()).unwrap();
+                    let name_event = name_raw.deserialize().unwrap();
+                    room.handle_state_event(&name_event);
+                    changes.add_state_event(room_id, name_event, name_raw);
+
+                    let topic_json: &JsonValue = &test_json::TOPIC;
+                    let topic_raw =
+                        serde_json::from_value::<Raw<AnySyncStateEvent>>(topic_json.clone()).unwrap();
+                    let topic_event = topic_raw.deserialize().unwrap();
+                    room.handle_state_event(&topic_event);
+                    changes.add_state_event(room_id, topic_event, topic_raw);
+
+                    let mut room_ambiguity_map = BTreeMap::new();
+                    let mut room_profiles = BTreeMap::new();
+                    let mut room_members = BTreeMap::new();
+
+                    let member_json: &JsonValue = &test_json::MEMBER;
+                    let member_event: SyncRoomMemberEvent =
+                        serde_json::from_value(member_json.clone()).unwrap();
+                    let displayname =
+                        member_event.as_original().unwrap().content.displayname.clone().unwrap();
+                    room_ambiguity_map.insert(
+                        displayname.clone(),
+                        BTreeSet::from([user_id.to_owned()]),
+                    );
+                    room_profiles.insert(user_id.to_owned(), (&member_event).into());
+                    room_members.insert(user_id.to_owned(), member_event);
+
+                    let member_state_raw =
+                        serde_json::from_value::<Raw<AnySyncStateEvent>>(member_json.clone()).unwrap();
+                    let member_state_event = member_state_raw.deserialize().unwrap();
+                    changes.add_state_event(room_id, member_state_event, member_state_raw);
+
+                    let invited_member_json: &JsonValue = &test_json::MEMBER_INVITE;
+                    // FIXME: Should be stripped room member event
+                    let invited_member_event: SyncRoomMemberEvent =
+                        serde_json::from_value(invited_member_json.clone()).unwrap();
+                    room_ambiguity_map
+                        .entry(displayname)
+                        .or_default()
+                        .insert(invited_user_id.to_owned());
+                    room_profiles.insert(invited_user_id.to_owned(), (&invited_member_event).into());
+                    room_members.insert(invited_user_id.to_owned(), invited_member_event);
+
+                    let invited_member_state_raw =
+                        serde_json::from_value::<Raw<AnySyncStateEvent>>(invited_member_json.clone()).unwrap();
+                    let invited_member_state_event = invited_member_state_raw.deserialize().unwrap();
+                    changes.add_state_event(room_id, invited_member_state_event, invited_member_state_raw);
+
+                    let receipt_json: &JsonValue = &test_json::READ_RECEIPT;
+                    let receipt_event =
+                        serde_json::from_value::<AnySyncEphemeralRoomEvent>(receipt_json.clone()).unwrap();
+                    let receipt_content = match receipt_event.content() {
+                        AnyEphemeralRoomEventContent::Receipt(content) => content,
+                        _ => panic!(),
+                    };
+                    changes.add_receipts(room_id, receipt_content);
+
+                    changes.ambiguity_maps.insert(room_id.to_owned(), room_ambiguity_map);
+                    changes.profiles.insert(room_id.to_owned(), room_profiles);
+                    changes.members.insert(room_id.to_owned(), room_members);
+                    changes.add_room(room);
+
+                    let mut stripped_room =
+                        store.get_or_create_stripped_room(stripped_room_id).await.clone_info();
+
+                    let stripped_name_json: &JsonValue = &test_json::NAME_STRIPPED;
+                    let stripped_name_raw =
+                        serde_json::from_value::<Raw<AnyStrippedStateEvent>>(stripped_name_json.clone())
+                            .unwrap();
+                    let stripped_name_event = stripped_name_raw.deserialize().unwrap();
+                    stripped_room.handle_stripped_state_event(&stripped_name_event);
+                    changes.stripped_state.insert(
+                        stripped_room_id.to_owned(),
+                        BTreeMap::from([(
+                            stripped_name_event.event_type(),
+                            BTreeMap::from([(
+                                stripped_name_event.state_key().to_owned(),
+                                stripped_name_raw.clone(),
+                            )]),
+                        )]),
+                    );
+
+                    changes.add_stripped_room(stripped_room);
+
+                    let stripped_member_json: &JsonValue = &test_json::MEMBER_STRIPPED;
+                    let stripped_member_event = serde_json::from_value::<StrippedRoomMemberEvent>(
+                        stripped_member_json.clone(),
+                    ).unwrap();
+                    changes.add_stripped_member(stripped_room_id, stripped_member_event);
+
+                    store.save_changes(&changes).await?;
+                    Ok(store)
+                }
+
+                fn power_level_event() -> Raw<AnySyncStateEvent> {
+                    let content = RoomPowerLevelsEventContent::default();
+
+                    let event = json!({
+                        "event_id": "$h29iv0s8:example.com",
+                        "content": content,
+                        "sender": user_id(),
+                        "type": "m.room.power_levels",
+                        "origin_server_ts": 0u64,
+                        "state_key": "",
+                    });
+
+                    serde_json::from_value(event).unwrap()
+                }
+
+                fn stripped_membership_event() -> StrippedRoomMemberEvent {
+                    custom_stripped_membership_event(user_id())
+                }
+
+                fn custom_stripped_membership_event(
+                    user_id: &UserId,
+                ) -> StrippedRoomMemberEvent {
+                    StrippedRoomMemberEvent {
+                        content: RoomMemberEventContent::new(MembershipState::Join),
+                        sender: user_id.to_owned(),
+                        state_key: user_id.to_owned(),
+                    }
+                }
+
+                fn membership_event() -> SyncRoomMemberEvent {
+                    custom_membership_event(user_id(), event_id!("$h29iv0s8:example.com").to_owned())
+                }
+
+                fn custom_membership_event(
+                    user_id: &UserId,
+                    event_id: OwnedEventId,
+                ) -> SyncRoomMemberEvent {
+                    SyncRoomMemberEvent::Original(OriginalSyncRoomMemberEvent {
+                        event_id,
+                        content: RoomMemberEventContent::new(MembershipState::Join),
+                        sender: user_id.to_owned(),
+                        origin_server_ts: MilliSecondsSinceUnixEpoch(198u32.into()),
+                        state_key: user_id.to_owned(),
+                        unsigned: StateUnsigned::default(),
+                    })
+                }
+
+                #[async_test]
+                async fn test_populate_store() -> StoreResult<()> {
+                    let room_id = room_id();
+                    let user_id = user_id();
+                    let inner_store = get_store().await?;
+
+                    let store = populated_store(Box::new(inner_store)).await?;
+
+                    assert!(store.get_sync_token().await?.is_some());
+                    assert!(store.get_presence_event(user_id).await?.is_some());
+                    assert_eq!(store.get_room_infos().await?.len(), 1, "Expected to find 1 room info ");
+                    assert_eq!(store.get_stripped_room_infos().await?.len(), 1, "Expected to find 1 stripped room info");
+                    assert!(store.get_account_data_event(GlobalAccountDataEventType::PushRules).await?.is_some());
+
+                    assert!(store.get_state_event(room_id, StateEventType::RoomName, "").await?.is_some());
+                    assert_eq!(store.get_state_events(room_id, StateEventType::RoomTopic).await?.len(), 1, "Expected to find 1 room topic");
+                    assert!(store.get_profile(room_id, user_id).await?.is_some());
+                    assert!(store.get_member_event(room_id, user_id).await?.is_some());
+                    assert_eq!(store.get_user_ids(room_id).await?.len(), 2, "Expected to find 2 members for room");
+                    assert_eq!(store.get_invited_user_ids(room_id).await?.len(), 1, "Expected to find 1 invited user ids");
+                    assert_eq!(store.get_joined_user_ids(room_id).await?.len(), 1, "Expected to find 1 joined user ids");
+                    assert_eq!(store.get_users_with_display_name(room_id, "example").await?.len(), 2, "Expected to find 2 display names for room");
+                    assert!(store
+                        .get_room_account_data_event(room_id, RoomAccountDataEventType::Tag)
+                        .await?
+                        .is_some());
+                    assert!(store
+                        .get_user_room_receipt_event(room_id, ReceiptType::Read, user_id)
+                        .await?
+                        .is_some());
+                    assert_eq!(
+                        store
+                            .get_event_room_receipt_events(room_id, ReceiptType::Read, first_receipt_event_id())
+                            .await?
+                            .len(),
+                        1, "Expected to find 1 read receipt");
+                    Ok(())
+                }
+
+                #[async_test]
+                async fn test_member_saving() {
+                    let store = get_store().await.unwrap();
+                    let room_id = room_id!("!test_member_saving:localhost");
+                    let user_id = user_id();
+
+                    assert!(store.get_member_event(room_id, user_id).await.unwrap().is_none());
+                    let mut changes = StateChanges::default();
+                    changes
+                        .members
+                        .entry(room_id.to_owned())
+                        .or_default()
+                        .insert(user_id.to_owned(), membership_event());
+
+                    store.save_changes(&changes).await.unwrap();
+                    assert!(store.get_member_event(room_id, user_id).await.unwrap().is_some());
+
+                    let members = store.get_user_ids(room_id).await.unwrap();
+                    assert!(!members.is_empty(), "We expected to find members for the room")
+                }
+
+                #[async_test]
+                async fn test_stripped_member_saving() {
+                    let store = get_store().await.unwrap();
+                    let room_id = room_id!("!test_stripped_member_saving:localhost");
+                    let user_id = user_id();
+
+                    assert!(store.get_member_event(room_id, user_id).await.unwrap().is_none());
+                    let mut changes = StateChanges::default();
+                    changes
+                        .stripped_members
+                        .entry(room_id.to_owned())
+                        .or_default()
+                        .insert(user_id.to_owned(), stripped_membership_event());
+
+                    store.save_changes(&changes).await.unwrap();
+                    assert!(store.get_member_event(room_id, user_id).await.unwrap().is_some());
+
+                    let members = store.get_user_ids(room_id).await.unwrap();
+                    assert!(!members.is_empty(), "We expected to find members for the room")
+                }
+
+                #[async_test]
+                async fn test_power_level_saving() {
+                    let store = get_store().await.unwrap();
+                    let room_id = room_id!("!test_power_level_saving:localhost");
+
+                    let raw_event = power_level_event();
+                    let event = raw_event.deserialize().unwrap();
+
+                    assert!(store
+                        .get_state_event(room_id, StateEventType::RoomPowerLevels, "")
+                        .await
+                        .unwrap()
+                        .is_none());
+                    let mut changes = StateChanges::default();
+                    changes.add_state_event(room_id, event, raw_event);
+
+                    store.save_changes(&changes).await.unwrap();
+                    assert!(store
+                        .get_state_event(room_id, StateEventType::RoomPowerLevels, "")
+                        .await
+                        .unwrap()
+                        .is_some());
+                }
+
+                #[async_test]
+                async fn test_receipts_saving() {
+                    let store = get_store().await.expect("creating store failed");
+
+                    let room_id = room_id!("!test_receipts_saving:localhost");
+
+                    let first_event_id = event_id!("$1435641916114394fHBLK:matrix.org");
+                    let second_event_id = event_id!("$fHBLK1435641916114394:matrix.org");
+
+                    let first_receipt_event = serde_json::from_value(json!({
+                        first_event_id: {
+                            "m.read": {
+                                user_id(): {
+                                    "ts": 1436451550453u64
+                                }
+                            }
+                        }
+                    }))
+                    .expect("json creation failed");
+
+                    let second_receipt_event = serde_json::from_value(json!({
+                        second_event_id: {
+                            "m.read": {
+                                user_id(): {
+                                    "ts": 1436451551453u64
+                                }
+                            }
+                        }
+                    }))
+                    .expect("json creation failed");
+
+                    assert!(store
+                        .get_user_room_receipt_event(room_id, ReceiptType::Read, user_id())
+                        .await
+                        .expect("failed to read user room receipt")
+                        .is_none());
+                    assert!(store
+                        .get_event_room_receipt_events(room_id, ReceiptType::Read, &first_event_id)
+                        .await
+                        .expect("failed to read user room receipt for 1")
+                        .is_empty());
+                    assert!(store
+                        .get_event_room_receipt_events(room_id, ReceiptType::Read, &second_event_id)
+                        .await
+                        .expect("failed to read user room receipt for 2")
+                        .is_empty());
+
+                    let mut changes = StateChanges::default();
+                    changes.add_receipts(room_id, first_receipt_event);
+
+                    store.save_changes(&changes).await.expect("writing changes fauked");
+                    assert!(store
+                        .get_user_room_receipt_event(room_id, ReceiptType::Read, user_id())
+                        .await
+                        .expect("failed to read user room receipt after save")
+                        .is_some());
+                    assert_eq!(
+                        store
+                            .get_event_room_receipt_events(room_id, ReceiptType::Read, &first_event_id)
+                            .await
+                            .expect("failed to read user room receipt for 1 after save")
+                            .len(),
+                        1,
+                        "Found a wrong number of receipts for 1 after save"
+                    );
+                    assert!(store
+                        .get_event_room_receipt_events(room_id, ReceiptType::Read, &second_event_id)
+                        .await
+                        .expect("failed to read user room receipt for 2 after save")
+                        .is_empty());
+
+                    let mut changes = StateChanges::default();
+                    changes.add_receipts(room_id, second_receipt_event);
+
+                    store.save_changes(&changes).await.expect("Saving works");
+                    assert!(store
+                        .get_user_room_receipt_event(room_id, ReceiptType::Read, user_id())
+                        .await
+                        .expect("Getting user room receipts failed")
+                        .is_some());
+                    assert!(store
+                        .get_event_room_receipt_events(room_id, ReceiptType::Read, &first_event_id)
+                        .await
+                        .expect("Getting event room receipt events for first event failed")
+                        .is_empty());
+                    assert_eq!(
+                        store
+                            .get_event_room_receipt_events(room_id, ReceiptType::Read, &second_event_id)
+                            .await
+                            .expect("Getting event room receipt events for second event failed")
+                            .len(),
+                        1,
+                        "Found a wrong number of receipts for second event after save"
+                    );
+                }
+
+                #[async_test]
+                async fn test_media_content() {
+                    let store = get_store().await.unwrap();
+
+                    let uri = mxc_uri!("mxc://localhost/media");
+                    let content: Vec<u8> = "somebinarydata".into();
+
+                    let request_file = MediaRequest {
+                        source: MediaSource::Plain(uri.to_owned()),
+                        format: MediaFormat::File,
+                    };
+
+                    let request_thumbnail = MediaRequest {
+                        source: MediaSource::Plain(uri.to_owned()),
+                        format: MediaFormat::Thumbnail(MediaThumbnailSize {
+                            method: Method::Crop,
+                            width: uint!(100),
+                            height: uint!(100),
+                        }),
+                    };
+
+                    assert!(store.get_media_content(&request_file).await.unwrap().is_none(), "unexpected media found");
+                    assert!(store.get_media_content(&request_thumbnail).await.unwrap().is_none(), "media not found");
+
+                    store.add_media_content(&request_file, content.clone()).await.expect("adding media failed");
+                    assert!(store.get_media_content(&request_file).await.unwrap().is_some(), "media not found though added");
+
+                    store.remove_media_content(&request_file).await.expect("removing media failed");
+                    assert!(store.get_media_content(&request_file).await.unwrap().is_none(), "media still there after removing");
+
+                    store.add_media_content(&request_file, content.clone()).await.expect("adding media again failed");
+                    assert!(store.get_media_content(&request_file).await.unwrap().is_some(), "media not found after adding again");
+
+                    store.add_media_content(&request_thumbnail, content.clone()).await.expect("adding thumbnail failed");
+                    assert!(store.get_media_content(&request_thumbnail).await.unwrap().is_some(), "thumbnail not found");
+
+                    store.remove_media_content_for_uri(uri).await.expect("removing all media for uri failed");
+                    assert!(store.get_media_content(&request_file).await.unwrap().is_none(), "media wasn't removed");
+                    assert!(store.get_media_content(&request_thumbnail).await.unwrap().is_none(), "thumbnail wasn't removed");
+                }
+
+                #[async_test]
+                async fn test_custom_storage() -> StoreResult<()> {
+                    let key = "my_key";
+                    let value = &[0, 1, 2, 3];
+                    let store = get_store().await?;
+
+                    store.set_custom_value(key.as_bytes(), value.to_vec()).await?;
+
+                    let read = store.get_custom_value(key.as_bytes()).await?;
+
+                    assert_eq!(Some(value.as_ref()), read.as_deref());
+
+                    Ok(())
+                }
+
+                #[async_test]
+                async fn test_persist_invited_room() -> StoreResult<()> {
+                    let stripped_room_id = stripped_room_id();
+                    let inner_store = get_store().await?;
+                    let store = populated_store(Box::new(inner_store)).await?;
+
+                    assert_eq!(store.get_stripped_room_infos().await?.len(), 1);
+                    assert!(store.get_stripped_room(stripped_room_id).is_some());
+
+                    // populate rooom
+                    Ok(())
+                }
+
+                #[async_test]
+                async fn test_room_removal() -> StoreResult<()>  {
+                    let room_id = room_id();
+                    let user_id = user_id();
+                    let inner_store = get_store().await?;
+                    let stripped_room_id = stripped_room_id();
+
+                    let store = populated_store(Box::new(inner_store)).await?;
+
+                    store.remove_room(room_id).await?;
+
+                    assert!(store.get_room_infos().await?.is_empty(), "room is still there");
+                    assert_eq!(store.get_stripped_room_infos().await?.len(), 1);
+
+                    assert!(store.get_state_event(room_id, StateEventType::RoomName, "").await?.is_none());
+                    assert!(store.get_state_events(room_id, StateEventType::RoomTopic).await?.is_empty(), "still state events found");
+                    assert!(store.get_profile(room_id, user_id).await?.is_none());
+                    assert!(store.get_member_event(room_id, user_id).await?.is_none());
+                    assert!(store.get_user_ids(room_id).await?.is_empty(), "still user ids found");
+                    assert!(store.get_invited_user_ids(room_id).await?.is_empty(), "still invited user ids found");
+                    assert!(store.get_joined_user_ids(room_id).await?.is_empty(), "still joined users found");
+                    assert!(store.get_users_with_display_name(room_id, "example").await?.is_empty(), "still display names found");
+                    assert!(store
+                        .get_room_account_data_event(room_id, RoomAccountDataEventType::Tag)
+                        .await?
+                        .is_none());
+                    assert!(store
+                        .get_user_room_receipt_event(room_id, ReceiptType::Read, user_id)
+                        .await?
+                        .is_none());
+                    assert!(
+                        store
+                            .get_event_room_receipt_events(room_id, ReceiptType::Read, first_receipt_event_id())
+                            .await?
+                            .is_empty(),
+                        "still event recepts in the store"
+                    );
+
+                    store.remove_room(stripped_room_id).await?;
+
+                    assert!(store.get_room_infos().await?.is_empty(), "still room info found");
+                    assert!(store.get_stripped_room_infos().await?.is_empty(), "still stripped room info found");
+                    Ok(())
+                }
+
+                #[async_test]
+                #[cfg(feature = "experimental-timeline")]
+                async fn test_room_timeline() {
+                    let store = get_store().await.unwrap();
+                    let mut stored_events = Vec::new();
+                    let room_id = room_id!("!SVkFJHzfwvuaIEawgC:localhost");
+
+                    // Before the first sync the timeline should be empty
+                    assert!(store.room_timeline(room_id).await.expect("failed to read timeline").is_none(), "TL wasn't empty");
+
+                    // Add sync response
+                    let sync = SyncResponse::try_from_http_response(
+                        Response::builder().body(serde_json::to_vec(&*test_json::MORE_SYNC).unwrap()).unwrap(),
+                        )
+                        .unwrap();
+
+                    let timeline = &sync.rooms.join[room_id].timeline;
+                    let events: Vec<SyncRoomEvent> = timeline.events.iter().cloned().map(Into::into).collect();
+
+                    stored_events.extend(events.iter().rev().cloned());
+
+                    let timeline_slice = TimelineSlice::new(
+                        events,
+                        sync.next_batch.clone(),
+                        timeline.prev_batch.clone(),
+                        false,
+                        true,
+                        );
+                    let mut changes = StateChanges::new(sync.next_batch.clone());
+                    changes.add_timeline(room_id, timeline_slice);
+                    store.save_changes(&changes).await.unwrap();
+
+                    check_timeline_events(room_id, &store, &stored_events, timeline.prev_batch.as_deref())
+                        .await;
+
+                    // Add message response
+                    let messages = MessageResponse::try_from_http_response(
+                        Response::builder()
+                        .body(serde_json::to_vec(&*test_json::SYNC_ROOM_MESSAGES_BATCH_1).unwrap())
+                        .unwrap(),
+                        )
+                        .unwrap();
+
+                    let events: Vec<SyncRoomEvent> = messages
+                        .chunk
+                        .iter()
+                        .cloned()
+                        .map(|event| RoomEvent { event, encryption_info: None }.into())
+                        .collect();
+
+                    stored_events.append(&mut events.clone());
+
+                    let timeline_slice =
+                        TimelineSlice::new(events, messages.start.clone(), messages.end.clone(), false, false);
+                    let mut changes = StateChanges::default();
+                    changes.add_timeline(room_id, timeline_slice);
+                    store.save_changes(&changes).await.unwrap();
+
+                    check_timeline_events(room_id, &store, &stored_events, messages.end.as_deref()).await;
+
+                    // Add second message response
+                    let messages = MessageResponse::try_from_http_response(
+                        Response::builder()
+                        .body(serde_json::to_vec(&*test_json::SYNC_ROOM_MESSAGES_BATCH_2).unwrap())
+                        .unwrap(),
+                        )
+                        .unwrap();
+
+                    let events: Vec<SyncRoomEvent> = messages
+                        .chunk
+                        .iter()
+                        .cloned()
+                        .map(|event| RoomEvent { event, encryption_info: None }.into())
+                        .collect();
+
+                    stored_events.append(&mut events.clone());
+
+                    let timeline_slice =
+                        TimelineSlice::new(events, messages.start.clone(), messages.end.clone(), false, false);
+                    let mut changes = StateChanges::default();
+                    changes.add_timeline(room_id, timeline_slice);
+                    store.save_changes(&changes).await.unwrap();
+
+                    check_timeline_events(room_id, &store, &stored_events, messages.end.as_deref()).await;
+
+                    // Add second sync response
+                    let sync = SyncResponse::try_from_http_response(
+                        Response::builder()
+                        .body(serde_json::to_vec(&*test_json::MORE_SYNC_2).unwrap())
+                        .unwrap(),
+                        )
+                        .unwrap();
+
+                    let timeline = &sync.rooms.join[room_id].timeline;
+                    let events: Vec<SyncRoomEvent> = timeline.events.iter().cloned().map(Into::into).collect();
+
+                    let prev_stored_events = stored_events;
+                    stored_events = events.iter().rev().cloned().collect();
+                    stored_events.extend(prev_stored_events);
+
+                    let timeline_slice = TimelineSlice::new(
+                        events,
+                        sync.next_batch.clone(),
+                        timeline.prev_batch.clone(),
+                        false,
+                        true,
+                        );
+                    let mut changes = StateChanges::new(sync.next_batch.clone());
+                    changes.add_timeline(room_id, timeline_slice);
+                    store.save_changes(&changes).await.unwrap();
+
+                    check_timeline_events(room_id, &store, &stored_events, messages.end.as_deref()).await;
+
+                    // Check if limited sync removes the stored timeline
+                    let end_token = Some("end token".to_owned());
+                    let timeline_slice = TimelineSlice::new(
+                        Vec::new(),
+                        "start token".to_owned(),
+                        end_token.clone(),
+                        true,
+                        true,
+                        );
+                    let mut changes = StateChanges::default();
+                    changes.add_timeline(room_id, timeline_slice);
+                    store.save_changes(&changes).await.unwrap();
+
+                    check_timeline_events(room_id, &store, &Vec::new(), end_token.as_deref()).await;
+                }
+
+                #[cfg(feature = "experimental-timeline")]
+                async fn check_timeline_events(
+                    room_id: &RoomId,
+                    store: &dyn StateStore,
+                    stored_events: &[SyncRoomEvent],
+                    expected_end_token: Option<&str>,
+                    ) {
+                    let (timeline_iter, end_token) = store.room_timeline(room_id).await.unwrap().unwrap();
+
+                    assert_eq!(end_token.as_deref(), expected_end_token);
+
+                    let timeline = timeline_iter.collect::<Vec<StoreResult<SyncRoomEvent>>>().await;
+
+                    let expected: Vec<OwnedEventId> = stored_events.iter().map(|a| a.event_id().expect("event id doesn't exist")).collect();
+                    let found: Vec<OwnedEventId> = timeline.iter().map(|a| a.as_ref().expect("object missing").event_id().clone().expect("event id missing")).collect();
+
+                    for (idx, (a, b)) in timeline
+                        .into_iter()
+                        .zip(stored_events.iter())
+                        .enumerate()
+                    {
+                        assert_eq!(a.expect("not a value").event_id(), b.event_id(), "pos {} not equal - expected: {:#?}, but found {:#?}", idx, expected, found);
+
+                    }
+                }
+
+            }
+
+        )*
+    }
+}

crates/matrix-sdk-base/src/store/memory_store.rs

@@ -12,65 +12,97 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#[cfg(feature = "experimental-timeline")]
+use std::collections::{BTreeMap, HashMap};
 use std::{
     collections::BTreeSet,
     sync::{Arc, RwLock},
 };
 
+#[cfg(feature = "experimental-timeline")]
+use async_stream::stream;
+use async_trait::async_trait;
 use dashmap::{DashMap, DashSet};
 use lru::LruCache;
-use matrix_sdk_common::{async_trait, instant::Instant, locks::Mutex};
+#[allow(unused_imports)]
+use matrix_sdk_common::{instant::Instant, locks::Mutex};
 use ruma::{
     events::{
         presence::PresenceEvent,
         receipt::Receipt,
-        room::member::{MemberEventContent, MembershipState},
+        room::member::{MembershipState, StrippedRoomMemberEvent, SyncRoomMemberEvent},
         AnyGlobalAccountDataEvent, AnyRoomAccountDataEvent, AnyStrippedStateEvent,
-        AnySyncStateEvent, EventType,
+        AnySyncStateEvent, GlobalAccountDataEventType, RoomAccountDataEventType, StateEventType,
     },
     receipt::ReceiptType,
     serde::Raw,
-    EventId, MxcUri, RoomId, UserId,
+    EventId, MxcUri, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UserId,
+};
+#[cfg(feature = "experimental-timeline")]
+use ruma::{
+    events::{room::redaction::SyncRoomRedactionEvent, AnySyncMessageLikeEvent, AnySyncRoomEvent},
+    signatures::{redact_in_place, CanonicalJsonObject},
+    RoomVersionId,
 };
-use tracing::info;
 
+#[cfg(feature = "experimental-timeline")]
+use super::BoxStream;
 use super::{Result, RoomInfo, StateChanges, StateStore};
 use crate::{
-    deserialized_responses::{MemberEvent, StrippedMemberEvent},
+    deserialized_responses::MemberEvent,
     media::{MediaRequest, UniqueKey},
+    MinimalRoomMemberEvent,
 };
+#[cfg(feature = "experimental-timeline")]
+use crate::{deserialized_responses::SyncRoomEvent, StoreError};
 
+/// In-Memory, non-persistent implementation of the `StateStore`
+///
+/// Default if no other is configured at startup.
+#[allow(clippy::type_complexity)]
 #[derive(Debug, Clone)]
 pub struct MemoryStore {
     sync_token: Arc<RwLock<Option<String>>>,
     filters: Arc<DashMap<String, String>>,
-    account_data: Arc<DashMap<String, Raw<AnyGlobalAccountDataEvent>>>,
-    members: Arc<DashMap<RoomId, DashMap<UserId, MemberEvent>>>,
-    profiles: Arc<DashMap<RoomId, DashMap<UserId, MemberEventContent>>>,
-    display_names: Arc<DashMap<RoomId, DashMap<String, BTreeSet<UserId>>>>,
-    joined_user_ids: Arc<DashMap<RoomId, DashSet<UserId>>>,
-    invited_user_ids: Arc<DashMap<RoomId, DashSet<UserId>>>,
-    room_info: Arc<DashMap<RoomId, RoomInfo>>,
-    #[allow(clippy::type_complexity)]
-    room_state: Arc<DashMap<RoomId, DashMap<String, DashMap<String, Raw<AnySyncStateEvent>>>>>,
-    room_account_data: Arc<DashMap<RoomId, DashMap<String, Raw<AnyRoomAccountDataEvent>>>>,
-    stripped_room_info: Arc<DashMap<RoomId, RoomInfo>>,
-    #[allow(clippy::type_complexity)]
-    stripped_room_state:
-        Arc<DashMap<RoomId, DashMap<String, DashMap<String, Raw<AnyStrippedStateEvent>>>>>,
-    stripped_members: Arc<DashMap<RoomId, DashMap<UserId, StrippedMemberEvent>>>,
-    presence: Arc<DashMap<UserId, Raw<PresenceEvent>>>,
-    #[allow(clippy::type_complexity)]
-    room_user_receipts: Arc<DashMap<RoomId, DashMap<String, DashMap<UserId, (EventId, Receipt)>>>>,
-    #[allow(clippy::type_complexity)]
-    room_event_receipts:
-        Arc<DashMap<RoomId, DashMap<String, DashMap<EventId, DashMap<UserId, Receipt>>>>>,
+    account_data: Arc<DashMap<GlobalAccountDataEventType, Raw<AnyGlobalAccountDataEvent>>>,
+    members: Arc<DashMap<OwnedRoomId, DashMap<OwnedUserId, SyncRoomMemberEvent>>>,
+    profiles: Arc<DashMap<OwnedRoomId, DashMap<OwnedUserId, MinimalRoomMemberEvent>>>,
+    display_names: Arc<DashMap<OwnedRoomId, DashMap<String, BTreeSet<OwnedUserId>>>>,
+    joined_user_ids: Arc<DashMap<OwnedRoomId, DashSet<OwnedUserId>>>,
+    invited_user_ids: Arc<DashMap<OwnedRoomId, DashSet<OwnedUserId>>>,
+    room_info: Arc<DashMap<OwnedRoomId, RoomInfo>>,
+    room_state:
+        Arc<DashMap<OwnedRoomId, DashMap<StateEventType, DashMap<String, Raw<AnySyncStateEvent>>>>>,
+    room_account_data:
+        Arc<DashMap<OwnedRoomId, DashMap<RoomAccountDataEventType, Raw<AnyRoomAccountDataEvent>>>>,
+    stripped_room_infos: Arc<DashMap<OwnedRoomId, RoomInfo>>,
+    stripped_room_state: Arc<
+        DashMap<OwnedRoomId, DashMap<StateEventType, DashMap<String, Raw<AnyStrippedStateEvent>>>>,
+    >,
+    stripped_members: Arc<DashMap<OwnedRoomId, DashMap<OwnedUserId, StrippedRoomMemberEvent>>>,
+    stripped_joined_user_ids: Arc<DashMap<OwnedRoomId, DashSet<OwnedUserId>>>,
+    stripped_invited_user_ids: Arc<DashMap<OwnedRoomId, DashSet<OwnedUserId>>>,
+    presence: Arc<DashMap<OwnedUserId, Raw<PresenceEvent>>>,
+    room_user_receipts:
+        Arc<DashMap<OwnedRoomId, DashMap<String, DashMap<OwnedUserId, (OwnedEventId, Receipt)>>>>,
+    room_event_receipts: Arc<
+        DashMap<OwnedRoomId, DashMap<String, DashMap<OwnedEventId, DashMap<OwnedUserId, Receipt>>>>,
+    >,
     media: Arc<Mutex<LruCache<String, Vec<u8>>>>,
     custom: Arc<DashMap<Vec<u8>, Vec<u8>>>,
+    #[cfg(feature = "experimental-timeline")]
+    room_timeline: Arc<DashMap<OwnedRoomId, TimelineData>>,
+}
+
+impl Default for MemoryStore {
+    fn default() -> Self {
+        Self::new()
+    }
 }
 
 impl MemoryStore {
     #[allow(dead_code)]
+    /// Create a new empty MemoryStore
     pub fn new() -> Self {
         Self {
             sync_token: Default::default(),
@@ -84,19 +116,23 @@ impl MemoryStore {
             room_info: Default::default(),
             room_state: Default::default(),
             room_account_data: Default::default(),
-            stripped_room_info: Default::default(),
+            stripped_room_infos: Default::default(),
             stripped_room_state: Default::default(),
             stripped_members: Default::default(),
+            stripped_joined_user_ids: Default::default(),
+            stripped_invited_user_ids: Default::default(),
             presence: Default::default(),
             room_user_receipts: Default::default(),
             room_event_receipts: Default::default(),
             media: Arc::new(Mutex::new(LruCache::new(100))),
             custom: DashMap::new().into(),
+            #[cfg(feature = "experimental-timeline")]
+            room_timeline: Default::default(),
         }
     }
 
     async fn save_filter(&self, filter_name: &str, filter_id: &str) -> Result<()> {
-        self.filters.insert(filter_name.to_string(), filter_id.to_string());
+        self.filters.insert(filter_name.to_owned(), filter_id.to_owned());
 
         Ok(())
     }
@@ -118,43 +154,43 @@ impl MemoryStore {
 
         for (room, events) in &changes.members {
             for event in events.values() {
-                match event.content.membership {
+                match event.membership() {
                     MembershipState::Join => {
                         self.joined_user_ids
                             .entry(room.clone())
-                            .or_insert_with(DashSet::new)
-                            .insert(event.state_key.clone());
+                            .or_default()
+                            .insert(event.state_key().to_owned());
                         self.invited_user_ids
                             .entry(room.clone())
-                            .or_insert_with(DashSet::new)
-                            .remove(&event.state_key);
+                            .or_default()
+                            .remove(event.state_key());
                     }
                     MembershipState::Invite => {
                         self.invited_user_ids
                             .entry(room.clone())
-                            .or_insert_with(DashSet::new)
-                            .insert(event.state_key.clone());
+                            .or_default()
+                            .insert(event.state_key().to_owned());
                         self.joined_user_ids
                             .entry(room.clone())
-                            .or_insert_with(DashSet::new)
-                            .remove(&event.state_key);
+                            .or_default()
+                            .remove(event.state_key());
                     }
                     _ => {
                         self.joined_user_ids
                             .entry(room.clone())
-                            .or_insert_with(DashSet::new)
-                            .remove(&event.state_key);
+                            .or_default()
+                            .remove(event.state_key());
                         self.invited_user_ids
                             .entry(room.clone())
-                            .or_insert_with(DashSet::new)
-                            .remove(&event.state_key);
+                            .or_default()
+                            .remove(event.state_key());
                     }
                 }
 
                 self.members
                     .entry(room.clone())
-                    .or_insert_with(DashMap::new)
-                    .insert(event.state_key.clone(), event.clone());
+                    .or_default()
+                    .insert(event.state_key().to_owned(), event.clone());
             }
         }
 
@@ -162,7 +198,7 @@ impl MemoryStore {
             for (user_id, profile) in users {
                 self.profiles
                     .entry(room.clone())
-                    .or_insert_with(DashMap::new)
+                    .or_default()
                     .insert(user_id.clone(), profile.clone());
             }
         }
@@ -171,21 +207,21 @@ impl MemoryStore {
             for (display_name, display_names) in map {
                 self.display_names
                     .entry(room.clone())
-                    .or_insert_with(DashMap::new)
+                    .or_default()
                     .insert(display_name.clone(), display_names.clone());
             }
         }
 
         for (event_type, event) in &changes.account_data {
-            self.account_data.insert(event_type.to_string(), event.clone());
+            self.account_data.insert(event_type.clone(), event.clone());
         }
 
         for (room, events) in &changes.room_account_data {
             for (event_type, event) in events {
                 self.room_account_data
                     .entry(room.clone())
-                    .or_insert_with(DashMap::new)
-                    .insert(event_type.to_string(), event.clone());
+                    .or_default()
+                    .insert(event_type.clone(), event.clone());
             }
         }
 
@@ -194,9 +230,9 @@ impl MemoryStore {
                 for (state_key, event) in events {
                     self.room_state
                         .entry(room.clone())
-                        .or_insert_with(DashMap::new)
-                        .entry(event_type.to_owned())
-                        .or_insert_with(DashMap::new)
+                        .or_default()
+                        .entry(event_type.clone())
+                        .or_default()
                         .insert(state_key.to_owned(), event.clone());
                 }
             }
@@ -210,15 +246,48 @@ impl MemoryStore {
             self.presence.insert(sender.clone(), event.clone());
         }
 
-        for (room_id, info) in &changes.invited_room_info {
-            self.stripped_room_info.insert(room_id.clone(), info.clone());
+        for (room_id, info) in &changes.stripped_room_infos {
+            self.stripped_room_infos.insert(room_id.clone(), info.clone());
         }
 
         for (room, events) in &changes.stripped_members {
             for event in events.values() {
+                match event.content.membership {
+                    MembershipState::Join => {
+                        self.stripped_joined_user_ids
+                            .entry(room.clone())
+                            .or_insert_with(DashSet::new)
+                            .insert(event.state_key.clone());
+                        self.stripped_invited_user_ids
+                            .entry(room.clone())
+                            .or_insert_with(DashSet::new)
+                            .remove(&event.state_key);
+                    }
+                    MembershipState::Invite => {
+                        self.stripped_invited_user_ids
+                            .entry(room.clone())
+                            .or_insert_with(DashSet::new)
+                            .insert(event.state_key.clone());
+                        self.stripped_joined_user_ids
+                            .entry(room.clone())
+                            .or_insert_with(DashSet::new)
+                            .remove(&event.state_key);
+                    }
+                    _ => {
+                        self.stripped_joined_user_ids
+                            .entry(room.clone())
+                            .or_insert_with(DashSet::new)
+                            .remove(&event.state_key);
+                        self.stripped_invited_user_ids
+                            .entry(room.clone())
+                            .or_insert_with(DashSet::new)
+                            .remove(&event.state_key);
+                    }
+                }
+
                 self.stripped_members
                     .entry(room.clone())
-                    .or_insert_with(DashMap::new)
+                    .or_default()
                     .insert(event.state_key.clone(), event.clone());
             }
         }
@@ -228,9 +297,9 @@ impl MemoryStore {
                 for (state_key, event) in events {
                     self.stripped_room_state
                         .entry(room.clone())
-                        .or_insert_with(DashMap::new)
-                        .entry(event_type.to_owned())
-                        .or_insert_with(DashMap::new)
+                        .or_default()
+                        .entry(event_type.clone())
+                        .or_default()
                         .insert(state_key.to_owned(), event.clone());
                 }
             }
@@ -244,9 +313,9 @@ impl MemoryStore {
                         if let Some((old_event, _)) = self
                             .room_user_receipts
                             .entry(room.clone())
-                            .or_insert_with(DashMap::new)
+                            .or_default()
                             .entry(receipt_type.to_string())
-                            .or_insert_with(DashMap::new)
+                            .or_default()
                             .insert(user_id.clone(), (event_id.clone(), receipt.clone()))
                         {
                             // Remove the old receipt from the room event receipts
@@ -262,50 +331,162 @@ impl MemoryStore {
                         // Add the receipt to the room event receipts
                         self.room_event_receipts
                             .entry(room.clone())
-                            .or_insert_with(DashMap::new)
+                            .or_default()
                             .entry(receipt_type.to_string())
-                            .or_insert_with(DashMap::new)
+                            .or_default()
                             .entry(event_id.clone())
-                            .or_insert_with(DashMap::new)
+                            .or_default()
                             .insert(user_id.clone(), receipt.clone());
                     }
                 }
             }
         }
 
-        info!("Saved changes in {:?}", now.elapsed());
+        #[cfg(feature = "experimental-timeline")]
+        for (room, timeline) in &changes.timeline {
+            if timeline.sync {
+                tracing::info!("Save new timeline batch from sync response for {}", room);
+            } else {
+                tracing::info!("Save new timeline batch from messages response for {}", room);
+            }
+
+            let mut delete_timeline = false;
+            if timeline.limited {
+                tracing::info!(
+                    "Delete stored timeline for {} because the sync response was limited",
+                    room
+                );
+                delete_timeline = true;
+            } else if let Some(mut data) = self.room_timeline.get_mut(room) {
+                if !timeline.sync && Some(&timeline.start) != data.end.as_ref() {
+                    // This should only happen when a developer adds a wrong timeline
+                    // batch to the `StateChanges` or the server returns a wrong response
+                    // to our request.
+                    tracing::warn!("Drop unexpected timeline batch for {}", room);
+                    return Ok(());
+                }
+
+                // Check if the event already exists in the store
+                for event in &timeline.events {
+                    if let Some(event_id) = event.event_id() {
+                        if data.event_id_to_position.contains_key(&event_id) {
+                            delete_timeline = true;
+                            break;
+                        }
+                    }
+                }
+
+                if !delete_timeline {
+                    if timeline.sync {
+                        data.start = timeline.start.clone();
+                    } else {
+                        data.end = timeline.end.clone();
+                    }
+                }
+            }
+
+            if delete_timeline {
+                tracing::info!("Delete stored timeline for {} because of duplicated events", room);
+                self.room_timeline.remove(room);
+            }
+
+            let mut data =
+                self.room_timeline.entry(room.to_owned()).or_insert_with(|| TimelineData {
+                    start: timeline.start.clone(),
+                    end: timeline.end.clone(),
+                    ..Default::default()
+                });
+
+            let make_room_version = || {
+                self.room_info
+                    .get(room)
+                    .and_then(|info| info.room_version().cloned())
+                    .unwrap_or_else(|| {
+                        tracing::warn!(
+                            "Unable to find the room version for {}, assume version 9",
+                            room
+                        );
+                        RoomVersionId::V9
+                    })
+            };
+
+            if timeline.sync {
+                let mut room_version = None;
+                for event in &timeline.events {
+                    // Redact events already in store only on sync response
+                    if let Ok(AnySyncRoomEvent::MessageLike(
+                        AnySyncMessageLikeEvent::RoomRedaction(SyncRoomRedactionEvent::Original(
+                            redaction,
+                        )),
+                    )) = event.event.deserialize()
+                    {
+                        let pos = data.event_id_to_position.get(&redaction.redacts).copied();
+
+                        if let Some(position) = pos {
+                            if let Some(mut full_event) = data.events.get_mut(&position.clone()) {
+                                let mut event_json: CanonicalJsonObject =
+                                    full_event.event.deserialize_as()?;
+                                let v = room_version.get_or_insert_with(make_room_version);
+
+                                redact_in_place(&mut event_json, v)
+                                    .map_err(StoreError::Redaction)?;
+                                full_event.event = Raw::new(&event_json)?.cast();
+                            }
+                        }
+                    }
+
+                    data.start_position -= 1;
+                    let start_position = data.start_position;
+                    // Only add event with id to the position map
+                    if let Some(event_id) = event.event_id() {
+                        data.event_id_to_position.insert(event_id, start_position);
+                    }
+                    data.events.insert(start_position, event.clone());
+                }
+            } else {
+                for event in &timeline.events {
+                    data.end_position += 1;
+                    let end_position = data.end_position;
+                    // Only add event with id to the position map
+                    if let Some(event_id) = event.event_id() {
+                        data.event_id_to_position.insert(event_id, end_position);
+                    }
+                    data.events.insert(end_position, event.clone());
+                }
+            }
+        }
+
+        tracing::info!("Saved changes in {:?}", now.elapsed());
 
         Ok(())
     }
 
     async fn get_presence_event(&self, user_id: &UserId) -> Result<Option<Raw<PresenceEvent>>> {
-        #[allow(clippy::map_clone)]
         Ok(self.presence.get(user_id).map(|p| p.clone()))
     }
 
     async fn get_state_event(
         &self,
         room_id: &RoomId,
-        event_type: EventType,
+        event_type: StateEventType,
         state_key: &str,
     ) -> Result<Option<Raw<AnySyncStateEvent>>> {
-        #[allow(clippy::map_clone)]
-        Ok(self.room_state.get(room_id).and_then(|e| {
-            e.get(event_type.as_ref()).and_then(|s| s.get(state_key).map(|e| e.clone()))
-        }))
+        Ok(self
+            .room_state
+            .get(room_id)
+            .and_then(|e| e.get(&event_type).and_then(|s| s.get(state_key).map(|e| e.clone()))))
     }
 
     async fn get_state_events(
         &self,
         room_id: &RoomId,
-        event_type: EventType,
+        event_type: StateEventType,
     ) -> Result<Vec<Raw<AnySyncStateEvent>>> {
-        #[allow(clippy::map_clone)]
         Ok(self
             .room_state
             .get(room_id)
             .and_then(|e| {
-                e.get(event_type.as_ref()).map(|s| s.iter().map(|e| e.clone()).collect::<Vec<_>>())
+                e.get(&event_type).map(|s| s.iter().map(|e| e.clone()).collect::<Vec<_>>())
             })
             .unwrap_or_default())
     }
@@ -314,8 +495,7 @@ impl MemoryStore {
         &self,
         room_id: &RoomId,
         user_id: &UserId,
-    ) -> Result<Option<MemberEventContent>> {
-        #[allow(clippy::map_clone)]
+    ) -> Result<Option<MinimalRoomMemberEvent>> {
         Ok(self.profiles.get(room_id).and_then(|p| p.get(user_id).map(|p| p.clone())))
     }
 
@@ -324,60 +504,78 @@ impl MemoryStore {
         room_id: &RoomId,
         state_key: &UserId,
     ) -> Result<Option<MemberEvent>> {
-        #[allow(clippy::map_clone)]
-        Ok(self.members.get(room_id).and_then(|m| m.get(state_key).map(|m| m.clone())))
+        if let Some(e) = self.members.get(room_id).and_then(|m| m.get(state_key).map(|m| m.clone()))
+        {
+            Ok(Some(MemberEvent::Sync(e)))
+        } else if let Some(e) =
+            self.stripped_members.get(room_id).and_then(|m| m.get(state_key).map(|m| m.clone()))
+        {
+            Ok(Some(MemberEvent::Stripped(e)))
+        } else {
+            Ok(None)
+        }
     }
 
-    fn get_user_ids(&self, room_id: &RoomId) -> Vec<UserId> {
-        #[allow(clippy::map_clone)]
-        self.members
-            .get(room_id)
-            .map(|u| u.iter().map(|u| u.key().clone()).collect())
-            .unwrap_or_default()
+    fn get_user_ids(&self, room_id: &RoomId) -> Vec<OwnedUserId> {
+        if let Some(u) = self.members.get(room_id) {
+            u.iter().map(|u| u.key().clone()).collect()
+        } else {
+            self.stripped_members
+                .get(room_id)
+                .map(|u| u.iter().map(|u| u.key().clone()).collect())
+                .unwrap_or_default()
+        }
     }
 
-    fn get_invited_user_ids(&self, room_id: &RoomId) -> Vec<UserId> {
-        #[allow(clippy::map_clone)]
+    fn get_invited_user_ids(&self, room_id: &RoomId) -> Vec<OwnedUserId> {
         self.invited_user_ids
             .get(room_id)
             .map(|u| u.iter().map(|u| u.clone()).collect())
             .unwrap_or_default()
     }
 
-    fn get_joined_user_ids(&self, room_id: &RoomId) -> Vec<UserId> {
-        #[allow(clippy::map_clone)]
+    fn get_joined_user_ids(&self, room_id: &RoomId) -> Vec<OwnedUserId> {
         self.joined_user_ids
             .get(room_id)
             .map(|u| u.iter().map(|u| u.clone()).collect())
             .unwrap_or_default()
     }
 
+    fn get_stripped_invited_user_ids(&self, room_id: &RoomId) -> Vec<OwnedUserId> {
+        self.stripped_invited_user_ids
+            .get(room_id)
+            .map(|u| u.iter().map(|u| u.clone()).collect())
+            .unwrap_or_default()
+    }
+
+    fn get_stripped_joined_user_ids(&self, room_id: &RoomId) -> Vec<OwnedUserId> {
+        self.stripped_joined_user_ids
+            .get(room_id)
+            .map(|u| u.iter().map(|u| u.clone()).collect())
+            .unwrap_or_default()
+    }
+
     fn get_room_infos(&self) -> Vec<RoomInfo> {
-        #[allow(clippy::map_clone)]
         self.room_info.iter().map(|r| r.clone()).collect()
     }
 
     fn get_stripped_room_infos(&self) -> Vec<RoomInfo> {
-        #[allow(clippy::map_clone)]
-        self.stripped_room_info.iter().map(|r| r.clone()).collect()
+        self.stripped_room_infos.iter().map(|r| r.clone()).collect()
     }
 
     async fn get_account_data_event(
         &self,
-        event_type: EventType,
+        event_type: GlobalAccountDataEventType,
     ) -> Result<Option<Raw<AnyGlobalAccountDataEvent>>> {
-        Ok(self.account_data.get(event_type.as_ref()).map(|e| e.clone()))
+        Ok(self.account_data.get(&event_type).map(|e| e.clone()))
     }
 
     async fn get_room_account_data_event(
         &self,
         room_id: &RoomId,
-        event_type: EventType,
+        event_type: RoomAccountDataEventType,
     ) -> Result<Option<Raw<AnyRoomAccountDataEvent>>> {
-        Ok(self
-            .room_account_data
-            .get(room_id)
-            .and_then(|m| m.get(event_type.as_ref()).map(|e| e.clone())))
+        Ok(self.room_account_data.get(room_id).and_then(|m| m.get(&event_type).map(|e| e.clone())))
     }
 
     async fn get_user_room_receipt_event(
@@ -385,7 +583,7 @@ impl MemoryStore {
         room_id: &RoomId,
         receipt_type: ReceiptType,
         user_id: &UserId,
-    ) -> Result<Option<(EventId, Receipt)>> {
+    ) -> Result<Option<(OwnedEventId, Receipt)>> {
         Ok(self.room_user_receipts.get(room_id).and_then(|m| {
             m.get(receipt_type.as_ref()).and_then(|m| m.get(user_id).map(|r| r.clone()))
         }))
@@ -396,7 +594,7 @@ impl MemoryStore {
         room_id: &RoomId,
         receipt_type: ReceiptType,
         event_id: &EventId,
-    ) -> Result<Vec<(UserId, Receipt)>> {
+    ) -> Result<Vec<(OwnedUserId, Receipt)>> {
         Ok(self
             .room_event_receipts
             .get(room_id)
@@ -406,7 +604,7 @@ impl MemoryStore {
                         .map(|m| m.iter().map(|r| (r.key().clone(), r.value().clone())).collect())
                 })
             })
-            .unwrap_or_else(Vec::new))
+            .unwrap_or_default())
     }
 
     async fn get_custom_value(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
@@ -449,6 +647,54 @@ impl MemoryStore {
 
         Ok(())
     }
+
+    async fn remove_room(&self, room_id: &RoomId) -> Result<()> {
+        self.members.remove(room_id);
+        self.profiles.remove(room_id);
+        self.display_names.remove(room_id);
+        self.joined_user_ids.remove(room_id);
+        self.invited_user_ids.remove(room_id);
+        self.room_info.remove(room_id);
+        self.room_state.remove(room_id);
+        self.room_account_data.remove(room_id);
+        self.stripped_room_infos.remove(room_id);
+        self.stripped_room_state.remove(room_id);
+        self.stripped_members.remove(room_id);
+        self.room_user_receipts.remove(room_id);
+        self.room_event_receipts.remove(room_id);
+
+        #[cfg(feature = "experimental-timeline")]
+        self.room_timeline.remove(room_id);
+
+        Ok(())
+    }
+
+    #[cfg(feature = "experimental-timeline")]
+    async fn room_timeline(
+        &self,
+        room_id: &RoomId,
+    ) -> Result<Option<(BoxStream<Result<SyncRoomEvent>>, Option<String>)>> {
+        let (events, end_token) = if let Some(data) = self.room_timeline.get(room_id) {
+            (data.events.clone(), data.end.clone())
+        } else {
+            tracing::info!("No timeline for {} was previously stored", room_id);
+            return Ok(None);
+        };
+
+        let stream = stream! {
+            for (_, item) in events {
+                yield Ok(item);
+            }
+        };
+
+        tracing::info!(
+            "Found previously stored timeline for {}, with end token {:?}",
+            room_id,
+            end_token
+        );
+
+        Ok(Some((Box::pin(stream), end_token)))
+    }
 }
 
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
@@ -477,7 +723,7 @@ impl StateStore for MemoryStore {
     async fn get_state_event(
         &self,
         room_id: &RoomId,
-        event_type: EventType,
+        event_type: StateEventType,
         state_key: &str,
     ) -> Result<Option<Raw<AnySyncStateEvent>>> {
         self.get_state_event(room_id, event_type, state_key).await
@@ -486,7 +732,7 @@ impl StateStore for MemoryStore {
     async fn get_state_events(
         &self,
         room_id: &RoomId,
-        event_type: EventType,
+        event_type: StateEventType,
     ) -> Result<Vec<Raw<AnySyncStateEvent>>> {
         self.get_state_events(room_id, event_type).await
     }
@@ -495,7 +741,7 @@ impl StateStore for MemoryStore {
         &self,
         room_id: &RoomId,
         user_id: &UserId,
-    ) -> Result<Option<MemberEventContent>> {
+    ) -> Result<Option<MinimalRoomMemberEvent>> {
         self.get_profile(room_id, user_id).await
     }
 
@@ -507,16 +753,24 @@ impl StateStore for MemoryStore {
         self.get_member_event(room_id, state_key).await
     }
 
-    async fn get_user_ids(&self, room_id: &RoomId) -> Result<Vec<UserId>> {
+    async fn get_user_ids(&self, room_id: &RoomId) -> Result<Vec<OwnedUserId>> {
         Ok(self.get_user_ids(room_id))
     }
 
-    async fn get_invited_user_ids(&self, room_id: &RoomId) -> Result<Vec<UserId>> {
-        Ok(self.get_invited_user_ids(room_id))
+    async fn get_invited_user_ids(&self, room_id: &RoomId) -> Result<Vec<OwnedUserId>> {
+        let v = self.get_invited_user_ids(room_id);
+        if !v.is_empty() {
+            return Ok(v);
+        }
+        Ok(self.get_stripped_invited_user_ids(room_id))
     }
 
-    async fn get_joined_user_ids(&self, room_id: &RoomId) -> Result<Vec<UserId>> {
-        Ok(self.get_joined_user_ids(room_id))
+    async fn get_joined_user_ids(&self, room_id: &RoomId) -> Result<Vec<OwnedUserId>> {
+        let v = self.get_joined_user_ids(room_id);
+        if !v.is_empty() {
+            return Ok(v);
+        }
+        Ok(self.get_stripped_joined_user_ids(room_id))
     }
 
     async fn get_room_infos(&self) -> Result<Vec<RoomInfo>> {
@@ -531,8 +785,7 @@ impl StateStore for MemoryStore {
         &self,
         room_id: &RoomId,
         display_name: &str,
-    ) -> Result<BTreeSet<UserId>> {
-        #[allow(clippy::map_clone)]
+    ) -> Result<BTreeSet<OwnedUserId>> {
         Ok(self
             .display_names
             .get(room_id)
@@ -542,7 +795,7 @@ impl StateStore for MemoryStore {
 
     async fn get_account_data_event(
         &self,
-        event_type: EventType,
+        event_type: GlobalAccountDataEventType,
     ) -> Result<Option<Raw<AnyGlobalAccountDataEvent>>> {
         self.get_account_data_event(event_type).await
     }
@@ -550,7 +803,7 @@ impl StateStore for MemoryStore {
     async fn get_room_account_data_event(
         &self,
         room_id: &RoomId,
-        event_type: EventType,
+        event_type: RoomAccountDataEventType,
     ) -> Result<Option<Raw<AnyRoomAccountDataEvent>>> {
         self.get_room_account_data_event(room_id, event_type).await
     }
@@ -560,7 +813,7 @@ impl StateStore for MemoryStore {
         room_id: &RoomId,
         receipt_type: ReceiptType,
         user_id: &UserId,
-    ) -> Result<Option<(EventId, Receipt)>> {
+    ) -> Result<Option<(OwnedEventId, Receipt)>> {
         self.get_user_room_receipt_event(room_id, receipt_type, user_id).await
     }
 
@@ -569,7 +822,7 @@ impl StateStore for MemoryStore {
         room_id: &RoomId,
         receipt_type: ReceiptType,
         event_id: &EventId,
-    ) -> Result<Vec<(UserId, Receipt)>> {
+    ) -> Result<Vec<(OwnedUserId, Receipt)>> {
         self.get_event_room_receipt_events(room_id, receipt_type, event_id).await
     }
 
@@ -596,154 +849,39 @@ impl StateStore for MemoryStore {
     async fn remove_media_content_for_uri(&self, uri: &MxcUri) -> Result<()> {
         self.remove_media_content_for_uri(uri).await
     }
+
+    async fn remove_room(&self, room_id: &RoomId) -> Result<()> {
+        self.remove_room(room_id).await
+    }
+
+    #[cfg(feature = "experimental-timeline")]
+    async fn room_timeline(
+        &self,
+        room_id: &RoomId,
+    ) -> Result<Option<(BoxStream<Result<SyncRoomEvent>>, Option<String>)>> {
+        self.room_timeline(room_id).await
+    }
+}
+
+#[derive(Debug, Default)]
+#[cfg(feature = "experimental-timeline")]
+struct TimelineData {
+    pub start: String,
+    pub start_position: isize,
+    pub end: Option<String>,
+    pub end_position: isize,
+    pub events: BTreeMap<isize, SyncRoomEvent>,
+    pub event_id_to_position: HashMap<OwnedEventId, isize>,
 }
 
 #[cfg(test)]
-mod test {
-    use matrix_sdk_test::async_test;
-    use ruma::{
-        api::client::r0::media::get_content_thumbnail::Method, event_id, mxc_uri,
-        receipt::ReceiptType, room_id, uint, user_id, UserId,
-    };
-    use serde_json::json;
+mod tests {
 
-    use super::{MemoryStore, StateChanges};
-    use crate::media::{MediaFormat, MediaRequest, MediaThumbnailSize, MediaType};
+    use super::{MemoryStore, Result, StateStore};
 
-    fn user_id() -> UserId {
-        user_id!("@example:localhost")
+    async fn get_store() -> Result<impl StateStore> {
+        Ok(MemoryStore::new())
     }
 
-    #[async_test]
-    async fn test_receipts_saving() {
-        let store = MemoryStore::new();
-
-        let room_id = room_id!("!test:localhost");
-
-        let first_event_id = event_id!("$1435641916114394fHBLK:matrix.org");
-        let second_event_id = event_id!("$fHBLK1435641916114394:matrix.org");
-
-        let first_receipt_event = serde_json::from_value(json!({
-            first_event_id.clone(): {
-                "m.read": {
-                    user_id(): {
-                        "ts": 1436451550453u64
-                    }
-                }
-            }
-        }))
-        .unwrap();
-
-        let second_receipt_event = serde_json::from_value(json!({
-            second_event_id.clone(): {
-                "m.read": {
-                    user_id(): {
-                        "ts": 1436451551453u64
-                    }
-                }
-            }
-        }))
-        .unwrap();
-
-        assert!(store
-            .get_user_room_receipt_event(&room_id, ReceiptType::Read, &user_id())
-            .await
-            .unwrap()
-            .is_none());
-        assert!(store
-            .get_event_room_receipt_events(&room_id, ReceiptType::Read, &first_event_id)
-            .await
-            .unwrap()
-            .is_empty());
-        assert!(store
-            .get_event_room_receipt_events(&room_id, ReceiptType::Read, &second_event_id)
-            .await
-            .unwrap()
-            .is_empty());
-
-        let mut changes = StateChanges::default();
-        changes.add_receipts(&room_id, first_receipt_event);
-
-        store.save_changes(&changes).await.unwrap();
-        assert!(store
-            .get_user_room_receipt_event(&room_id, ReceiptType::Read, &user_id())
-            .await
-            .unwrap()
-            .is_some(),);
-        assert_eq!(
-            store
-                .get_event_room_receipt_events(&room_id, ReceiptType::Read, &first_event_id)
-                .await
-                .unwrap()
-                .len(),
-            1
-        );
-        assert!(store
-            .get_event_room_receipt_events(&room_id, ReceiptType::Read, &second_event_id)
-            .await
-            .unwrap()
-            .is_empty());
-
-        let mut changes = StateChanges::default();
-        changes.add_receipts(&room_id, second_receipt_event);
-
-        store.save_changes(&changes).await.unwrap();
-        assert!(store
-            .get_user_room_receipt_event(&room_id, ReceiptType::Read, &user_id())
-            .await
-            .unwrap()
-            .is_some());
-        assert!(store
-            .get_event_room_receipt_events(&room_id, ReceiptType::Read, &first_event_id)
-            .await
-            .unwrap()
-            .is_empty());
-        assert_eq!(
-            store
-                .get_event_room_receipt_events(&room_id, ReceiptType::Read, &second_event_id)
-                .await
-                .unwrap()
-                .len(),
-            1
-        );
-    }
-
-    #[async_test]
-    async fn test_media_content() {
-        let store = MemoryStore::new();
-
-        let uri = mxc_uri!("mxc://localhost/media");
-        let content: Vec<u8> = "somebinarydata".into();
-
-        let request_file =
-            MediaRequest { media_type: MediaType::Uri(uri.clone()), format: MediaFormat::File };
-
-        let request_thumbnail = MediaRequest {
-            media_type: MediaType::Uri(uri.clone()),
-            format: MediaFormat::Thumbnail(MediaThumbnailSize {
-                method: Method::Crop,
-                width: uint!(100),
-                height: uint!(100),
-            }),
-        };
-
-        assert!(store.get_media_content(&request_file).await.unwrap().is_none());
-        assert!(store.get_media_content(&request_thumbnail).await.unwrap().is_none());
-
-        store.add_media_content(&request_file, content.clone()).await.unwrap();
-        assert!(store.get_media_content(&request_file).await.unwrap().is_some());
-
-        store.remove_media_content(&request_file).await.unwrap();
-        assert!(store.get_media_content(&request_file).await.unwrap().is_none());
-
-        store.add_media_content(&request_file, content.clone()).await.unwrap();
-        assert!(store.get_media_content(&request_file).await.unwrap().is_some());
-
-        store.add_media_content(&request_thumbnail, content.clone()).await.unwrap();
-        assert!(store.get_media_content(&request_thumbnail).await.unwrap().is_some());
-
-        store.remove_media_content_for_uri(&uri).await.unwrap();
-        assert!(store.get_media_content(&request_file).await.unwrap().is_none());
-        assert!(store.get_media_content(&request_thumbnail).await.unwrap().is_none());
-    }
+    statestore_integration_tests! { integration }
 }

crates/matrix-sdk-base/src/store/mod.rs

@@ -12,63 +12,75 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(feature = "sled_state_store")]
-use std::path::Path;
+//! The state store holds the overall state for rooms, users and their
+//! profiles and their timelines. It is an overall cache for faster access
+//! and convenience- accessible through `Store`.
+//!
+//! Implementing the `StateStore` trait, you can plug any storage backend
+//! into the store for the actual storage. By default this brings an in-memory
+//! store.
+
 use std::{
     collections::{BTreeMap, BTreeSet},
     ops::Deref,
+    pin::Pin,
+    result::Result as StdResult,
     sync::Arc,
 };
 
+#[cfg(any(test, feature = "testing"))]
+#[macro_use]
+pub mod integration_tests;
+
+use async_trait::async_trait;
 use dashmap::DashMap;
-use matrix_sdk_common::{async_trait, locks::RwLock, AsyncTraitDeps};
+use matrix_sdk_common::{locks::RwLock, AsyncTraitDeps};
+#[cfg(feature = "e2e-encryption")]
+use matrix_sdk_crypto::store::CryptoStore;
 use ruma::{
-    api::client::r0::push::get_notifications::Notification,
+    api::client::push::get_notifications::v3::Notification,
     events::{
         presence::PresenceEvent,
         receipt::{Receipt, ReceiptEventContent},
-        room::member::MemberEventContent,
+        room::member::{StrippedRoomMemberEvent, SyncRoomMemberEvent},
         AnyGlobalAccountDataEvent, AnyRoomAccountDataEvent, AnyStrippedStateEvent,
-        AnySyncStateEvent, EventContent, EventType,
+        AnySyncStateEvent, GlobalAccountDataEventType, RoomAccountDataEventType, StateEventType,
     },
     receipt::ReceiptType,
     serde::Raw,
-    EventId, MxcUri, RoomId, UserId,
+    EventId, MxcUri, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UserId,
 };
-#[cfg(feature = "sled_state_store")]
-use sled::Db;
 
+/// BoxStream of owned Types
+pub type BoxStream<T> = Pin<Box<dyn futures_util::Stream<Item = T> + Send>>;
+
+#[cfg(feature = "experimental-timeline")]
+use crate::deserialized_responses::{SyncRoomEvent, TimelineSlice};
 use crate::{
-    deserialized_responses::{MemberEvent, StrippedMemberEvent},
+    deserialized_responses::MemberEvent,
     media::MediaRequest,
     rooms::{RoomInfo, RoomType},
-    Room, Session,
+    MinimalRoomMemberEvent, Room, Session,
 };
 
 pub(crate) mod ambiguity_map;
 mod memory_store;
-#[cfg(feature = "sled_state_store")]
-mod sled_store;
 
-#[cfg(not(feature = "sled_state_store"))]
-use self::memory_store::MemoryStore;
-#[cfg(feature = "sled_state_store")]
-use self::sled_store::SledStore;
+pub use self::memory_store::MemoryStore;
 
 /// State store specific error type.
 #[derive(Debug, thiserror::Error)]
 pub enum StoreError {
-    /// An error happened in the underlying sled database.
-    #[cfg(feature = "sled_state_store")]
+    /// An error happened in the underlying database backend.
     #[error(transparent)]
-    Sled(#[from] sled::Error),
+    Backend(Box<dyn std::error::Error + Send + Sync>),
     /// An error happened while serializing or deserializing some data.
     #[error(transparent)]
     Json(#[from] serde_json::Error),
     /// An error happened while deserializing a Matrix identifier, e.g. an user
     /// id.
     #[error(transparent)]
-    Identifier(#[from] ruma::identifiers::Error),
+    Identifier(#[from] ruma::IdParseError),
     /// The store is locked with a passphrase and an incorrect passphrase was
     /// given.
     #[error("The store failed to be unlocked")]
@@ -79,6 +91,34 @@ pub enum StoreError {
     /// The store failed to encrypt or decrypt some data.
     #[error("Error encrypting or decrypting data from the store: {0}")]
     Encryption(String),
+    /// The store failed to encode or decode some data.
+    #[error("Error encoding or decoding data from the store: {0}")]
+    Codec(String),
+
+    /// The database format has changed in a backwards incompatible way.
+    #[error(
+        "The database format changed in an incompatible way, current \
+        version: {0}, latest version: {1}"
+    )]
+    UnsupportedDatabaseVersion(usize, usize),
+    /// Redacting an event in the store has failed.
+    ///
+    /// This should never happen.
+    #[error("Redaction failed: {0}")]
+    Redaction(#[source] ruma::signatures::Error),
+}
+
+impl StoreError {
+    /// Create a new [`Backend`][Self::Backend] error.
+    ///
+    /// Shorthand for `StoreError::Backend(Box::new(error))`.
+    #[inline]
+    pub fn backend<E>(error: E) -> Self
+    where
+        E: std::error::Error + Send + Sync + 'static,
+    {
+        Self::Backend(Box::new(error))
+    }
 }
 
 /// A `StateStore` specific result type.
@@ -129,11 +169,11 @@ pub trait StateStore: AsyncTraitDeps {
     async fn get_state_event(
         &self,
         room_id: &RoomId,
-        event_type: EventType,
+        event_type: StateEventType,
         state_key: &str,
     ) -> Result<Option<Raw<AnySyncStateEvent>>>;
 
-    /// Get a list of state events for a given room and `EventType`.
+    /// Get a list of state events for a given room and `StateEventType`.
     ///
     /// # Arguments
     ///
@@ -143,7 +183,7 @@ pub trait StateStore: AsyncTraitDeps {
     async fn get_state_events(
         &self,
         room_id: &RoomId,
-        event_type: EventType,
+        event_type: StateEventType,
     ) -> Result<Vec<Raw<AnySyncStateEvent>>>;
 
     /// Get the current profile for the given user in the given room.
@@ -157,9 +197,9 @@ pub trait StateStore: AsyncTraitDeps {
         &self,
         room_id: &RoomId,
         user_id: &UserId,
-    ) -> Result<Option<MemberEventContent>>;
+    ) -> Result<Option<MinimalRoomMemberEvent>>;
 
-    /// Get a raw `MemberEvent` for the given state key in the given room id.
+    /// Get the `MemberEvent` for the given state key in the given room id.
     ///
     /// # Arguments
     ///
@@ -172,16 +212,17 @@ pub trait StateStore: AsyncTraitDeps {
         state_key: &UserId,
     ) -> Result<Option<MemberEvent>>;
 
-    /// Get all the user ids of members for a given room.
-    async fn get_user_ids(&self, room_id: &RoomId) -> Result<Vec<UserId>>;
+    /// Get all the user ids of members for a given room, for stripped and
+    /// regular rooms alike.
+    async fn get_user_ids(&self, room_id: &RoomId) -> Result<Vec<OwnedUserId>>;
 
     /// Get all the user ids of members that are in the invited state for a
-    /// given room.
-    async fn get_invited_user_ids(&self, room_id: &RoomId) -> Result<Vec<UserId>>;
+    /// given room, for stripped and regular rooms alike.
+    async fn get_invited_user_ids(&self, room_id: &RoomId) -> Result<Vec<OwnedUserId>>;
 
     /// Get all the user ids of members that are in the joined state for a
-    /// given room.
-    async fn get_joined_user_ids(&self, room_id: &RoomId) -> Result<Vec<UserId>>;
+    /// given room, for stripped and regular rooms alike.
+    async fn get_joined_user_ids(&self, room_id: &RoomId) -> Result<Vec<OwnedUserId>>;
 
     /// Get all the pure `RoomInfo`s the store knows about.
     async fn get_room_infos(&self) -> Result<Vec<RoomInfo>>;
@@ -201,7 +242,7 @@ pub trait StateStore: AsyncTraitDeps {
         &self,
         room_id: &RoomId,
         display_name: &str,
-    ) -> Result<BTreeSet<UserId>>;
+    ) -> Result<BTreeSet<OwnedUserId>>;
 
     /// Get an event out of the account data store.
     ///
@@ -210,7 +251,7 @@ pub trait StateStore: AsyncTraitDeps {
     /// * `event_type` - The event type of the account data event.
     async fn get_account_data_event(
         &self,
-        event_type: EventType,
+        event_type: GlobalAccountDataEventType,
     ) -> Result<Option<Raw<AnyGlobalAccountDataEvent>>>;
 
     /// Get an event out of the room account data store.
@@ -225,7 +266,7 @@ pub trait StateStore: AsyncTraitDeps {
     async fn get_room_account_data_event(
         &self,
         room_id: &RoomId,
-        event_type: EventType,
+        event_type: RoomAccountDataEventType,
     ) -> Result<Option<Raw<AnyRoomAccountDataEvent>>>;
 
     /// Get an event out of the user room receipt store.
@@ -243,7 +284,7 @@ pub trait StateStore: AsyncTraitDeps {
         room_id: &RoomId,
         receipt_type: ReceiptType,
         user_id: &UserId,
-    ) -> Result<Option<(EventId, Receipt)>>;
+    ) -> Result<Option<(OwnedEventId, Receipt)>>;
 
     /// Get events out of the event room receipt store.
     ///
@@ -261,7 +302,7 @@ pub trait StateStore: AsyncTraitDeps {
         room_id: &RoomId,
         receipt_type: ReceiptType,
         event_id: &EventId,
-    ) -> Result<Vec<(UserId, Receipt)>>;
+    ) -> Result<Vec<(OwnedUserId, Receipt)>>;
 
     /// Get arbitrary data from the custom store
     ///
@@ -309,6 +350,27 @@ pub trait StateStore: AsyncTraitDeps {
     ///
     /// * `uri` - The `MxcUri` of the media files.
     async fn remove_media_content_for_uri(&self, uri: &MxcUri) -> Result<()>;
+
+    /// Removes a room and all elements associated from the state store.
+    ///
+    /// # Arguments
+    ///
+    /// * `room_id` - The `RoomId` of the room to delete.
+    async fn remove_room(&self, room_id: &RoomId) -> Result<()>;
+
+    /// Get a stream of the stored timeline
+    ///
+    /// # Arguments
+    ///
+    /// * `room_id` - The `RoomId` of the room to delete.
+    ///
+    /// Returns a stream of events and a token that can be used to request
+    /// previous events.
+    #[cfg(feature = "experimental-timeline")]
+    async fn room_timeline(
+        &self,
+        room_id: &RoomId,
+    ) -> Result<Option<(BoxStream<Result<SyncRoomEvent>>, Option<String>)>>;
 }
 
 /// A state store wrapper for the SDK.
@@ -320,12 +382,22 @@ pub struct Store {
     inner: Arc<dyn StateStore>,
     pub(crate) session: Arc<RwLock<Option<Session>>>,
     pub(crate) sync_token: Arc<RwLock<Option<String>>>,
-    rooms: Arc<DashMap<RoomId, Room>>,
-    stripped_rooms: Arc<DashMap<RoomId, Room>>,
+    rooms: Arc<DashMap<OwnedRoomId, Room>>,
+    stripped_rooms: Arc<DashMap<OwnedRoomId, Room>>,
 }
 
 impl Store {
-    fn new(inner: Box<dyn StateStore>) -> Self {
+    /// Create a new Store with the default `MemoryStore`
+    pub fn open_memory_store() -> Self {
+        let inner = Box::new(MemoryStore::new());
+
+        Self::new(inner)
+    }
+}
+
+impl Store {
+    /// Create a new store, wrappning the given `StateStore`
+    pub fn new(inner: Box<dyn StateStore>) -> Self {
         Self {
             inner: inner.into(),
             session: Default::default(),
@@ -335,7 +407,9 @@ impl Store {
         }
     }
 
-    pub(crate) async fn restore_session(&self, session: Session) -> Result<()> {
+    /// Restore the access to the Store from the given `Session`, overwrites any
+    /// previously existing access to the Store.
+    pub async fn restore_session(&self, session: Session) -> Result<()> {
         for info in self.inner.get_room_infos().await? {
             let room = Room::restore(&session.user_id, self.inner.clone(), info);
             self.rooms.insert(room.room_id().to_owned(), room);
@@ -354,39 +428,6 @@ impl Store {
         Ok(())
     }
 
-    #[cfg(not(feature = "sled_state_store"))]
-    pub(crate) fn open_memory_store() -> Self {
-        let inner = Box::new(MemoryStore::new());
-
-        Self::new(inner)
-    }
-
-    /// Open the default Sled store.
-    ///
-    /// # Arguments
-    ///
-    /// * `path` - The path where the store should reside in.
-    ///
-    /// * `passphrase` - A passphrase that should be used to encrypt the state
-    /// store.
-    #[cfg(feature = "sled_state_store")]
-    pub fn open_default(path: impl AsRef<Path>, passphrase: Option<&str>) -> Result<(Self, Db)> {
-        let inner = if let Some(passphrase) = passphrase {
-            SledStore::open_with_passphrase(path, passphrase)?
-        } else {
-            SledStore::open_with_path(path)?
-        };
-
-        Ok((Self::new(Box::new(inner.clone())), inner.inner))
-    }
-
-    #[cfg(feature = "sled_state_store")]
-    pub(crate) fn open_temporary() -> Result<(Self, Db)> {
-        let inner = SledStore::open()?;
-
-        Ok((Self::new(Box::new(inner.clone())), inner.inner))
-    }
-
     /// Get all the rooms this store knows about.
     pub fn get_rooms(&self) -> Vec<Room> {
         self.rooms.iter().filter_map(|r| self.get_room(r.key())).collect()
@@ -404,26 +445,40 @@ impl Store {
             .or_else(|| self.get_stripped_room(room_id))
     }
 
-    fn get_stripped_room(&self, room_id: &RoomId) -> Option<Room> {
+    /// Get all the rooms this store knows about.
+    pub fn get_stripped_rooms(&self) -> Vec<Room> {
+        self.stripped_rooms.iter().filter_map(|r| self.get_stripped_room(r.key())).collect()
+    }
+
+    /// Get the stripped room with the given room id.
+    pub fn get_stripped_room(&self, room_id: &RoomId) -> Option<Room> {
         self.stripped_rooms.get(room_id).map(|r| r.clone())
     }
 
-    pub(crate) async fn get_or_create_stripped_room(&self, room_id: &RoomId) -> Room {
+    /// Lookup the stripped Room for the given RoomId, or create one, if it
+    /// didn't exist yet in the store
+    pub async fn get_or_create_stripped_room(&self, room_id: &RoomId) -> Room {
         let session = self.session.read().await;
         let user_id = &session.as_ref().expect("Creating room while not being logged in").user_id;
 
         self.stripped_rooms
-            .entry(room_id.clone())
+            .entry(room_id.to_owned())
             .or_insert_with(|| Room::new(user_id, self.inner.clone(), room_id, RoomType::Invited))
             .clone()
     }
 
-    pub(crate) async fn get_or_create_room(&self, room_id: &RoomId, room_type: RoomType) -> Room {
+    /// Lookup the Room for the given RoomId, or create one, if it didn't exist
+    /// yet in the store
+    pub async fn get_or_create_room(&self, room_id: &RoomId, room_type: RoomType) -> Room {
+        if room_type == RoomType::Invited {
+            return self.get_or_create_stripped_room(room_id).await;
+        }
+
         let session = self.session.read().await;
         let user_id = &session.as_ref().expect("Creating room while not being logged in").user_id;
 
         self.rooms
-            .entry(room_id.clone())
+            .entry(room_id.to_owned())
             .or_insert_with(|| Room::new(user_id, self.inner.clone(), room_id, room_type))
             .clone()
     }
@@ -446,39 +501,49 @@ pub struct StateChanges {
     /// associated user account.
     pub session: Option<Session>,
     /// A mapping of event type string to `AnyBasicEvent`.
-    pub account_data: BTreeMap<String, Raw<AnyGlobalAccountDataEvent>>,
+    pub account_data: BTreeMap<GlobalAccountDataEventType, Raw<AnyGlobalAccountDataEvent>>,
     /// A mapping of `UserId` to `PresenceEvent`.
-    pub presence: BTreeMap<UserId, Raw<PresenceEvent>>,
+    pub presence: BTreeMap<OwnedUserId, Raw<PresenceEvent>>,
 
-    /// A mapping of `RoomId` to a map of users and their `MemberEvent`.
-    pub members: BTreeMap<RoomId, BTreeMap<UserId, MemberEvent>>,
-    /// A mapping of `RoomId` to a map of users and their `MemberEventContent`.
-    pub profiles: BTreeMap<RoomId, BTreeMap<UserId, MemberEventContent>>,
+    /// A mapping of `RoomId` to a map of users and their `SyncRoomMemberEvent`.
+    pub members: BTreeMap<OwnedRoomId, BTreeMap<OwnedUserId, SyncRoomMemberEvent>>,
+    /// A mapping of `RoomId` to a map of users and their
+    /// `MinimalRoomMemberEvent`.
+    pub profiles: BTreeMap<OwnedRoomId, BTreeMap<OwnedUserId, MinimalRoomMemberEvent>>,
 
     /// A mapping of `RoomId` to a map of event type string to a state key and
     /// `AnySyncStateEvent`.
-    pub state: BTreeMap<RoomId, BTreeMap<String, BTreeMap<String, Raw<AnySyncStateEvent>>>>,
+    pub state:
+        BTreeMap<OwnedRoomId, BTreeMap<StateEventType, BTreeMap<String, Raw<AnySyncStateEvent>>>>,
     /// A mapping of `RoomId` to a map of event type string to `AnyBasicEvent`.
-    pub room_account_data: BTreeMap<RoomId, BTreeMap<String, Raw<AnyRoomAccountDataEvent>>>,
+    pub room_account_data:
+        BTreeMap<OwnedRoomId, BTreeMap<RoomAccountDataEventType, Raw<AnyRoomAccountDataEvent>>>,
     /// A map of `RoomId` to `RoomInfo`.
-    pub room_infos: BTreeMap<RoomId, RoomInfo>,
+    pub room_infos: BTreeMap<OwnedRoomId, RoomInfo>,
     /// A map of `RoomId` to `ReceiptEventContent`.
-    pub receipts: BTreeMap<RoomId, ReceiptEventContent>,
+    pub receipts: BTreeMap<OwnedRoomId, ReceiptEventContent>,
 
     /// A mapping of `RoomId` to a map of event type to a map of state key to
     /// `AnyStrippedStateEvent`.
-    pub stripped_state:
-        BTreeMap<RoomId, BTreeMap<String, BTreeMap<String, Raw<AnyStrippedStateEvent>>>>,
-    /// A mapping of `RoomId` to a map of users and their `StrippedMemberEvent`.
-    pub stripped_members: BTreeMap<RoomId, BTreeMap<UserId, StrippedMemberEvent>>,
-    /// A map of `RoomId` to `RoomInfo`.
-    pub invited_room_info: BTreeMap<RoomId, RoomInfo>,
+    pub stripped_state: BTreeMap<
+        OwnedRoomId,
+        BTreeMap<StateEventType, BTreeMap<String, Raw<AnyStrippedStateEvent>>>,
+    >,
+    /// A mapping of `RoomId` to a map of users and their
+    /// `StrippedRoomMemberEvent`.
+    pub stripped_members: BTreeMap<OwnedRoomId, BTreeMap<OwnedUserId, StrippedRoomMemberEvent>>,
+    /// A map of `RoomId` to `RoomInfo` for stripped rooms (e.g. for invites or
+    /// while knocking)
+    pub stripped_room_infos: BTreeMap<OwnedRoomId, RoomInfo>,
 
     /// A map from room id to a map of a display name and a set of user ids that
     /// share that display name in the given room.
-    pub ambiguity_maps: BTreeMap<RoomId, BTreeMap<String, BTreeSet<UserId>>>,
+    pub ambiguity_maps: BTreeMap<OwnedRoomId, BTreeMap<String, BTreeSet<OwnedUserId>>>,
     /// A map of `RoomId` to a vector of `Notification`s
-    pub notifications: BTreeMap<RoomId, Vec<Notification>>,
+    pub notifications: BTreeMap<OwnedRoomId, Vec<Notification>>,
+    /// A mapping of `RoomId` to a `TimelineSlice`
+    #[cfg(feature = "experimental-timeline")]
+    pub timeline: BTreeMap<OwnedRoomId, TimelineSlice>,
 }
 
 impl StateChanges {
@@ -499,7 +564,7 @@ impl StateChanges {
 
     /// Update the `StateChanges` struct with the given `RoomInfo`.
     pub fn add_stripped_room(&mut self, room: RoomInfo) {
-        self.room_infos.insert(room.room_id.as_ref().to_owned(), room);
+        self.stripped_room_infos.insert(room.room_id.as_ref().to_owned(), room);
     }
 
     /// Update the `StateChanges` struct with the given `AnyBasicEvent`.
@@ -508,7 +573,7 @@ impl StateChanges {
         event: AnyGlobalAccountDataEvent,
         raw_event: Raw<AnyGlobalAccountDataEvent>,
     ) {
-        self.account_data.insert(event.content().event_type().to_owned(), raw_event);
+        self.account_data.insert(event.event_type(), raw_event);
     }
 
     /// Update the `StateChanges` struct with the given room with a new
@@ -521,19 +586,16 @@ impl StateChanges {
     ) {
         self.room_account_data
             .entry(room_id.to_owned())
-            .or_insert_with(BTreeMap::new)
-            .insert(event.content().event_type().to_owned(), raw_event);
+            .or_default()
+            .insert(event.event_type(), raw_event);
     }
 
     /// Update the `StateChanges` struct with the given room with a new
     /// `StrippedMemberEvent`.
-    pub fn add_stripped_member(&mut self, room_id: &RoomId, event: StrippedMemberEvent) {
+    pub fn add_stripped_member(&mut self, room_id: &RoomId, event: StrippedRoomMemberEvent) {
         let user_id = event.state_key.clone();
 
-        self.stripped_members
-            .entry(room_id.to_owned())
-            .or_insert_with(BTreeMap::new)
-            .insert(user_id, event);
+        self.stripped_members.entry(room_id.to_owned()).or_default().insert(user_id, event);
     }
 
     /// Update the `StateChanges` struct with the given room with a new
@@ -546,16 +608,16 @@ impl StateChanges {
     ) {
         self.state
             .entry(room_id.to_owned())
-            .or_insert_with(BTreeMap::new)
-            .entry(event.content().event_type().to_string())
-            .or_insert_with(BTreeMap::new)
-            .insert(event.state_key().to_string(), raw_event);
+            .or_default()
+            .entry(event.event_type())
+            .or_default()
+            .insert(event.state_key().to_owned(), raw_event);
     }
 
     /// Update the `StateChanges` struct with the given room with a new
     /// `Notification`.
     pub fn add_notification(&mut self, room_id: &RoomId, notification: Notification) {
-        self.notifications.entry(room_id.to_owned()).or_insert_with(Vec::new).push(notification);
+        self.notifications.entry(room_id.to_owned()).or_default().push(notification);
     }
 
     /// Update the `StateChanges` struct with the given room with a new
@@ -563,4 +625,58 @@ impl StateChanges {
     pub fn add_receipts(&mut self, room_id: &RoomId, event: ReceiptEventContent) {
         self.receipts.insert(room_id.to_owned(), event);
     }
+
+    /// Update the `StateChanges` struct with the given room with a new
+    /// `TimelineSlice`.
+    #[cfg(feature = "experimental-timeline")]
+    pub fn add_timeline(&mut self, room_id: &RoomId, timeline: TimelineSlice) {
+        self.timeline.insert(room_id.to_owned(), timeline);
+    }
+}
+
+/// Configuration for the state store and, when `encryption` is enabled, for the
+/// crypto store.
+///
+/// # Example
+///
+/// ```
+/// # use matrix_sdk_base::store::StoreConfig;
+///
+/// let store_config = StoreConfig::new();
+/// ```
+#[derive(Default)]
+pub struct StoreConfig {
+    #[cfg(feature = "e2e-encryption")]
+    pub(crate) crypto_store: Option<Box<dyn CryptoStore>>,
+    pub(crate) state_store: Option<Box<dyn StateStore>>,
+}
+
+#[cfg(not(tarpaulin_include))]
+impl std::fmt::Debug for StoreConfig {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> StdResult<(), std::fmt::Error> {
+        fmt.debug_struct("StoreConfig").finish()
+    }
+}
+
+impl StoreConfig {
+    /// Create a new default `StoreConfig`.
+    #[must_use]
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    /// Set a custom implementation of a `CryptoStore`.
+    ///
+    /// The crypto store must be opened before being set.
+    #[cfg(feature = "e2e-encryption")]
+    pub fn crypto_store(mut self, store: Box<dyn CryptoStore>) -> Self {
+        self.crypto_store = Some(store);
+        self
+    }
+
+    /// Set a custom implementation of a `StateStore`.
+    pub fn state_store(mut self, store: Box<dyn StateStore>) -> Self {
+        self.state_store = Some(store);
+        self
+    }
 }

crates/matrix-sdk-base/src/timeline_stream.rs

@@ -0,0 +1,248 @@
+use std::{fmt, pin::Pin, sync::Arc};
+
+use dashmap::DashSet;
+use futures_channel::mpsc;
+use futures_core::{
+    stream::{BoxStream, Stream},
+    task::{Context, Poll},
+};
+use ruma::OwnedEventId;
+use thiserror::Error;
+use tracing::trace;
+
+use crate::{
+    deserialized_responses::{SyncRoomEvent, TimelineSlice},
+    store::Result,
+};
+
+const CHANNEL_LIMIT: usize = 10;
+
+/// Errors in a timeline stream
+#[derive(Error, Debug)]
+pub enum TimelineStreamError {
+    /// The end of the stored timeline was reached.
+    #[error("the end of the stored timeline was reached")]
+    EndCache {
+        /// The given token should be used to request more events from the
+        /// server.
+        fetch_more_token: String,
+    },
+    /// The event in the store produced an error
+    #[error("the event in the store produced an error")]
+    Store(crate::StoreError),
+}
+
+/// A `Stream` of timeline of a room
+pub struct TimelineStreamBackward<'a> {
+    receiver: mpsc::Receiver<TimelineSlice>,
+    stored_events: Option<BoxStream<'a, Result<SyncRoomEvent>>>,
+    pending: Vec<SyncRoomEvent>,
+    event_ids: Arc<DashSet<OwnedEventId>>,
+    token: Option<String>,
+}
+
+#[cfg(not(tarpaulin_include))]
+impl fmt::Debug for TimelineStreamBackward<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("TimelineStream")
+            .field("event_ids", &self.event_ids)
+            .field("token", &self.token)
+            .field("pending", &self.pending)
+            .finish()
+    }
+}
+
+impl<'a> TimelineStreamBackward<'a> {
+    /// Creates a tuple containing `Self` and a `Sender` to send new events to
+    /// this stream
+    ///
+    /// The stream is terminated when the entire timeline was returned,
+    /// otherwise an error will be returned till the issue is resolved, aka
+    /// new messages are loaded.
+    ///
+    /// # Arguments
+    ///
+    /// * `event_ids` - A set to store `EventId`s already seen by the stream.
+    ///   This should be shared
+    /// between the `TimelineStreamBackward` and `TimelineStreamForward` stream.
+    ///
+    /// * `token` - The current sync token or the token that identifies the end
+    ///   of the `stored_events`.
+    ///
+    /// * `stored_events` - A stream of events that are currently stored
+    ///   locally.
+    pub(crate) fn new(
+        event_ids: Arc<DashSet<OwnedEventId>>,
+        token: Option<String>,
+        stored_events: Option<BoxStream<'a, Result<SyncRoomEvent>>>,
+    ) -> (Self, mpsc::Sender<TimelineSlice>) {
+        let (sender, receiver) = mpsc::channel(CHANNEL_LIMIT);
+        let self_ = Self { event_ids, pending: Vec::new(), stored_events, token, receiver };
+
+        (self_, sender)
+    }
+
+    fn handle_new_slice(
+        &mut self,
+        slice: TimelineSlice,
+    ) -> Poll<Option<Result<SyncRoomEvent, TimelineStreamError>>> {
+        // Check if this is the batch we are expecting
+        if self.token.is_some() && self.token != Some(slice.start) {
+            trace!("Store received a timeline batch that wasn't expected");
+            return Poll::Pending;
+        }
+
+        // There is a gap in the timeline. Therefore, terminate the stream.
+        if slice.limited {
+            return Poll::Ready(None);
+        }
+
+        // The end of the timeline was reached
+        if slice.events.is_empty() {
+            return Poll::Ready(None);
+        }
+
+        for event in slice.events.into_iter().rev().filter(|event| {
+            self.event_ids
+                .insert(event.event_id().expect("Timeline events always have an event id."))
+        }) {
+            self.pending.push(event);
+        }
+        self.token = slice.end;
+
+        if let Some(event) = self.pending.pop() {
+            Poll::Ready(Some(Ok(event)))
+        } else if let Some(token) = &self.token {
+            Poll::Ready(Some(Err(TimelineStreamError::EndCache {
+                fetch_more_token: token.to_string(),
+            })))
+        } else {
+            Poll::Ready(None)
+        }
+    }
+}
+
+impl<'a> Stream for TimelineStreamBackward<'a> {
+    type Item = Result<SyncRoomEvent, TimelineStreamError>;
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let this = self.get_mut();
+        if let Some(stored_events) = &mut this.stored_events {
+            match Pin::new(stored_events).poll_next(cx) {
+                Poll::Ready(None) => {}
+                Poll::Pending => return Poll::Pending,
+                Poll::Ready(Some(event)) => {
+                    return Poll::Ready(Some(event.map_err(TimelineStreamError::Store)))
+                }
+            }
+
+            // We returned all events the store gave to us
+            this.stored_events = None;
+        }
+
+        if let Some(event) = this.pending.pop() {
+            Poll::Ready(Some(Ok(event)))
+        } else {
+            loop {
+                match Pin::new(&mut this.receiver).poll_next(cx) {
+                    Poll::Ready(Some(slice)) => match this.handle_new_slice(slice) {
+                        Poll::Pending => continue,
+                        other => break other,
+                    },
+                    Poll::Ready(None) => break Poll::Ready(None),
+                    Poll::Pending => {
+                        if let Some(token) = &this.token {
+                            // We tell the consumer that there are no more evens in cache
+                            break Poll::Ready(Some(Err(TimelineStreamError::EndCache {
+                                fetch_more_token: token.to_string(),
+                            })));
+                        } else {
+                            break Poll::Ready(None);
+                        }
+                    }
+                };
+            }
+        }
+    }
+}
+
+/// A `Stream` of timeline of a room
+pub struct TimelineStreamForward {
+    receiver: mpsc::Receiver<TimelineSlice>,
+    pending: Vec<SyncRoomEvent>,
+    event_ids: Arc<DashSet<OwnedEventId>>,
+}
+
+#[cfg(not(tarpaulin_include))]
+impl fmt::Debug for TimelineStreamForward {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("TimelineStream")
+            .field("event_ids", &self.event_ids)
+            .field("pending", &self.pending)
+            .finish()
+    }
+}
+
+impl TimelineStreamForward {
+    /// Creates a tuple containing `Self` and a `Sender` to send new events to
+    /// this stream
+    ///
+    /// The stream is only terminated when there would be a gap in the timeline.
+    /// This happens when a `SyncResponse` is obtained with a limited
+    /// timeline.
+    ///
+    /// # Arguments
+    ///
+    /// * `event_ids` - A set to store `EventId`s already seen by the stream.
+    ///   This should be shared
+    /// between the `TimelineStreamBackward` and `TimelineStreamForward` stream.
+    pub(crate) fn new(
+        event_ids: Arc<DashSet<OwnedEventId>>,
+    ) -> (Self, mpsc::Sender<TimelineSlice>) {
+        let (sender, receiver) = mpsc::channel(CHANNEL_LIMIT);
+        let self_ = Self { event_ids, pending: Vec::new(), receiver };
+
+        (self_, sender)
+    }
+
+    fn handle_new_slice(&mut self, slice: TimelineSlice) -> Poll<Option<SyncRoomEvent>> {
+        // There is a gap in the timeline. Therefore, terminate the stream.
+        if slice.limited {
+            return Poll::Ready(None);
+        }
+
+        for event in slice.events.into_iter().rev().filter(|event| {
+            self.event_ids
+                .insert(event.event_id().expect("Timeline events always have an event id."))
+        }) {
+            self.pending.push(event);
+        }
+
+        if let Some(event) = self.pending.pop() {
+            Poll::Ready(Some(event))
+        } else {
+            Poll::Pending
+        }
+    }
+}
+
+impl Stream for TimelineStreamForward {
+    type Item = SyncRoomEvent;
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let this = self.get_mut();
+
+        if let Some(event) = this.pending.pop() {
+            Poll::Ready(Some(event))
+        } else {
+            loop {
+                match Pin::new(&mut this.receiver).poll_next(cx) {
+                    Poll::Ready(Some(slice)) => match this.handle_new_slice(slice) {
+                        Poll::Pending => continue,
+                        other => break other,
+                    },
+                    Poll::Ready(None) => break Poll::Ready(None),
+                    Poll::Pending => break Poll::Pending,
+                }
+            }
+        }
+    }
+}

crates/matrix-sdk-base/src/utils.rs

@@ -0,0 +1,148 @@
+use ruma::{
+    events::{
+        room::member::{MembershipState, RoomMemberEventContent},
+        RedactContent, RedactedEventContent, StateEventContent, StrippedStateEvent, SyncStateEvent,
+    },
+    EventId, OwnedEventId, RoomVersionId,
+};
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+
+// #[serde(bound)] instead of DeserializeOwned in type where clause does not
+// work, it can only be a single bound that replaces the default and if a helper
+// trait is used, the compiler still complains about Deserialize not being
+// implemented for C::Redacted.
+//
+// It is unclear why a Serialize bound on C::Redacted is not also required.
+
+/// A minimal state event.
+///
+/// This type can holding an possibly-redacted state event with an optional
+/// event ID. The event ID is optional so this type can also hold events from
+/// invited rooms, where event IDs are not available.
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub enum MinimalStateEvent<C: StateEventContent + RedactContent>
+where
+    C::Redacted: StateEventContent + RedactedEventContent + DeserializeOwned,
+{
+    /// An unredacted event.
+    Original(OriginalMinimalStateEvent<C>),
+    /// A redacted event.
+    Redacted(RedactedMinimalStateEvent<C::Redacted>),
+}
+
+/// An unredacted minimal state event.
+///
+/// For more details see [`MinimalStateEvent`].
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct OriginalMinimalStateEvent<C>
+where
+    C: StateEventContent,
+{
+    /// The event's content.
+    pub content: C,
+    /// The event's ID, if known.
+    pub event_id: Option<OwnedEventId>,
+}
+
+/// A redacted minimal state event.
+///
+/// For more details see [`MinimalStateEvent`].
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct RedactedMinimalStateEvent<C>
+where
+    C: StateEventContent + RedactedEventContent,
+{
+    /// The event's content.
+    pub content: C,
+    /// The event's ID, if known.
+    pub event_id: Option<OwnedEventId>,
+}
+
+impl<C> MinimalStateEvent<C>
+where
+    C: StateEventContent + RedactContent,
+    C::Redacted: StateEventContent + RedactedEventContent + DeserializeOwned,
+{
+    /// Get the inner event's ID.
+    pub fn event_id(&self) -> Option<&EventId> {
+        match self {
+            MinimalStateEvent::Original(ev) => ev.event_id.as_deref(),
+            MinimalStateEvent::Redacted(ev) => ev.event_id.as_deref(),
+        }
+    }
+
+    /// Returns the inner event, if it isn't redacted.
+    pub fn as_original(&self) -> Option<&OriginalMinimalStateEvent<C>> {
+        match self {
+            MinimalStateEvent::Original(ev) => Some(ev),
+            MinimalStateEvent::Redacted(_) => None,
+        }
+    }
+
+    /// Converts `self` to the inner `OriginalMinimalStateEvent<C>`, if it isn't
+    /// redacted.
+    pub fn into_original(self) -> Option<OriginalMinimalStateEvent<C>> {
+        match self {
+            MinimalStateEvent::Original(ev) => Some(ev),
+            MinimalStateEvent::Redacted(_) => None,
+        }
+    }
+
+    /// Redacts this event.
+    ///
+    /// Does nothing if it is already redacted.
+    pub fn redact(&mut self, room_version: &RoomVersionId)
+    where
+        C: Clone,
+    {
+        if let MinimalStateEvent::Original(ev) = self {
+            *self = MinimalStateEvent::Redacted(RedactedMinimalStateEvent {
+                content: ev.content.clone().redact(room_version),
+                event_id: ev.event_id.clone(),
+            });
+        }
+    }
+}
+
+/// A minimal `m.room.member` event.
+pub type MinimalRoomMemberEvent = MinimalStateEvent<RoomMemberEventContent>;
+
+impl MinimalRoomMemberEvent {
+    /// Obtain the membership state, regardless of whether this event is
+    /// redacted.
+    pub fn membership(&self) -> &MembershipState {
+        match self {
+            MinimalStateEvent::Original(ev) => &ev.content.membership,
+            MinimalStateEvent::Redacted(ev) => &ev.content.membership,
+        }
+    }
+}
+
+impl<C> From<&SyncStateEvent<C>> for MinimalStateEvent<C>
+where
+    C: Clone + StateEventContent + RedactContent,
+    C::Redacted: Clone + StateEventContent + RedactedEventContent + DeserializeOwned,
+{
+    fn from(ev: &SyncStateEvent<C>) -> Self {
+        match ev {
+            SyncStateEvent::Original(ev) => Self::Original(OriginalMinimalStateEvent {
+                content: ev.content.clone(),
+                event_id: Some(ev.event_id.clone()),
+            }),
+            SyncStateEvent::Redacted(ev) => Self::Redacted(RedactedMinimalStateEvent {
+                content: ev.content.clone(),
+                event_id: Some(ev.event_id.clone()),
+            }),
+        }
+    }
+}
+
+impl<C> From<&StrippedStateEvent<C>> for MinimalStateEvent<C>
+where
+    C: Clone + StateEventContent + RedactContent,
+    C::Redacted: StateEventContent + RedactedEventContent + DeserializeOwned,
+{
+    fn from(ev: &StrippedStateEvent<C>) -> Self {
+        Self::Original(OriginalMinimalStateEvent { content: ev.content.clone(), event_id: None })
+    }
+}

crates/matrix-sdk-common/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+authors = ["Damir Jelić <poljar@termina.org.uk>"]
+description = "Collection of common types and imports used in the matrix-sdk"
+edition = "2021"
+homepage = "https://github.com/matrix-org/matrix-rust-sdk"
+keywords = ["matrix", "chat", "messaging", "ruma", "nio"]
+license = "Apache-2.0"
+name = "matrix-sdk-common"
+readme = "README.md"
+repository = "https://github.com/matrix-org/matrix-rust-sdk"
+rust-version = "1.60"
+version = "0.5.0"
+
+[package.metadata.docs.rs]
+default-target = "x86_64-unknown-linux-gnu"
+targets = ["x86_64-unknown-linux-gnu", "wasm32-unknown-unknown"]
+
+[dependencies]
+ruma = { version = "0.6.2", features = ["client-api-c"] }
+serde = "1.0.136"
+
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+async-lock = "2.5.0"
+instant = { version = "0.1.12", features = ["wasm-bindgen", "inaccurate"] }
+futures-util = { version = "0.3.21", default-features = false, features = ["channel"] }
+wasm-bindgen-futures = "0.4.30"
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+tokio = { version = "1.17.0", default-features = false, features = ["rt", "sync"] }
+instant = { version = "0.1.12", features = ["now"] }

crates/matrix-sdk-common/src/deserialized_responses.rs

@@ -1,19 +1,22 @@
-use std::{collections::BTreeMap, convert::TryFrom};
+use std::collections::BTreeMap;
 
 use ruma::{
-    api::client::r0::{
-        push::get_notifications::Notification,
-        sync::sync_events::{
+    api::client::{
+        push::get_notifications::v3::Notification,
+        sync::sync_events::v3::{
             DeviceLists, Ephemeral, GlobalAccountData, InvitedRoom, Presence, RoomAccountData,
             State, ToDevice, UnreadNotificationsCount as RumaUnreadNotificationsCount,
         },
     },
     events::{
-        room::member::MemberEventContent, AnySyncRoomEvent, StateEvent, StrippedStateEvent,
-        SyncStateEvent, Unsigned,
+        room::member::{
+            MembershipState, RoomMemberEvent, RoomMemberEventContent, StrippedRoomMemberEvent,
+            SyncRoomMemberEvent,
+        },
+        AnyRoomEvent, AnySyncRoomEvent,
     },
     serde::Raw,
-    DeviceIdBox, DeviceKeyAlgorithm, EventId, MilliSecondsSinceUnixEpoch, RoomId, UserId,
+    DeviceKeyAlgorithm, OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedUserId, UserId,
 };
 use serde::{Deserialize, Serialize};
 
@@ -25,9 +28,9 @@ pub struct AmbiguityChange {
     /// event itself ambiguous because of the event.
     pub member_ambiguous: bool,
     /// Has another user been disambiguated because of this event.
-    pub disambiguated_member: Option<UserId>,
+    pub disambiguated_member: Option<OwnedUserId>,
     /// Has another user become ambiguous because of this event.
-    pub ambiguated_member: Option<UserId>,
+    pub ambiguated_member: Option<OwnedUserId>,
 }
 
 /// Collection of ambiguioty changes that room member events trigger.
@@ -35,7 +38,7 @@ pub struct AmbiguityChange {
 pub struct AmbiguityChanges {
     /// A map from room id to a map of an event id to the `AmbiguityChange` that
     /// the event with the given id caused.
-    pub changes: BTreeMap<RoomId, BTreeMap<EventId, AmbiguityChange>>,
+    pub changes: BTreeMap<OwnedRoomId, BTreeMap<OwnedEventId, AmbiguityChange>>,
 }
 
 /// The verification state of the device that sent an event to us.
@@ -72,10 +75,10 @@ pub enum AlgorithmInfo {
 pub struct EncryptionInfo {
     /// The user ID of the event sender, note this is untrusted data unless the
     /// `verification_state` is as well trusted.
-    pub sender: UserId,
+    pub sender: OwnedUserId,
     /// The device ID of the device that sent us the event, note this is
     /// untrusted data unless `verification_state` is as well trusted.
-    pub sender_device: DeviceIdBox,
+    pub sender_device: OwnedDeviceId,
     /// Information about the algorithm that was used to encrypt the event.
     pub algorithm_info: AlgorithmInfo,
     /// The verification state of the device that sent us the event, note this
@@ -95,12 +98,29 @@ pub struct SyncRoomEvent {
     pub encryption_info: Option<EncryptionInfo>,
 }
 
+impl SyncRoomEvent {
+    /// Get the event id of this `SyncRoomEvent` if the event has any valid id.
+    pub fn event_id(&self) -> Option<OwnedEventId> {
+        self.event.get_field::<OwnedEventId>("event_id").ok().flatten()
+    }
+}
+
 impl From<Raw<AnySyncRoomEvent>> for SyncRoomEvent {
     fn from(inner: Raw<AnySyncRoomEvent>) -> Self {
         Self { encryption_info: None, event: inner }
     }
 }
 
+impl From<RoomEvent> for SyncRoomEvent {
+    fn from(o: RoomEvent) -> Self {
+        // This conversion is unproblematic since a SyncRoomEvent is just a
+        // RoomEvent without the room_id. By converting the raw value in this
+        // way, we simply cause the `room_id` field in the json to be ignored by
+        // a subsequent deserialization.
+        Self { encryption_info: o.encryption_info, event: o.event.cast() }
+    }
+}
+
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct SyncResponse {
     /// The batch token to supply in the `since` param of the next `/sync`
@@ -124,7 +144,7 @@ pub struct SyncResponse {
     /// Collection of ambiguity changes that room member events trigger.
     pub ambiguity_changes: AmbiguityChanges,
     /// New notifications per room.
-    pub notifications: BTreeMap<RoomId, Vec<Notification>>,
+    pub notifications: BTreeMap<OwnedRoomId, Vec<Notification>>,
 }
 
 impl SyncResponse {
@@ -136,7 +156,7 @@ impl SyncResponse {
 #[derive(Clone, Debug)]
 pub struct RoomEvent {
     /// The actual event.
-    pub event: Raw<ruma::events::AnyRoomEvent>,
+    pub event: Raw<AnyRoomEvent>,
     /// The encryption info about the event. Will be `None` if the event was not
     /// encrypted.
     pub encryption_info: Option<EncryptionInfo>,
@@ -145,11 +165,11 @@ pub struct RoomEvent {
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct Rooms {
     /// The rooms that the user has left or been banned from.
-    pub leave: BTreeMap<RoomId, LeftRoom>,
+    pub leave: BTreeMap<OwnedRoomId, LeftRoom>,
     /// The rooms that the user has joined.
-    pub join: BTreeMap<RoomId, JoinedRoom>,
+    pub join: BTreeMap<OwnedRoomId, JoinedRoom>,
     /// The rooms that the user has been invited to.
-    pub invite: BTreeMap<RoomId, InvitedRoom>,
+    pub invite: BTreeMap<OwnedRoomId, InvitedRoom>,
 }
 
 /// Updates to joined rooms.
@@ -243,96 +263,69 @@ impl Timeline {
     }
 }
 
+/// A slice of the timeline in the room.
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+pub struct TimelineSlice {
+    /// The `next_batch` or `from` token used to obtain this slice
+    pub start: String,
+
+    /// The `prev_batch` or `to` token used to obtain this slice
+    /// If `None` this `TimelineSlice` is the beginning of the room
+    pub end: Option<String>,
+
+    /// Whether the number of events returned for this slice was limited
+    /// by a `limit`-filter when requesting
+    pub limited: bool,
+
+    /// A list of events.
+    pub events: Vec<SyncRoomEvent>,
+
+    /// Whether this is a timeline slice obtained from a `SyncResponse`
+    pub sync: bool,
+}
+
+impl TimelineSlice {
+    pub fn new(
+        events: Vec<SyncRoomEvent>,
+        start: String,
+        end: Option<String>,
+        limited: bool,
+        sync: bool,
+    ) -> Self {
+        Self { start, end, events, limited, sync }
+    }
+}
+
+/// Wrapper around both MemberEvent-Types
+#[allow(clippy::large_enum_variant)]
 #[derive(Clone, Debug, Deserialize, Serialize)]
-#[serde(
-    try_from = "SyncStateEvent<MemberEventContent>",
-    into = "SyncStateEvent<MemberEventContent>"
-)]
-pub struct MemberEvent {
-    pub content: MemberEventContent,
-    pub event_id: EventId,
-    pub origin_server_ts: MilliSecondsSinceUnixEpoch,
-    pub prev_content: Option<MemberEventContent>,
-    pub sender: UserId,
-    pub state_key: UserId,
-    pub unsigned: Unsigned,
+pub enum MemberEvent {
+    Sync(SyncRoomMemberEvent),
+    Stripped(StrippedRoomMemberEvent),
 }
 
-impl TryFrom<SyncStateEvent<MemberEventContent>> for MemberEvent {
-    type Error = ruma::identifiers::Error;
-
-    fn try_from(event: SyncStateEvent<MemberEventContent>) -> Result<Self, Self::Error> {
-        Ok(MemberEvent {
-            content: event.content,
-            event_id: event.event_id,
-            origin_server_ts: event.origin_server_ts,
-            prev_content: event.prev_content,
-            sender: event.sender,
-            state_key: UserId::try_from(event.state_key)?,
-            unsigned: event.unsigned,
-        })
-    }
-}
-
-impl TryFrom<StateEvent<MemberEventContent>> for MemberEvent {
-    type Error = ruma::identifiers::Error;
-
-    fn try_from(event: StateEvent<MemberEventContent>) -> Result<Self, Self::Error> {
-        Ok(MemberEvent {
-            content: event.content,
-            event_id: event.event_id,
-            origin_server_ts: event.origin_server_ts,
-            prev_content: event.prev_content,
-            sender: event.sender,
-            state_key: UserId::try_from(event.state_key)?,
-            unsigned: event.unsigned,
-        })
-    }
-}
-
-impl From<MemberEvent> for SyncStateEvent<MemberEventContent> {
-    fn from(other: MemberEvent) -> SyncStateEvent<MemberEventContent> {
-        SyncStateEvent {
-            content: other.content,
-            event_id: other.event_id,
-            sender: other.sender,
-            origin_server_ts: other.origin_server_ts,
-            state_key: other.state_key.to_string(),
-            prev_content: other.prev_content,
-            unsigned: other.unsigned,
+impl MemberEvent {
+    /// The inner Content of the wrapped Event
+    pub fn original_content(&self) -> Option<&RoomMemberEventContent> {
+        match self {
+            MemberEvent::Sync(e) => e.as_original().map(|e| &e.content),
+            MemberEvent::Stripped(e) => Some(&e.content),
         }
     }
-}
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
-#[serde(
-    try_from = "StrippedStateEvent<MemberEventContent>",
-    into = "StrippedStateEvent<MemberEventContent>"
-)]
-pub struct StrippedMemberEvent {
-    pub content: MemberEventContent,
-    pub sender: UserId,
-    pub state_key: UserId,
-}
-
-impl TryFrom<StrippedStateEvent<MemberEventContent>> for StrippedMemberEvent {
-    type Error = ruma::identifiers::Error;
-
-    fn try_from(event: StrippedStateEvent<MemberEventContent>) -> Result<Self, Self::Error> {
-        Ok(StrippedMemberEvent {
-            content: event.content,
-            sender: event.sender,
-            state_key: UserId::try_from(event.state_key)?,
-        })
+    /// The membership state of the user
+    pub fn membership(&self) -> &MembershipState {
+        match self {
+            MemberEvent::Sync(e) => e.membership(),
+            MemberEvent::Stripped(e) => &e.content.membership,
+        }
     }
-}
 
-impl From<StrippedMemberEvent> for StrippedStateEvent<MemberEventContent> {
-    fn from(other: StrippedMemberEvent) -> Self {
-        Self {
-            content: other.content,
-            sender: other.sender,
-            state_key: other.state_key.to_string(),
+    /// The user id associated to this member event
+    pub fn user_id(&self) -> &UserId {
+        match self {
+            MemberEvent::Sync(e) => e.state_key(),
+            MemberEvent::Stripped(e) => &e.state_key,
         }
     }
 }
@@ -343,7 +336,54 @@ impl From<StrippedMemberEvent> for StrippedStateEvent<MemberEventContent> {
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct MembersResponse {
     /// The list of members events.
-    pub chunk: Vec<MemberEvent>,
+    pub chunk: Vec<RoomMemberEvent>,
     /// Collection of ambiguity changes that room member events trigger.
     pub ambiguity_changes: AmbiguityChanges,
 }
+
+#[cfg(test)]
+mod tests {
+    use ruma::{
+        event_id,
+        events::{
+            room::message::RoomMessageEventContent, AnySyncMessageLikeEvent, AnySyncRoomEvent,
+            MessageLikeUnsigned, OriginalMessageLikeEvent, SyncMessageLikeEvent,
+        },
+        room_id,
+        serde::Raw,
+        user_id, MilliSecondsSinceUnixEpoch,
+    };
+
+    use super::{RoomEvent, SyncRoomEvent};
+
+    #[test]
+    fn room_event_to_sync_room_event() {
+        let content = RoomMessageEventContent::text_plain("foobar");
+
+        let event = OriginalMessageLikeEvent {
+            content,
+            event_id: event_id!("$xxxxx:example.org").to_owned(),
+            room_id: room_id!("!someroom:example.com").to_owned(),
+            origin_server_ts: MilliSecondsSinceUnixEpoch::now(),
+            sender: user_id!("@carl:example.com").to_owned(),
+            unsigned: MessageLikeUnsigned::default(),
+        };
+
+        let room_event =
+            RoomEvent { event: Raw::new(&event).unwrap().cast(), encryption_info: None };
+
+        let converted_room_event: SyncRoomEvent = room_event.into();
+
+        let converted_event: AnySyncRoomEvent = converted_room_event.event.deserialize().unwrap();
+
+        let sync_event = AnySyncRoomEvent::MessageLike(AnySyncMessageLikeEvent::RoomMessage(
+            SyncMessageLikeEvent::Original(event.into()),
+        ));
+
+        // There is no PartialEq implementation for AnySyncRoomEvent, so we
+        // just compare a couple of fields here. The important thing is that
+        // the deserialization above worked.
+        assert_eq!(converted_event.event_id(), sync_event.event_id());
+        assert_eq!(converted_event.sender(), sync_event.sender());
+    }
+}

crates/matrix-sdk-common/src/executor.rs

@@ -2,12 +2,13 @@
 //! we do usually.
 #[cfg(target_arch = "wasm32")]
 use std::{
+    future::Future,
     pin::Pin,
     task::{Context, Poll},
 };
 
 #[cfg(target_arch = "wasm32")]
-use futures::{future::RemoteHandle, Future, FutureExt};
+use futures_util::{future::RemoteHandle, FutureExt};
 #[cfg(not(target_arch = "wasm32"))]
 pub use tokio::spawn;
 #[cfg(target_arch = "wasm32")]

crates/matrix-sdk-common/src/lib.rs

@@ -1,16 +1,7 @@
 #![doc = include_str!("../README.md")]
-#![deny(
-    missing_debug_implementations,
-    trivial_casts,
-    trivial_numeric_casts,
-    unused_extern_crates,
-    unused_import_braces,
-    unused_qualifications
-)]
+#![warn(missing_debug_implementations)]
 
-pub use async_trait::async_trait;
 pub use instant;
-pub use uuid;
 
 pub mod deserialized_responses;
 pub mod executor;
@@ -28,6 +19,6 @@ impl<T: std::fmt::Debug + Send + Sync> AsyncTraitDeps for T {}
 /// it's used on WASM. WASM targets will not require `Send` and `Sync` to have
 /// implemented, while other targets will.
 #[cfg(target_arch = "wasm32")]
-pub trait AsyncTraitDeps: std::fmt::Debug + Send + Sync {}
+pub trait AsyncTraitDeps: std::fmt::Debug {}
 #[cfg(target_arch = "wasm32")]
-impl<T: std::fmt::Debug + Send + Sync> AsyncTraitDeps for T {}
+impl<T: std::fmt::Debug> AsyncTraitDeps for T {}

crates/matrix-sdk-common/src/locks.rs

@@ -0,0 +1,4 @@
+#[cfg(target_arch = "wasm32")]
+pub use async_lock::{Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard};
+#[cfg(not(target_arch = "wasm32"))]
+pub use tokio::sync::{Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard};

crates/matrix-sdk-crypto-ffi/Cargo.toml

@@ -0,0 +1,64 @@
+[package]
+name = "matrix-sdk-crypto-ffi"
+version = "0.1.0"
+authors = ["Damir Jelić <poljar@termina.org.uk>"]
+edition = "2018"
+rust-version = "1.60"
+description = "Uniffi based bindings for the Rust SDK crypto crate"
+repository = "https://github.com/matrix-org/matrix-rust-sdk"
+license = "Apache-2.0"
+publish = false
+
+[lib]
+crate-type = ["cdylib", "lib"]
+name = "matrix_crypto"
+
+[dependencies]
+anyhow = "1.0.57"
+base64 = "0.13.0"
+hmac = "0.12.1"
+http = "0.2.6"
+pbkdf2 = "0.11.0"
+rand = "0.8.5"
+ruma = { version = "0.6.1", features = ["client-api-c"] }
+serde = "1.0.136"
+serde_json = "1.0.79"
+sha2 = "0.10.2"
+thiserror = "1.0.30"
+tracing = "0.1.34"
+tracing-subscriber = { version = "0.3.11", features = ["env-filter"] }
+uniffi = "0.17.0"
+zeroize = { version = "1.3.0", features = ["zeroize_derive"] }
+
+[dependencies.js_int]
+version = "0.2.2"
+features = ["lax_deserialize"]
+
+[dependencies.matrix-sdk-common]
+path = "../matrix-sdk-common"
+version = "0.5.0"
+
+[dependencies.matrix-sdk-crypto]
+path = "../matrix-sdk-crypto"
+version = "0.5.0"
+features = ["qrcode", "backups_v1"]
+
+[dependencies.matrix-sdk-sled]
+path = "../matrix-sdk-sled"
+version = "0.1.0"
+default_features = false
+features = ["crypto-store"]
+
+[dependencies.tokio]
+version = "1.17.0"
+default_features = false
+features = ["rt-multi-thread"]
+
+[dependencies.vodozemac]
+version = "0.2.0"
+
+[build-dependencies]
+uniffi_build = { version = "0.17.0", features = ["builtin-bindgen"] }
+
+[dev-dependencies]
+tempfile = "3.3.0"

crates/matrix-sdk-crypto-ffi/README.md

@@ -0,0 +1,84 @@
+# Uniffi based bindings for the Rust SDK crypto crate.
+
+This crate contains Uniffi based bindings for the `matrix-sdk-crypto` crate. The
+README mainly describes how to build and integrate the bindings into a Kotlin
+based Android project, but the Android specific bits can be skipped if you are
+targeting an x86 Linux project.
+
+## Prerequisites
+
+### Rust
+
+To build the bindings [Rust] will be needed it can be either installed using an
+OS specific package manager or directly with the provided [installer](https://rustup.rs/).
+
+### Android NDK
+
+The Android NDK will be required as well, it can be installed either through
+Android Studio or directly using an [installer](https://developer.android.com/ndk/downloads).
+
+### Configuring Rust for cross compilation
+
+First we'll need to install the Rust target for our desired Android architecture,
+for example:
+
+```
+# rustup target add aarch64-linux-android
+```
+
+This will add support to cross-compile for the aarch64-linux-android target,
+Rust supports many different [targets], you'll have to make sure to pick the
+right one for your device or emulator.
+
+After this is done, we'll have to configure [Cargo] to use the correct linker
+for our target. Cargo is configured using a TOML file that will be found in
+`%USERPROFILE%\.cargo\config.toml` on Windows or `$HOME/.cargo/config` on Unix
+platforms. More details and configuration options for Cargo can be found in the
+official docs over [here](https://doc.rust-lang.org/cargo/reference/config.html).
+
+```
+[target.aarch64-linux-android]
+ar = "NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/ar"
+linker = "NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android30-clang"
+```
+
+## Building
+
+To enable cross compilation for `olm-sys` which builds our `libolm` C library
+we'll need to set the `ANDROID_NDK` environment variable to the location of our
+Android NDK installation.
+
+```
+$ export ANDROID_NDK=$HOME/Android/Sdk/ndk/22.0.7026061/
+```
+
+### Building for a target
+
+The bindings can built for the `aarch64` target with:
+
+```
+$ cargo build --target aarch64-linux-android
+```
+
+After that, a dynamic library can be found in the `target/aarch64-linux-android/debug` directory.
+The library will be called `libmatrix_crypto.so` and needs to be renamed and
+copied into the `jniLibs` directory of your Android project, for Element Android:
+
+```
+$ cp ../../target/aarch64-linux-android/debug/libmatrix_crypto.so \
+     /home/example/matrix-sdk-android/src/main/jniLibs/aarch64/libuniffi_olm.so
+```
+
+## Minimum Supported Rust Version (MSRV)
+
+These crates are built with the Rust language version 2021 and require a minimum compiler version of `1.60`.
+
+## License
+
+[Apache-2.0](https://www.apache.org/licenses/LICENSE-2.0)
+
+[Rust]: https://www.rust-lang.org/
+[installer]: https://rustup.rs/
+[targets]: https://doc.rust-lang.org/nightly/rustc/platform-support.html
+[Cargo]: https://doc.rust-lang.org/cargo/
+[uniffi]: https://github.com/mozilla/uniffi-rs/

crates/matrix-sdk-crypto-ffi/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    uniffi_build::generate_scaffolding("./src/olm.udl").unwrap();
+}

crates/matrix-sdk-crypto-ffi/src/backup_recovery_key.rs

@@ -0,0 +1,157 @@
+use std::{collections::HashMap, iter};
+
+use hmac::Hmac;
+use matrix_sdk_crypto::{
+    backups::OlmPkDecryptionError,
+    store::{CryptoStoreError as InnerStoreError, RecoveryKey},
+};
+use pbkdf2::pbkdf2;
+use rand::{distributions::Alphanumeric, thread_rng, Rng};
+use sha2::Sha512;
+use thiserror::Error;
+use zeroize::Zeroize;
+
+/// The private part of the backup key, the one used for recovery.
+pub struct BackupRecoveryKey {
+    inner: RecoveryKey,
+    passphrase_info: Option<PassphraseInfo>,
+}
+
+/// Error type for the decryption of backed up room keys.
+#[derive(Debug, Error)]
+pub enum PkDecryptionError {
+    /// An internal libolm error happened during decryption.
+    #[error("Error decryption a PkMessage {0}")]
+    Olm(#[from] OlmPkDecryptionError),
+}
+
+/// Error type for the decoding and storing of the backup key.
+#[derive(Debug, Error)]
+pub enum DecodeError {
+    /// An error happened while decoding the recovery key.
+    #[error(transparent)]
+    Decode(#[from] matrix_sdk_crypto::backups::DecodeError),
+    /// An error happened in the storage layer while trying to save the
+    /// decoded recovery key.
+    #[error(transparent)]
+    CryptoStore(#[from] InnerStoreError),
+}
+
+/// Struct containing info about the way the backup key got derived from a
+/// passphrase.
+#[derive(Debug, Clone)]
+pub struct PassphraseInfo {
+    /// The salt that was used during key derivation.
+    pub private_key_salt: String,
+    /// The number of PBKDF rounds that were used for key derivation.
+    pub private_key_iterations: i32,
+}
+
+/// The public part of the backup key.
+pub struct MegolmV1BackupKey {
+    /// The actual base64 encoded public key.
+    pub public_key: String,
+    /// Signatures that have signed our backup key.
+    pub signatures: HashMap<String, HashMap<String, String>>,
+    /// The passphrase info, if the key was derived from one.
+    pub passphrase_info: Option<PassphraseInfo>,
+    /// Get the full name of the backup algorithm this backup key supports.
+    pub backup_algorithm: String,
+}
+
+impl BackupRecoveryKey {
+    const KEY_SIZE: usize = 32;
+    const SALT_SIZE: usize = 32;
+    const PBKDF_ROUNDS: i32 = 500_000;
+
+    /// Create a new random [`BackupRecoveryKey`].
+    #[allow(clippy::new_without_default)]
+    pub fn new() -> Self {
+        Self {
+            inner: RecoveryKey::new()
+                .expect("Can't gather enough randomness to create a recovery key"),
+            passphrase_info: None,
+        }
+    }
+
+    /// Try to create a [`BackupRecoveryKey`] from a base 64 encoded string.
+    pub fn from_base64(key: String) -> Result<Self, DecodeError> {
+        Ok(Self { inner: RecoveryKey::from_base64(&key)?, passphrase_info: None })
+    }
+
+    /// Try to create a [`BackupRecoveryKey`] from a base 58 encoded string.
+    pub fn from_base58(key: String) -> Result<Self, DecodeError> {
+        Ok(Self { inner: RecoveryKey::from_base58(&key)?, passphrase_info: None })
+    }
+
+    /// Create a new [`BackupRecoveryKey`] from the given passphrase.
+    pub fn new_from_passphrase(passphrase: String) -> Self {
+        let mut rng = thread_rng();
+        let salt: String = iter::repeat(())
+            .map(|()| rng.sample(Alphanumeric))
+            .map(char::from)
+            .take(Self::SALT_SIZE)
+            .collect();
+
+        Self::from_passphrase(passphrase, salt, Self::PBKDF_ROUNDS)
+    }
+
+    /// Restore a [`BackupRecoveryKey`] from the given passphrase.
+    pub fn from_passphrase(passphrase: String, salt: String, rounds: i32) -> Self {
+        let mut key = Box::new([0u8; Self::KEY_SIZE]);
+        let rounds = rounds as u32;
+
+        pbkdf2::<Hmac<Sha512>>(passphrase.as_bytes(), salt.as_bytes(), rounds, &mut *key);
+
+        let recovery_key = RecoveryKey::from_bytes(&key);
+
+        key.zeroize();
+
+        Self {
+            inner: recovery_key,
+            passphrase_info: Some(PassphraseInfo {
+                private_key_salt: salt,
+                private_key_iterations: rounds as i32,
+            }),
+        }
+    }
+
+    /// Get the public part of the backup key.
+    pub fn megolm_v1_public_key(&self) -> MegolmV1BackupKey {
+        let public_key = self.inner.megolm_v1_public_key();
+
+        let signatures: HashMap<String, HashMap<String, String>> = public_key
+            .signatures()
+            .into_iter()
+            .map(|(k, v)| (k.to_string(), v.into_iter().map(|(k, v)| (k.to_string(), v)).collect()))
+            .collect();
+
+        MegolmV1BackupKey {
+            public_key: public_key.to_base64(),
+            signatures,
+            passphrase_info: self.passphrase_info.clone(),
+            backup_algorithm: public_key.backup_algorithm().to_owned(),
+        }
+    }
+
+    /// Convert the recovery key to a base 58 encoded string.
+    pub fn to_base58(&self) -> String {
+        self.inner.to_base58()
+    }
+
+    /// Convert the recovery key to a base 64 encoded string.
+    pub fn to_base64(&self) -> String {
+        self.inner.to_base64()
+    }
+
+    /// Try to decrypt a message that was encrypted using the public part of the
+    /// backup key.
+    pub fn decrypt_v1(
+        &self,
+        ephemeral_key: String,
+        mac: String,
+        ciphertext: String,
+    ) -> Result<String, PkDecryptionError> {
+        self.inner.decrypt_v1(ephemeral_key, mac, ciphertext).map_err(|e| e.into())
+    }
+}

crates/matrix-sdk-crypto-ffi/src/device.rs

@@ -0,0 +1,42 @@
+use std::collections::HashMap;
+
+use matrix_sdk_crypto::Device as InnerDevice;
+
+/// An E2EE capable Matrix device.
+pub struct Device {
+    /// The device owner.
+    pub user_id: String,
+    /// The unique ID of the device.
+    pub device_id: String,
+    /// The published public identity keys of the devices
+    ///
+    /// A map from the key type (e.g. curve25519) to the base64 encoded key.
+    pub keys: HashMap<String, String>,
+    /// The supported algorithms of the device.
+    pub algorithms: Vec<String>,
+    /// The human readable name of the device.
+    pub display_name: Option<String>,
+    /// A flag indicating if the device has been blocked, blocked devices don't
+    /// receive any room keys from us.
+    pub is_blocked: bool,
+    /// Is the device locally trusted
+    pub locally_trusted: bool,
+    /// Is our cross signing identity trusted and does the identity trust the
+    /// device.
+    pub cross_signing_trusted: bool,
+}
+
+impl From<InnerDevice> for Device {
+    fn from(d: InnerDevice) -> Self {
+        Device {
+            user_id: d.user_id().to_string(),
+            device_id: d.device_id().to_string(),
+            keys: d.keys().iter().map(|(k, v)| (k.to_string(), v.to_base64())).collect(),
+            algorithms: d.algorithms().iter().map(|a| a.to_string()).collect(),
+            display_name: d.display_name().map(|d| d.to_owned()),
+            is_blocked: d.is_blacklisted(),
+            locally_trusted: d.is_locally_trusted(),
+            cross_signing_trusted: d.is_cross_signing_trusted(),
+        }
+    }
+}

crates/matrix-sdk-crypto-ffi/src/error.rs

@@ -0,0 +1,63 @@
+#![allow(missing_docs)]
+
+use matrix_sdk_crypto::{
+    store::CryptoStoreError as InnerStoreError, KeyExportError, MegolmError, OlmError,
+    SecretImportError as RustSecretImportError, SignatureError as InnerSignatureError,
+};
+use ruma::{IdParseError, OwnedUserId};
+
+#[derive(Debug, thiserror::Error)]
+pub enum KeyImportError {
+    #[error(transparent)]
+    Export(#[from] KeyExportError),
+    #[error(transparent)]
+    CryptoStore(#[from] InnerStoreError),
+    #[error(transparent)]
+    Json(#[from] serde_json::Error),
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum SecretImportError {
+    #[error(transparent)]
+    CryptoStore(#[from] InnerStoreError),
+    #[error(transparent)]
+    Import(#[from] RustSecretImportError),
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum SignatureError {
+    #[error(transparent)]
+    Signature(#[from] InnerSignatureError),
+    #[error(transparent)]
+    Identifier(#[from] IdParseError),
+    #[error(transparent)]
+    CryptoStore(#[from] InnerStoreError),
+    #[error("Unknown device {0} {1}")]
+    UnknownDevice(OwnedUserId, String),
+    #[error("Unknown user identity {0}")]
+    UnknownUserIdentity(String),
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum CryptoStoreError {
+    #[error(transparent)]
+    CryptoStore(#[from] InnerStoreError),
+    #[error(transparent)]
+    OlmError(#[from] OlmError),
+    #[error(transparent)]
+    Serialization(#[from] serde_json::Error),
+    #[error("The given string is not a valid user ID: source {0}, error {1}")]
+    InvalidUserId(String, IdParseError),
+    #[error(transparent)]
+    Identifier(#[from] IdParseError),
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum DecryptionError {
+    #[error(transparent)]
+    Serialization(#[from] serde_json::Error),
+    #[error(transparent)]
+    Identifier(#[from] IdParseError),
+    #[error(transparent)]
+    Megolm(#[from] MegolmError),
+}

crates/matrix-sdk-crypto-ffi/src/lib.rs

@@ -0,0 +1,568 @@
+//! Uniffi based bindings for the `matrix-sdk-crypto` crate.
+//!
+//! This crate can be used to introduce E2EE support into an existing Matrix
+//! client or client library in any of the language targets Uniffi supports.
+
+#![warn(missing_docs)]
+
+mod backup_recovery_key;
+mod device;
+mod error;
+mod logger;
+mod machine;
+mod responses;
+mod users;
+mod verification;
+
+use std::{collections::HashMap, convert::TryFrom, str::FromStr, sync::Arc};
+
+pub use backup_recovery_key::{
+    BackupRecoveryKey, DecodeError, MegolmV1BackupKey, PassphraseInfo, PkDecryptionError,
+};
+pub use device::Device;
+pub use error::{
+    CryptoStoreError, DecryptionError, KeyImportError, SecretImportError, SignatureError,
+};
+use js_int::UInt;
+pub use logger::{set_logger, Logger};
+pub use machine::{KeyRequestPair, OlmMachine};
+pub use responses::{
+    BootstrapCrossSigningResult, DeviceLists, KeysImportResult, OutgoingVerificationRequest,
+    Request, RequestType, SignatureUploadRequest, UploadSigningKeysRequest,
+};
+use ruma::{DeviceId, DeviceKeyAlgorithm, OwnedUserId, RoomId, SecondsSinceUnixEpoch, UserId};
+use serde::{Deserialize, Serialize};
+pub use users::UserIdentity;
+pub use verification::{
+    CancelInfo, ConfirmVerificationResult, QrCode, RequestVerificationResult, Sas, ScanResult,
+    StartSasResult, Verification, VerificationRequest,
+};
+
+/// Struct collecting data that is important to migrate to the rust-sdk
+#[derive(Deserialize, Serialize)]
+pub struct MigrationData {
+    /// The pickled version of the Olm Account
+    account: PickledAccount,
+    /// The list of pickleds Olm Sessions.
+    sessions: Vec<PickledSession>,
+    /// The list of Megolm inbound group sessions.
+    inbound_group_sessions: Vec<PickledInboundGroupSession>,
+    /// The Olm pickle key that was used to pickle all the Olm objects.
+    pickle_key: String,
+    /// The backup version that is currently active.
+    backup_version: Option<String>,
+    // The backup recovery key, as a base58 encoded string.
+    backup_recovery_key: Option<String>,
+    /// The private cross signing keys.
+    cross_signing: CrossSigningKeyExport,
+    /// The list of users that the Rust SDK should track.
+    tracked_users: Vec<String>,
+}
+
+/// A pickled version of an `Account`.
+///
+/// Holds all the information that needs to be stored in a database to restore
+/// an account.
+#[derive(Debug, Deserialize, Serialize)]
+pub struct PickledAccount {
+    /// The user id of the account owner.
+    pub user_id: String,
+    /// The device id of the account owner.
+    pub device_id: String,
+    /// The pickled version of the Olm account.
+    pub pickle: String,
+    /// Was the account shared.
+    pub shared: bool,
+    /// The number of uploaded one-time keys we have on the server.
+    pub uploaded_signed_key_count: i64,
+}
+
+/// A pickled version of a `Session`.
+///
+/// Holds all the information that needs to be stored in a database to restore
+/// a Session.
+#[derive(Debug, Deserialize, Serialize)]
+pub struct PickledSession {
+    /// The pickle string holding the Olm Session.
+    pub pickle: String,
+    /// The curve25519 key of the other user that we share this session with.
+    pub sender_key: String,
+    /// Was the session created using a fallback key.
+    pub created_using_fallback_key: bool,
+    /// The Unix timestamp when the session was created.
+    pub creation_time: String,
+    /// The Unix timestamp when the session was last used.
+    pub last_use_time: String,
+}
+
+/// A pickled version of an `InboundGroupSession`.
+///
+/// Holds all the information that needs to be stored in a database to restore
+/// an InboundGroupSession.
+#[derive(Debug, Deserialize, Serialize)]
+pub struct PickledInboundGroupSession {
+    /// The pickle string holding the InboundGroupSession.
+    pub pickle: String,
+    /// The public curve25519 key of the account that sent us the session
+    pub sender_key: String,
+    /// The public ed25519 key of the account that sent us the session.
+    pub signing_key: HashMap<String, String>,
+    /// The id of the room that the session is used in.
+    pub room_id: String,
+    /// The list of claimed ed25519 that forwarded us this key. Will be empty if
+    /// we directly received this session.
+    pub forwarding_chains: Vec<String>,
+    /// Flag remembering if the session was directly sent to us by the sender
+    /// or if it was imported.
+    pub imported: bool,
+    /// Flag remembering if the session has been backed up.
+    pub backed_up: bool,
+}
+
+/// Error type for the migration process.
+#[derive(thiserror::Error, Debug)]
+pub enum MigrationError {
+    /// Generic catch all error variant.
+    #[error("error migrating database: {error_message}")]
+    Generic {
+        /// The error message
+        error_message: String,
+    },
+}
+
+impl From<anyhow::Error> for MigrationError {
+    fn from(e: anyhow::Error) -> MigrationError {
+        MigrationError::Generic { error_message: e.to_string() }
+    }
+}
+
+/// Migrate a libolm based setup to a vodozemac based setup stored in a Sled
+/// store.
+///
+/// # Arguments
+///
+/// * `data` - The data that should be migrated over to the Sled store.
+///
+/// * `path` - The path where the Sled store should be created.
+///
+/// * `passphrase` - The passphrase that should be used to encrypt the data at
+/// rest in the Sled store. **Warning**, if no passphrase is given, the store
+/// and all its data will remain unencrypted.
+///
+/// * `progress_listener` - A callback that can be used to introspect the
+/// progress of the migration.
+pub fn migrate(
+    mut data: MigrationData,
+    path: &str,
+    passphrase: Option<String>,
+    progress_listener: Box<dyn ProgressListener>,
+) -> anyhow::Result<()> {
+    use matrix_sdk_crypto::{
+        olm::PrivateCrossSigningIdentity,
+        store::{Changes as RustChanges, CryptoStore, RecoveryKey},
+    };
+    use matrix_sdk_sled::CryptoStore as SledStore;
+    use tokio::runtime::Runtime;
+    use vodozemac::{
+        megolm::InboundGroupSession,
+        olm::{Account, Session},
+        Curve25519PublicKey,
+    };
+    use zeroize::Zeroize;
+
+    // The total steps here include all the sessions/inbound group sessions and
+    // additionally some static number of steps:
+    //
+    // 1. opening the store
+    // 2. the Account
+    // 3. the cross signing keys
+    // 4. the tracked users
+    // 5. the final save operation
+    let total_steps = 5 + data.sessions.len() + data.inbound_group_sessions.len();
+    let mut processed_steps = 0;
+    let listener = |progress: usize, total: usize| {
+        progress_listener.on_progress(progress as i32, total as i32)
+    };
+
+    let store = SledStore::open_with_passphrase(path, passphrase.as_deref())?;
+    let runtime = Runtime::new()?;
+
+    processed_steps += 1;
+    listener(processed_steps, total_steps);
+
+    let user_id: Arc<UserId> = (&*parse_user_id(&data.account.user_id)?).into();
+    let device_id: Box<DeviceId> = data.account.device_id.into();
+    let device_id: Arc<DeviceId> = device_id.into();
+
+    let account = Account::from_libolm_pickle(&data.account.pickle, &data.pickle_key)?;
+    let pickle = account.pickle();
+    let identity_keys = Arc::new(account.identity_keys());
+    let pickled_account = matrix_sdk_crypto::olm::PickledAccount {
+        user_id: parse_user_id(&data.account.user_id)?,
+        device_id: device_id.as_ref().to_owned(),
+        pickle,
+        shared: data.account.shared,
+        uploaded_signed_key_count: data.account.uploaded_signed_key_count as u64,
+    };
+    let account = matrix_sdk_crypto::olm::ReadOnlyAccount::from_pickle(pickled_account)?;
+
+    processed_steps += 1;
+    listener(processed_steps, total_steps);
+
+    let mut sessions = Vec::new();
+
+    for session_pickle in data.sessions {
+        let pickle =
+            Session::from_libolm_pickle(&session_pickle.pickle, &data.pickle_key)?.pickle();
+
+        let creation_time = SecondsSinceUnixEpoch(UInt::from_str(&session_pickle.creation_time)?);
+        let last_use_time = SecondsSinceUnixEpoch(UInt::from_str(&session_pickle.last_use_time)?);
+
+        let pickle = matrix_sdk_crypto::olm::PickledSession {
+            pickle,
+            sender_key: Curve25519PublicKey::from_base64(&session_pickle.sender_key)?,
+            created_using_fallback_key: session_pickle.created_using_fallback_key,
+            creation_time,
+            last_use_time,
+        };
+
+        let session = matrix_sdk_crypto::olm::Session::from_pickle(
+            user_id.clone(),
+            device_id.clone(),
+            identity_keys.clone(),
+            pickle,
+        );
+
+        sessions.push(session);
+        processed_steps += 1;
+        listener(processed_steps, total_steps);
+    }
+
+    let mut inbound_group_sessions = Vec::new();
+
+    for session in data.inbound_group_sessions {
+        let pickle =
+            InboundGroupSession::from_libolm_pickle(&session.pickle, &data.pickle_key)?.pickle();
+
+        let pickle = matrix_sdk_crypto::olm::PickledInboundGroupSession {
+            pickle,
+            sender_key: session.sender_key,
+            signing_key: session
+                .signing_key
+                .into_iter()
+                .map(|(k, v)| Ok((DeviceKeyAlgorithm::try_from(k)?, v)))
+                .collect::<anyhow::Result<_>>()?,
+            room_id: RoomId::parse(session.room_id)?,
+            forwarding_chains: session.forwarding_chains,
+            imported: session.imported,
+            backed_up: session.backed_up,
+            history_visibility: None,
+        };
+
+        let session = matrix_sdk_crypto::olm::InboundGroupSession::from_pickle(pickle)?;
+
+        inbound_group_sessions.push(session);
+        processed_steps += 1;
+        listener(processed_steps, total_steps);
+    }
+
+    let recovery_key =
+        data.backup_recovery_key.map(|k| RecoveryKey::from_base58(k.as_str())).transpose()?;
+
+    let cross_signing = PrivateCrossSigningIdentity::empty((*user_id).into());
+    runtime.block_on(cross_signing.import_secrets_unchecked(
+        data.cross_signing.master_key.as_deref(),
+        data.cross_signing.self_signing_key.as_deref(),
+        data.cross_signing.user_signing_key.as_deref(),
+    ))?;
+
+    data.cross_signing.master_key.zeroize();
+    data.cross_signing.self_signing_key.zeroize();
+    data.cross_signing.user_signing_key.zeroize();
+
+    processed_steps += 1;
+    listener(processed_steps, total_steps);
+
+    let tracked_users: Vec<_> = data
+        .tracked_users
+        .into_iter()
+        .map(|u| Ok(((parse_user_id(&u)?), true)))
+        .collect::<anyhow::Result<_>>()?;
+
+    let tracked_users: Vec<_> = tracked_users.iter().map(|(u, d)| (&**u, *d)).collect();
+
+    runtime.block_on(store.save_tracked_users(tracked_users.as_slice()))?;
+
+    processed_steps += 1;
+    listener(processed_steps, total_steps);
+
+    let changes = RustChanges {
+        account: Some(account),
+        private_identity: Some(cross_signing),
+        sessions,
+        inbound_group_sessions,
+        recovery_key,
+        backup_version: data.backup_version,
+        ..Default::default()
+    };
+    runtime.block_on(store.save_changes(changes))?;
+
+    processed_steps += 1;
+    listener(processed_steps, total_steps);
+
+    Ok(())
+}
+
+/// Callback that will be passed over the FFI to report progress
+pub trait ProgressListener {
+    /// The callback that should be called on the Rust side
+    ///
+    /// # Arguments
+    ///
+    /// * `progress` - The current number of items that have been handled
+    ///
+    /// * `total` - The total number of items that will be handled
+    fn on_progress(&self, progress: i32, total: i32);
+}
+
+impl<T: Fn(i32, i32)> ProgressListener for T {
+    fn on_progress(&self, progress: i32, total: i32) {
+        self(progress, total)
+    }
+}
+
+/// An event that was successfully decrypted.
+pub struct DecryptedEvent {
+    /// The decrypted version of the event.
+    pub clear_event: String,
+    /// The claimed curve25519 key of the sender.
+    pub sender_curve25519_key: String,
+    /// The claimed ed25519 key of the sender.
+    pub claimed_ed25519_key: Option<String>,
+    /// The curve25519 chain of the senders that forwarded the Megolm decryption
+    /// key to us. Is empty if the key came directly from the sender of the
+    /// event.
+    pub forwarding_curve25519_chain: Vec<String>,
+}
+
+/// Struct representing the state of our private cross signing keys, it shows
+/// which private cross signing keys we have locally stored.
+#[derive(Debug, Clone)]
+pub struct CrossSigningStatus {
+    /// Do we have the master key.
+    pub has_master: bool,
+    /// Do we have the self signing key, this one is necessary to sign our own
+    /// devices.
+    pub has_self_signing: bool,
+    /// Do we have the user signing key, this one is necessary to sign other
+    /// users.
+    pub has_user_signing: bool,
+}
+
+/// A struct containing private cross signing keys that can be backed up or
+/// uploaded to the secret store.
+#[derive(Deserialize, Serialize)]
+pub struct CrossSigningKeyExport {
+    /// The seed of the master key encoded as unpadded base64.
+    pub master_key: Option<String>,
+    /// The seed of the self signing key encoded as unpadded base64.
+    pub self_signing_key: Option<String>,
+    /// The seed of the user signing key encoded as unpadded base64.
+    pub user_signing_key: Option<String>,
+}
+
+/// Struct holding the number of room keys we have.
+pub struct RoomKeyCounts {
+    /// The total number of room keys.
+    pub total: i64,
+    /// The number of backed up room keys.
+    pub backed_up: i64,
+}
+
+/// Backup keys and information we load from the store.
+pub struct BackupKeys {
+    /// The recovery key as a base64 encoded string.
+    pub recovery_key: String,
+    /// The version that is used with the recovery key.
+    pub backup_version: String,
+}
+
+impl TryFrom<matrix_sdk_crypto::store::BackupKeys> for BackupKeys {
+    type Error = ();
+
+    fn try_from(keys: matrix_sdk_crypto::store::BackupKeys) -> Result<Self, Self::Error> {
+        Ok(Self {
+            recovery_key: keys.recovery_key.ok_or(())?.to_base64(),
+            backup_version: keys.backup_version.ok_or(())?,
+        })
+    }
+}
+
+impl From<matrix_sdk_crypto::store::RoomKeyCounts> for RoomKeyCounts {
+    fn from(count: matrix_sdk_crypto::store::RoomKeyCounts) -> Self {
+        Self { total: count.total as i64, backed_up: count.backed_up as i64 }
+    }
+}
+
+impl From<matrix_sdk_crypto::CrossSigningKeyExport> for CrossSigningKeyExport {
+    fn from(e: matrix_sdk_crypto::CrossSigningKeyExport) -> Self {
+        Self {
+            master_key: e.master_key.clone(),
+            self_signing_key: e.self_signing_key.clone(),
+            user_signing_key: e.user_signing_key.clone(),
+        }
+    }
+}
+
+impl From<CrossSigningKeyExport> for matrix_sdk_crypto::CrossSigningKeyExport {
+    fn from(e: CrossSigningKeyExport) -> Self {
+        matrix_sdk_crypto::CrossSigningKeyExport {
+            master_key: e.master_key,
+            self_signing_key: e.self_signing_key,
+            user_signing_key: e.user_signing_key,
+        }
+    }
+}
+
+impl From<matrix_sdk_crypto::CrossSigningStatus> for CrossSigningStatus {
+    fn from(s: matrix_sdk_crypto::CrossSigningStatus) -> Self {
+        Self {
+            has_master: s.has_master,
+            has_self_signing: s.has_self_signing,
+            has_user_signing: s.has_user_signing,
+        }
+    }
+}
+
+fn parse_user_id(user_id: &str) -> Result<OwnedUserId, CryptoStoreError> {
+    ruma::UserId::parse(user_id).map_err(|e| CryptoStoreError::InvalidUserId(user_id.to_owned(), e))
+}
+
+#[allow(warnings)]
+mod generated {
+    use super::*;
+    include!(concat!(env!("OUT_DIR"), "/olm.uniffi.rs"));
+}
+
+pub use generated::*;
+
+#[cfg(test)]
+mod test {
+    use anyhow::Result;
+    use serde_json::{json, Value};
+    use tempfile::tempdir;
+
+    use super::MigrationData;
+    use crate::{migrate, OlmMachine};
+
+    #[test]
+    fn android_migration() -> Result<()> {
+        let data: Value = json!({
+            "account":{
+               "user_id":"@ganfra146:matrix.org",
+               "device_id":"DEWRCMENGS",
+               "pickle":"FFGTGho89T3Xgd56l+EedOPV37s09RR8aYnS9305qPKF66LG+ly29YpCibjJOvkwm0dZwN9A2bOH/z7WscriqwZn/p0GE6YSNwLzffCy5iROzYzpYzFe0HtiyJmCQWCezvLc5lHV8YsfD00C1pKGX2R9M1wwp3/n4/3VjtTyPsdnmtwAPu4WdcPSkisCaQ3a6JaSKqv8zYzUjnpzgcpXHvPUR5d5+TzXgrVz3BeCOe8NEOWIW6xYUxFtGteYP0BczOkkJ22t7Css0tSMSrYgCll4zZUGNrd6D9b/z7KwcDnb978epsZ16DcZ/aaTxPdM5uDIkHgF/qHWerfxcaqsqs4EQfJdSgOTeqhjHBw1k0uWF2bByJLK+n7sGkYXEAuTzc4+0XvSFvu3Qp+1bHZuT7QejngRZzyxznORyBxd8la3/JjeJlehSK80OL7zSmohoYZD59S6i3tFWfopjQThJ0/eIyVOhEN/c3tfIcVr3lFEQeokgpCRNOVldhPcQWq994NHaL7jtb6yhUqT1gShY4zYayFL/VRz6nBSXXYwzrC9jho67knqXSri3lIKYevP9aOi384IvzbkinQdumc804dYwiCbs5hZppfEnfhfgiDDm+kVrJ9WaPRF4SySCTlS8jdGmBeL2CfCQ5IcZ5nK6X7tZM3tmtYwva0RuQiTNltp3XTfbMa0EoaEBximv25165hFTpzrWgoszBTpZPfgsMuWENWBcIX4AcLSk0CJ0qzPDeUwvmRcFStstGYV4drs5u5HEqovFSI48CoHPSEZfwwERCI4c/0efZ0CVEfnm8VcMv3AbnAfedD7v3QNdVwWOEhz/fGR76BQi2WjZP4MWvYRJ/vsLO5hcVWUvaJGQs5kANUFZMWuJQeJv3DmkV9kKKXnyfFUerlQ4Uk/5tp2mXiG+adHjuRp/Eeh5V/biCcIaX3rNuIY6MJaPz6SOwlFe79MMBaNwaS3j4Kh/Aq9BRw0QXdjO4CqMI4p2xCE1N5QTPdeaRTHTZ3r7mLkHX3FpZMxitc8vDl9L2FRoSOMMh/sRD1boBCkjrsty9rvTUGYY3li05jBuTXnYMjA4zj79dC9TGo4g+/wi+h537EhtP5+170LwqnIzfHt8yfjbsMMC7iwLpC1C57sTwxpMkNo3nQEvZOfqCxjq+ihiGuL9iN5lSstu9/C4qP2tQll86ASXf1axxRZQlUB0hlLHbEW6/7O7xOU6FTs4yXAZC04souRkggmfhDzZ9kQmN/zRTbqlATFI7l9/0VGxwLOVnCMUhgiDX5yL8CYK9I4ENMLf5zOuO6P3GbYISjEoHC7fUOzQ6OwGgLyI0wCEVdSJzQcdKh+W15VV+eDjhE/qEJHQWx024hTQFTKYHlDn95+lMmRI9BJLP1HU2JW6onVWsTsE5zSYu9jLj739EKfV4gS/pWzoQDRa7a9ZG6+m+RrwyJhCso3gkUekDNobhFlDX6YeH+Btj91N0uS3F9qr8lbo491s/z2fNV42zT4NYObzgrAYDQAV/2WYF8tXtxLV/Jzk8AMmyr/cfNaT2dXxVJKWq+nN2BYHBmg9CCWPJ2aB/1WWIcHfcDOlngtH991gP6246f/DEaVC/Ayxz7bPtSH5tlZ4Xbpc2P4BYxaRp/yxhhQ2C9H2I/PTt3mnNNgky/t8PZrN3W5+eiSVE9sONF8G3mYsa4XFqM+KxfbPUqsrEnrRBmvmJ250hpTPkFcIF775RvvRRKALXdlTKs+S4HKDW7KoP0Dm9+r4RlO0UHpWND9w0WSMItvWQyo0VViXJgZfBjYtWDoO0Ud+Kc7PLWNX6RUKY7RlDjXadJTC4adH6CN3UBC/ouqqfTrYvPOkyd2oKf4RLjEVcFAUIftFbLy+WBcWv8072nnAFJIlm3CxGq++80TyjqFR45P+qfIJavxQNIt5zhHPfMgHjX27OA3+l7rHDxqfMLBPxhtARwlyF+qx1IJiSWbmlHkdz2ylD9unoLSpf+DmmFvvgTj+3EEP4bY2jA/t91XFeG3uaTQSy3ryDvhbX21U7G2HGOEl9rCkmz+hG0YRB/6KxZZ0eMIDr7OWfpPEuHV8oYwDNYbsT9zCGsR1hHxBJtdo60b36mjMemtf761DhJ/oQZ4eU738yzx1hvVS3aCJsfyp70H5u+pUjgrA565uG2lEMNLu4T4NFVw0UdrVudyrhmT8P7vF4v+mR4pp+OzRbLf8AtZrKmHlMqRst+/wOHUHug/Tpz6EwZPDWGiQyFyPUkjHWW7ACouegBFOWFabsk+zCDhyxoSNrSMCtdB1L+qK72jRPGOvXk8p/1kBOIJfAjaK1ZWz8hTc30hOSWYxkRP296zPHiQF0ibNYSPNZ9tNxgq9nV/cEQ68TsNr3SULfDr0TSjCPf4AfmJ0k1k5xphSYv/TtGIbjg/9yGVFqclg4Y/6rrfkApbx36PQEBNxLiRsZ4hGpCfVU6h0jOekk8TV6CAguXVX/G31UqsAEa4sOD2g10Ir+5JD7bdd3JE/999kHGdiCqc0DNcgSqWYbq2QYwrN/mb+mMUbiQSNMcc34kK1n+7dGxppnt7YN7UsJqBWJdH0Lw1Epxi11ViTeVma9bqioJYXi6N5exdpZTT7KmcGYFsoTqO958EX6AppgcML7N9oP3TO8qSgCpV3Bbbemq4bvjV43aM6Rdx17pC4GZo0jjU97p4K8jE4PvgoHlYkuPwSJDOSAdnYPh+Inq/vCk48UfIlup0ATJFVUXD7uf84v9roZSwZPXZ5j/88+MkHBIJwPv8cugmz5uN2EuBW5IScMuEqG7Cmk72SU3/QA39G79S0Gpw7iPhTos5LXxhfvohGcnSaNEvfNeecQf7fpVciTdHwuvcgqJizUKpSFg2P+LDBiO44mJD15RNAaT37Rrj5P06YITO4PDj+FMdc6gx+JQUFbcSRhScE/0gfsVm0P1BYIH5q0k/QDgEVoerf/n19lITTzPib1F2OHP4hyF3BEq1pd9NwuPhhsVVqTVTK5MzFwFIOH7cwJyY7aBykmsWBavdb2J7UA5wjKqMHl1auUGPlNL+lZjqG4tw05bchtFAF+PGWQXJhJCtRSkkzTOCrLRyYyyI9mWYEjoc23cGLanlIs7WA1Nd0Jz+5RSNlf9Gtnd65yQp/W1eqY6yzURPHUUa7FrynyORmjaR9adT9utSQkXy8++IeDNzhMtFr+SqQ/gKECLe0GeuyTs6E5bImUtqpN+xopBXnEeq8wp+bvLf76d98qPE5ibTRwlsSyCE4c1Y7vrJrlc15Yc2R9ciIuKUS8rUKLSdGBFe/TD4R3cPhCKAnnRLGWnJiPPgxoTVwHVZMISdsAjNaWblBmiAOzFcu7443d3PCLyXVcfR9xgvW51HTumo91t5Qyx4HIXGoZxayZYFm2hrhSlieUqLnDL2j2gYgGU5NGoQl4OnEY2QqobpRUF4xJ4HhLzYbLrBeXmTDPvj0MasC3kKsRlm/HrsRRWZ2iPSMw9601tLvDfyjG53ddPISiVNnkdXcaAN5np7dwipdBOC1s4a0sEmKakNbkkDb8LsGBNte/g6UYs5yYaKr0bnXlDjMCznHQa7pypBjE7S55T3UeRpwo3IvZ1tfIGdb+z9RIA/PDvUksxJ3Xq3lqtZzkZJF5aeedfIOekGS/G0LiCSYsELgRceH5veknHqoGoL6xi4Q6/VjmfpZVXT19bDcTNtaR9Dlaq4LDjpQl9rl5C3O/X1hgADvJUuINCiLrD114sLY1DG/TDXE0sp+TK7utnjLAoHuAuj+6anY5vN66CSbwyUNmvo+m8li/AMkRYdtSDoPWkV7Y1ixMBPcua0Llwn2HSKKwnCjvhDIDIIVwbWwb1s6b9cztH81WF5RWUgFujewPvTElM1Sy10y7BcZohKw28uLRFVsKunc9yX2PiQoTSB4PHBHRA4U5dEQV3GHQJ93nee7VT3oeQPMVebWhuhOhi34Z33LQajzpCF3OjIbJb0tOPP6L6N/ODqkNsYViI3kgCnkNhexadOuGFWIqen2Q8iv2uOZWbPirt0YEeKZIk2dpND07L8Q3OsoQCk2rjpnw9LuFrjgu7gN9gFyPq25HJRBn7PM/lS60DF+xVkJq94PwN+CiZWC43SVcBGx65DFZIs/N78MZCUzZbFlsS7FsIrDJt878cp9eZdq/Ai4LZhL8QYHpVUrQxRxZGSqooA755N6nOxw66JkA1VPnjECCMgoNNtWox0JzhMe8PBdh2ZliXf8yQ6/eTvsG6FD84F+49pc7m0L99pfWHb9ClyO3KRHscp/MOIC1MJmqoB4dNxV20U+z8/lSTIvcmM8DiaAZj/yxlst90drlGydlyPjQzYd/XtIYcO5gHoeD1KUCZRapE5dkyk5vh97WZJn/JkR8hsslU3D6x3rNGwJbQVRu0IiA3PpeAQNZBNAJHHfv8IzIYxPhMJdYq0YqLIGSUYu87D04cDOxJY7hgawYs+ExOWb7XkbpuRoITQd8zpwVDFlSCS+wFO+qah3Vn8RBTc6cXHO5xRWfUNj+NrEtPdVmax+9EXqXtHQyFpxaauvL96RH+mGwpKHOk3aisXbZ6gLE2mF4egGjjJOIJdHyb2ZR+kj+4GIvkoBwipDgUfr4UBXY8pvFxQOxRgtI4LgOY9Z1Aco7Mwp6qi1KoMFJW8d+gJwsgM3cMsyEeYH1n/mdpJW6VDbIWzOHkP5n+OKKNm2vJTkQFFwF9eOtGy9fNBtS4qo4jvOUJnnAPsrPbGMbBYd1dMC3daHLEwvIKCAVBn7q1Z2c4zAD5eEoY0EwZj/j8x8lGQ8TswFT81ZotW7ZBDai/YtV8mkGfuaWJRI5yHc/bV7GWLF+yrMji/jicBF5jy2UoqwxseqjgTut49FRgBH3h1qwnfYbXD3FvQljyAAgBCiZV726pFRG+sZv0FjDbq0iCKILVSEUDZgmQ",
+               "shared":true,
+               "uploaded_signed_key_count":50
+
+            },
+            "sessions":[
+               {
+                  "pickle":"cryZlFaQv0hwWe6tTgv75RExFKGnC8tMHBXJYMHOw4s+SdrKUYAMUdGcYD7QukrPklEOy7fJho9YGK/jV04QdA8JABiOfD+ngJTR4V8eZdmDuG08+Q5EL79V81hQwU2fKndP0y/9nAXPUIADYq0Zrg4EsOnXz7aE+hAeBAm0IBog1s8RYUvynZ15uwjbd/OTLP+gpqpX33DwVg2leiBkQetiUSpOpZCuQ8CcZwIA0MoGCqvaT7h76VHX9JxJx+2fCMhsJMx1nhd99WJH1W9ge5CtdbC4KUP92OSxIrPOnMrNcOPJPp/paZP+HFNQ3PDL+z8pGKXmCnrXGSbd7iPHurPYESrVkBzr",
+                  "sender_key":"WJ6Ce7U67a6jqkHYHd8o0+5H4bqdi9hInZdk0+swuXs",
+                  "created_using_fallback_key":false,
+                  "creation_time":"1649425011424",
+                  "last_use_time":"1649425011424"
+               },
+               {
+                  "pickle":"cryZlFaQv0hwWe6tTgv75RExFKGnC8tMHBXJYMHOw4t2W/lowyrV6SXVZp+uG59im0AAfNSKjhjZuiOpQlX7MS+AOJkCNvyujJ2g3KSjLZ94IkoHxkBDHLWSjwaLPu40rfOzJPDpm0XZsR6bQrsxKOmXLGEw2qw5jOTouzMVL2gvuuTix97nSYSU8j3XvTMRUoh0AF/tUpRLcvEFZeGrdUYmTMlyTv4na+FVUalUZ+jrk8t1/sM99JNq3SY1IBSjrBq/0rCOHieiippz0sw2fe2b87id4rqj1g3R9w2MWTWEdOz3ugjMGYF1YDBQZA1tJZ/hmgppk2AU2xKQXE2X3DgSC6fC66D4",
+                  "sender_key":"RzRROfmHNlBfzxnNCUYBfn/5oZNQ11XYjDg59hS+mV0",
+                  "created_using_fallback_key":false,
+                  "creation_time":"1649425011503",
+                  "last_use_time":"1649425011503"
+               },
+               {
+                  "pickle":"cryZlFaQv0hwWe6tTgv75RExFKGnC8tMHBXJYMHOw4titbL3SS12PYHpcBPJc6hXnOnZXqrjtjYOD545fck+3utEo8cqqwWubc9tsvxGW3tOWPttLBdAW30Vn8V1M8ebqVCNVWEAb1GKjV4ni8xG7G9SlEcCjLjnF4lJpddSZkqVMFoN0ITr9aSz/eJwXpc3HLreUFXwc8LuQp7krQ4Vt1e5EE/klduqsdurZf5V14RHsmWz2lKjt7nVgtIz/dhtF5F/sGJdg8kCGaHIMSbGAPuPPpa4/Laicb/5otrYt4pg4W4KdFpSGJIcvUQNjXaOZMx3cu/RPJIOyNhx7whG1QiYAUBqAJvr",
+                  "sender_key":"IXSZugAHig1v8MowE1jxi2wDDDfuZBeJynHlegJVwUc",
+                  "created_using_fallback_key":false,
+                  "creation_time":"1649425011566",
+                  "last_use_time":"1649425011566"
+               },
+               {
+                  "pickle":"SmkDiFZjNukiarQ7XHQo25FILHsuhNOnxy56cMSQU/Y71jaGbJes4YrvN4Dfy4RSONfejEDXDkbW2JudlHHRP/rWEmnfJiGbK6ArbrG2puqIZgOecPnOUgPfCisr49p1Gmf36dPaO5lm/ZSrngfSoxahoeJJE/CcJN98sYM15XytRk2LBwc+CyYDqr4V1qxfsBt6tzJ4+tsAZeRdD0UtipQgysgH56o8N7nKTCkaZz5lfpYCl3FEgwXpLJ0MGQvtQmbORFvOLqR1jZ/EbmNGKiqDDIYsqG0sf78ii1jqfpLDBXLuYDccsg",
+                  "sender_key":"EB9SC4jVAydKhM6/GcwMc9biKwVNywqW3TerNTrtb1M",
+                  "created_using_fallback_key":false,
+                  "creation_time":"1649542063182",
+                  "last_use_time":"1649542063182"
+               }
+            ],
+            "inbound_group_sessions":[
+               {
+                  "pickle":"KoA0sxDNQ7lz0vylU9zlmar0VCVQRMCfRrIfTh1bdMhlAgy8/D2ToT+oKRaKy1HiW6H862bzdpgxprlseSjmip9OfLbIuyd2YZcEinwc2666oEI/hpk4oTlE61uE1M+ThfdFf41yGCmaAP7mhjwF234ZrZ6i/F/qx42TLQ8Unc30wDJaJgyheO5eW85SD/0g0cdg2WnEKrx2/wl7Vg/YijT3JMDZ+OsdfJfSZtxBNjlG+PQ/9D31qb1eHfaovc8vFZh5QLfAFg/5rBrF1PhRsC7xOAZbUNdrTbvypNfMM4mznf84C2SzZRSMeAfg5v/YticM3Keg4eHuEj1WO9DrmRXYl6b/pITdf1xuk5euVT0pyxJpXmq41AoAZKAo1l94HGy1LG1RpruD1uQPhiponh5PGHSOf43Q",
+                  "sender_key":"vJfH7wiYmGos3C8U1XcJ//YWSmkueAYqrmUA6/ukfAM",
+                  "signing_key":{
+                     "ed25519":"JGgPQRuYj3ScMdPS+A0P+k/1qS9Hr3qeKXLscI+hS78"
+                  },
+                  "room_id":"!AZkqtjvtwPAuyNOXEt:matrix.org",
+                  "forwarding_chains":[
+                  ],
+                  "imported":true,
+                  "backed_up":true
+               },
+               {
+                  "pickle":"9RF6GBu9CvjZZx2hxIlw2gMdKs36LFhXhLTHAPrLSjT2OTbeE/jK263+iiFdSpF7Cblp/lXzljPKJN6sL8JGzoT7ssYh56nI0kKsp7/y88z+tTOH/5NYYTmZzHYw6yy4Cmaxh0pdHDs+RQpSSIe9jhF/EJJna5jcKYXxDY52m8H4LECQzVuDlYfblCr9zoYWhQrVhiRDGy7eLhk4X6Rp0Yoek4YUKcCQArDfZ/Vf43qfHUpOJgRpm5Oyj42HA/j4xZBb5U0Fmo6YHRPt0/KuWrDfpgJSGiN0zza7641IfADg8f3WdhlPAWMyri7k4vOZMBjlwFNcMpc0wM2TaTmbi2zqXEKZy9Oh/eJqBapFx0oNWaQ1VQ++iXxGUbZhwy7x2vd6UkqUTwYeym+aP23ee3TCtnNWN0aC",
+                  "sender_key":"EB9SC4jVAydKhM6/GcwMc9biKwVNywqW3TerNTrtb1M",
+                  "signing_key":{
+                     "ed25519":"1NXa5GyJ+p2ruAClEque+TL1VktrBzMW4dZFNfNGrvc"
+                  },
+                  "room_id":"!CWLUCoEWXSFyTCOtfL:matrix.org",
+                  "forwarding_chains":[],
+                  "imported":true,
+                  "backed_up":true
+               }
+            ],
+            "pickle_key":"\u{0011}$xJ_N8$>{\u{0005}iJoF03eBVt\u{000e}rUU\\,GYc7J",
+            "backup_version":"3",
+            "backup_recovery_key":"EsTHScmRV5oT1WBhe2mj2Gn3odeYantZ4NEk7L51p6L8hrmB",
+            "cross_signing":{
+               "master_key":"trnK/dBv/M2x2zZt8lnORHQqmFHWvjYE6rdlAONRUPY",
+               "self_signing_key":"SJhsj9jXC4hxhqS/1B3RZ65zWMHuF+1fUjWHrzVRh6w",
+               "user_signing_key":"LPYrV11T9Prm4ZIUxrq2a8Y/F64R1+NaGNyo6GlXjGg"
+            },
+            "tracked_users":[
+               "@ganfra146:matrix.org",
+               "@this-is-me:matrix.org",
+               "@Amandine:matrix.org",
+               "@ganfra:matrix.org"
+            ]
+        });
+
+        let migration_data: MigrationData = serde_json::from_value(data)?;
+
+        let dir = tempdir()?;
+        let path =
+            dir.path().to_str().expect("Creating a string from the tempdir path should not fail");
+
+        migrate(migration_data, path, None, Box::new(|_, _| {}))?;
+
+        let machine = OlmMachine::new("@ganfra146:matrix.org", "DEWRCMENGS", path, None)?;
+
+        assert_eq!(
+            machine.identity_keys()["ed25519"],
+            "JGgPQRuYj3ScMdPS+A0P+k/1qS9Hr3qeKXLscI+hS78"
+        );
+
+        let room_keys = machine.runtime.block_on(machine.inner.export_keys(|_| true))?;
+        assert_eq!(room_keys.len(), 2);
+
+        let cross_signing_status = machine.cross_signing_status();
+        assert!(cross_signing_status.has_master);
+        assert!(cross_signing_status.has_user_signing);
+        assert!(cross_signing_status.has_self_signing);
+
+        let backup_keys = machine.get_backup_keys()?;
+        assert!(backup_keys.is_some());
+
+        Ok(())
+    }
+}

crates/matrix-sdk-crypto-ffi/src/logger.rs

@@ -0,0 +1,56 @@
+use std::{
+    io::{Result, Write},
+    sync::{Arc, Mutex},
+};
+
+use tracing_subscriber::{fmt::MakeWriter, EnvFilter};
+
+/// Trait that can be used to forward Rust logs over FFI to a language specific
+/// logger.
+pub trait Logger: Send {
+    /// Called every time the Rust side wants to post a log line.
+    fn log(&self, log_line: String);
+    // TODO add support for different log levels, do this by adding more methods
+    // to the trait.
+}
+
+impl Write for LoggerWrapper {
+    fn write(&mut self, buf: &[u8]) -> Result<usize> {
+        let data = String::from_utf8_lossy(buf).to_string();
+        self.inner.lock().unwrap().log(data);
+
+        Ok(buf.len())
+    }
+
+    fn flush(&mut self) -> Result<()> {
+        Ok(())
+    }
+}
+
+impl MakeWriter<'_> for LoggerWrapper {
+    type Writer = LoggerWrapper;
+
+    fn make_writer(&self) -> Self::Writer {
+        self.clone()
+    }
+}
+
+#[derive(Clone)]
+pub struct LoggerWrapper {
+    inner: Arc<Mutex<Box<dyn Logger>>>,
+}
+
+/// Set the logger that should be used to forward Rust logs over FFI.
+pub fn set_logger(logger: Box<dyn Logger>) {
+    let logger = LoggerWrapper { inner: Arc::new(Mutex::new(logger)) };
+
+    let filter = EnvFilter::from_default_env().add_directive(
+        "matrix_sdk_crypto=trace".parse().expect("Can't parse logging filter directive"),
+    );
+
+    let _ = tracing_subscriber::fmt()
+        .with_writer(logger)
+        .with_env_filter(filter)
+        .without_time()
+        .try_init();
+}

crates/matrix-sdk-crypto-ffi/src/olm.udl

@@ -0,0 +1,483 @@
+namespace olm {
+    void set_logger(Logger logger);
+    [Throws=MigrationError]
+    void migrate(
+        MigrationData data,
+        [ByRef] string path,
+        string? passphrase,
+        ProgressListener progress_listener
+    );
+};
+
+[Error]
+interface MigrationError {
+  Generic(string error_message);
+};
+
+callback interface Logger {
+    void log(string log_line);
+};
+
+callback interface ProgressListener {
+    void on_progress(i32 progress, i32 total);
+};
+
+[Error]
+enum PkDecryptionError {
+    "Olm",
+};
+
+[Error]
+enum KeyImportError {
+    "Export",
+    "CryptoStore",
+    "Json",
+};
+
+[Error]
+enum SignatureError {
+    "Signature",
+    "Identifier",
+    "CryptoStore",
+    "UnknownDevice",
+    "UnknownUserIdentity",
+};
+
+[Error]
+enum SecretImportError {
+    "Import",
+    "CryptoStore",
+};
+
+
+[Error]
+enum CryptoStoreError {
+    "CryptoStore",
+    "OlmError",
+    "Serialization",
+    "Identifier",
+    "InvalidUserId",
+};
+
+[Error]
+enum DecryptionError {
+    "Identifier",
+    "Serialization",
+    "Megolm",
+};
+
+dictionary DeviceLists {
+    sequence<string> changed;
+    sequence<string> left;
+};
+
+dictionary KeysImportResult {
+    i64 imported;
+    i64 total;
+    record<DOMString, record<DOMString, sequence<string>>> keys;
+};
+
+dictionary DecryptedEvent {
+    string clear_event;
+    string sender_curve25519_key;
+    string? claimed_ed25519_key;
+    sequence<string> forwarding_curve25519_chain;
+};
+
+dictionary Device {
+    string user_id;
+    string device_id;
+    record<DOMString, string> keys;
+    sequence<string> algorithms;
+    string? display_name;
+    boolean is_blocked;
+    boolean locally_trusted;
+    boolean cross_signing_trusted;
+};
+
+[Enum]
+interface UserIdentity {
+    Own(
+        string user_id,
+        boolean trusts_our_own_device,
+        string master_key,
+        string self_signing_key,
+        string user_signing_key
+    );
+    Other(
+        string user_id,
+        string master_key,
+        string self_signing_key
+    );
+};
+
+dictionary CrossSigningStatus {
+    boolean has_master;
+    boolean has_self_signing;
+    boolean has_user_signing;
+};
+
+dictionary CrossSigningKeyExport {
+    string? master_key;
+    string? self_signing_key;
+    string? user_signing_key;
+};
+
+dictionary UploadSigningKeysRequest {
+    string master_key;
+    string self_signing_key;
+    string user_signing_key;
+};
+
+dictionary BootstrapCrossSigningResult {
+   UploadSigningKeysRequest upload_signing_keys_request;
+   SignatureUploadRequest signature_request;
+};
+
+dictionary CancelInfo {
+    string cancel_code;
+    string reason;
+    boolean cancelled_by_us;
+};
+
+dictionary StartSasResult {
+    Sas sas;
+    OutgoingVerificationRequest request;
+};
+
+dictionary Sas {
+    string other_user_id;
+    string other_device_id;
+    string flow_id;
+    string? room_id;
+    boolean we_started;
+    boolean has_been_accepted;
+    boolean can_be_presented;
+    boolean supports_emoji;
+    boolean have_we_confirmed;
+    boolean is_done;
+    boolean is_cancelled;
+    CancelInfo? cancel_info;
+};
+
+dictionary ScanResult {
+    QrCode qr;
+    OutgoingVerificationRequest request;
+};
+
+dictionary QrCode {
+    string other_user_id;
+    string other_device_id;
+    string flow_id;
+    string? room_id;
+    boolean we_started;
+    boolean other_side_scanned;
+    boolean has_been_confirmed;
+    boolean reciprocated;
+    boolean is_done;
+    boolean is_cancelled;
+    CancelInfo? cancel_info;
+};
+
+dictionary VerificationRequest {
+    string other_user_id;
+    string? other_device_id;
+    string flow_id;
+    string? room_id;
+    boolean we_started;
+    boolean is_ready;
+    boolean is_passive;
+    boolean is_done;
+    boolean is_cancelled;
+    CancelInfo? cancel_info;
+    sequence<string>? their_methods;
+    sequence<string>? our_methods;
+
+};
+
+dictionary RequestVerificationResult {
+    VerificationRequest verification;
+    OutgoingVerificationRequest request;
+};
+
+dictionary ConfirmVerificationResult {
+    sequence<OutgoingVerificationRequest> requests;
+    SignatureUploadRequest? signature_request;
+};
+
+[Enum]
+interface Verification {
+    SasV1(Sas sas);
+    QrCodeV1(QrCode qrcode);
+};
+
+dictionary KeyRequestPair {
+    Request? cancellation;
+    Request key_request;
+};
+
+[Enum]
+interface OutgoingVerificationRequest {
+    ToDevice(string request_id, string event_type, string body);
+    InRoom(string request_id, string room_id, string event_type, string content);
+};
+
+[Enum]
+interface Request {
+    ToDevice(string request_id, string event_type, string body);
+    KeysUpload(string request_id, string body);
+    KeysQuery(string request_id, sequence<string> users);
+    KeysClaim(string request_id, record<DOMString, record<DOMString, string>> one_time_keys);
+    KeysBackup(string request_id, string version, string rooms);
+    RoomMessage(string request_id, string room_id, string event_type, string content);
+    SignatureUpload(string request_id, string body);
+};
+
+dictionary SignatureUploadRequest {
+    string body;
+};
+
+enum RequestType {
+    "KeysQuery",
+    "KeysClaim",
+    "KeysUpload",
+    "ToDevice",
+    "SignatureUpload",
+    "KeysBackup",
+    "RoomMessage",
+};
+
+interface OlmMachine {
+    [Throws=CryptoStoreError]
+    constructor(
+        [ByRef] string user_id,
+        [ByRef] string device_id,
+        [ByRef] string path,
+        string? passphrase
+    );
+
+    record<DOMString, string> identity_keys();
+    string user_id();
+    string device_id();
+
+    [Throws=CryptoStoreError]
+    string receive_sync_changes([ByRef] string events,
+                              DeviceLists device_changes,
+                              record<DOMString, i32> key_counts,
+                              sequence<string>? unused_fallback_keys);
+    [Throws=CryptoStoreError]
+    sequence<Request> outgoing_requests();
+    [Throws=CryptoStoreError]
+    void mark_request_as_sent(
+        [ByRef] string request_id,
+        RequestType request_type,
+        [ByRef] string response
+    );
+
+    [Throws=DecryptionError]
+    DecryptedEvent decrypt_room_event([ByRef] string event, [ByRef] string room_id);
+    [Throws=CryptoStoreError]
+    string encrypt([ByRef] string room_id, [ByRef] string event_type, [ByRef] string content);
+
+    [Throws=CryptoStoreError]
+    UserIdentity? get_identity([ByRef] string user_id);
+    [Throws=SignatureError]
+    SignatureUploadRequest verify_identity([ByRef] string user_id);
+    [Throws=CryptoStoreError]
+    Device? get_device([ByRef] string user_id, [ByRef] string device_id);
+    [Throws=CryptoStoreError]
+    void mark_device_as_trusted([ByRef] string user_id, [ByRef] string device_id);
+    [Throws=SignatureError]
+    SignatureUploadRequest verify_device([ByRef] string user_id, [ByRef] string device_id);
+    [Throws=CryptoStoreError]
+    sequence<Device> get_user_devices([ByRef] string user_id);
+
+    [Throws=CryptoStoreError]
+    boolean is_user_tracked([ByRef] string user_id);
+    void update_tracked_users(sequence<string> users);
+    [Throws=CryptoStoreError]
+    Request? get_missing_sessions(sequence<string> users);
+    [Throws=CryptoStoreError]
+    sequence<Request> share_room_key([ByRef] string room_id, sequence<string> users);
+
+    [Throws=CryptoStoreError]
+    void receive_unencrypted_verification_event([ByRef] string event, [ByRef] string room_id);
+    sequence<VerificationRequest> get_verification_requests([ByRef] string user_id);
+    VerificationRequest? get_verification_request([ByRef] string user_id, [ByRef] string flow_id);
+    Verification? get_verification([ByRef] string user_id, [ByRef] string flow_id);
+
+    [Throws=CryptoStoreError]
+    VerificationRequest? request_verification(
+        [ByRef] string user_id,
+        [ByRef] string room_id,
+        [ByRef] string event_id,
+        sequence<string> methods
+    );
+    [Throws=CryptoStoreError]
+    string? verification_request_content(
+        [ByRef] string user_id,
+        sequence<string> methods
+    );
+    [Throws=CryptoStoreError]
+    RequestVerificationResult? request_self_verification(sequence<string> methods);
+    [Throws=CryptoStoreError]
+    RequestVerificationResult? request_verification_with_device(
+        [ByRef] string user_id,
+        [ByRef] string device_id,
+        sequence<string> methods
+    );
+
+    OutgoingVerificationRequest? accept_verification_request(
+        [ByRef] string user_id,
+        [ByRef] string flow_id,
+        sequence<string> methods
+    );
+
+    [Throws=CryptoStoreError]
+    ConfirmVerificationResult? confirm_verification([ByRef] string user_id, [ByRef] string flow_id);
+    OutgoingVerificationRequest? cancel_verification(
+        [ByRef] string user_id,
+        [ByRef] string flow_id,
+        [ByRef] string cancel_code
+    );
+
+    [Throws=CryptoStoreError]
+    StartSasResult? start_sas_with_device([ByRef] string user_id, [ByRef] string device_id);
+    [Throws=CryptoStoreError]
+    StartSasResult? start_sas_verification([ByRef] string user_id, [ByRef] string flow_id);
+    OutgoingVerificationRequest? accept_sas_verification([ByRef] string user_id, [ByRef] string flow_id);
+    sequence<i32>? get_emoji_index([ByRef] string user_id, [ByRef] string flow_id);
+    sequence<i32>? get_decimals([ByRef] string user_id, [ByRef] string flow_id);
+
+    [Throws=CryptoStoreError]
+    QrCode? start_qr_verification([ByRef] string user_id, [ByRef] string flow_id);
+    ScanResult? scan_qr_code([ByRef] string user_id, [ByRef] string flow_id, [ByRef] string data);
+    string? generate_qr_code([ByRef] string user_id, [ByRef] string flow_id);
+
+    [Throws=DecryptionError]
+    KeyRequestPair request_room_key([ByRef] string event, [ByRef] string room_id);
+
+    [Throws=CryptoStoreError]
+    string export_keys([ByRef] string passphrase, i32 rounds);
+    [Throws=KeyImportError]
+    KeysImportResult import_keys(
+        [ByRef] string keys,
+        [ByRef] string passphrase,
+        ProgressListener progress_listener
+    );
+    [Throws=KeyImportError]
+    KeysImportResult import_decrypted_keys(
+        [ByRef] string keys,
+        ProgressListener progress_listener
+    );
+    [Throws=CryptoStoreError]
+    void discard_room_key([ByRef] string room_id);
+
+    CrossSigningStatus cross_signing_status();
+    [Throws=CryptoStoreError]
+    BootstrapCrossSigningResult bootstrap_cross_signing();
+    CrossSigningKeyExport? export_cross_signing_keys();
+    [Throws=SecretImportError]
+    void import_cross_signing_keys(CrossSigningKeyExport export);
+    [Throws=CryptoStoreError]
+    boolean is_identity_verified([ByRef] string user_id);
+
+    record<DOMString, record<DOMString, string>> sign([ByRef] string message);
+    [Throws=DecodeError]
+    void enable_backup_v1(MegolmV1BackupKey key, string version);
+    [Throws=CryptoStoreError]
+    void disable_backup();
+    [Throws=CryptoStoreError]
+    Request? backup_room_keys();
+    [Throws=CryptoStoreError]
+    void save_recovery_key(string? key, string? version);
+    [Throws=CryptoStoreError]
+    RoomKeyCounts room_key_counts();
+    [Throws=CryptoStoreError]
+    BackupKeys? get_backup_keys();
+    boolean backup_enabled();
+    [Throws=CryptoStoreError]
+    boolean verify_backup([ByRef] string auth_data);
+};
+
+dictionary PassphraseInfo {
+    string private_key_salt;
+    i32 private_key_iterations;
+};
+
+dictionary MegolmV1BackupKey {
+    string public_key;
+    record<DOMString, record<DOMString, string>> signatures;
+    PassphraseInfo? passphrase_info;
+    string backup_algorithm;
+};
+
+dictionary BackupKeys {
+    string recovery_key;
+    string backup_version;
+};
+
+dictionary RoomKeyCounts {
+    i64 total;
+    i64 backed_up;
+};
+
+[Error]
+enum DecodeError {
+    "Decode",
+    "CryptoStore",
+};
+
+interface BackupRecoveryKey {
+    constructor();
+    [Name=from_passphrase]
+    constructor(string passphrase, string salt, i32 rounds);
+    [Name=new_from_passphrase]
+    constructor(string passphrase);
+    [Name=from_base64, Throws=DecodeError]
+    constructor(string key);
+    [Name=from_base58, Throws=DecodeError]
+    constructor(string key);
+    string to_base58();
+    string to_base64();
+    MegolmV1BackupKey megolm_v1_public_key();
+    [Throws=PkDecryptionError]
+    string decrypt_v1(string ephemeral_key, string mac, string ciphertext);
+};
+
+dictionary MigrationData {
+    PickledAccount account;
+    sequence<PickledSession> sessions;
+    sequence<PickledInboundGroupSession> inbound_group_sessions;
+    string? backup_version;
+    string? backup_recovery_key;
+    string pickle_key;
+    CrossSigningKeyExport cross_signing;
+    sequence<string> tracked_users;
+};
+
+dictionary PickledAccount {
+    string user_id;
+    string device_id;
+    string pickle;
+    boolean shared;
+    i64 uploaded_signed_key_count;
+};
+
+dictionary PickledSession {
+    string pickle;
+    string sender_key;
+    boolean created_using_fallback_key;
+    string creation_time;
+    string last_use_time;
+};
+
+dictionary PickledInboundGroupSession {
+    string pickle;
+    string sender_key;
+    record<DOMString, string> signing_key;
+    string room_id;
+    sequence<string> forwarding_chains;
+    boolean imported;
+    boolean backed_up;
+};

crates/matrix-sdk-crypto-ffi/src/responses.rs

@@ -0,0 +1,331 @@
+#![allow(missing_docs)]
+
+use std::collections::HashMap;
+
+use http::Response;
+use matrix_sdk_crypto::{
+    IncomingResponse, OutgoingRequest, OutgoingVerificationRequest as SdkVerificationRequest,
+    RoomMessageRequest, ToDeviceRequest, UploadSigningKeysRequest as RustUploadSigningKeysRequest,
+};
+use ruma::{
+    api::client::{
+        backup::add_backup_keys::v3::Response as KeysBackupResponse,
+        keys::{
+            claim_keys::v3::{Request as KeysClaimRequest, Response as KeysClaimResponse},
+            get_keys::v3::Response as KeysQueryResponse,
+            upload_keys::v3::Response as KeysUploadResponse,
+            upload_signatures::v3::{
+                Request as RustSignatureUploadRequest, Response as SignatureUploadResponse,
+            },
+        },
+        message::send_message_event::v3::Response as RoomMessageResponse,
+        sync::sync_events::v3::DeviceLists as RumaDeviceLists,
+        to_device::send_event_to_device::v3::Response as ToDeviceResponse,
+    },
+    assign,
+    events::EventContent,
+    OwnedTransactionId, UserId,
+};
+use serde_json::json;
+
+pub struct SignatureUploadRequest {
+    pub body: String,
+}
+
+impl From<RustSignatureUploadRequest> for SignatureUploadRequest {
+    fn from(r: RustSignatureUploadRequest) -> Self {
+        Self {
+            body: serde_json::to_string(&r.signed_keys)
+                .expect("Can't serialize signature upload request"),
+        }
+    }
+}
+
+pub struct UploadSigningKeysRequest {
+    pub master_key: String,
+    pub self_signing_key: String,
+    pub user_signing_key: String,
+}
+
+impl From<RustUploadSigningKeysRequest> for UploadSigningKeysRequest {
+    fn from(r: RustUploadSigningKeysRequest) -> Self {
+        Self {
+            master_key: serde_json::to_string(
+                &r.master_key.expect("Request didn't contain a master key"),
+            )
+            .expect("Can't serialize cross signing master key"),
+            self_signing_key: serde_json::to_string(
+                &r.self_signing_key.expect("Request didn't contain a self-signing key"),
+            )
+            .expect("Can't serialize cross signing self-signing key"),
+            user_signing_key: serde_json::to_string(
+                &r.user_signing_key.expect("Request didn't contain a user-signing key"),
+            )
+            .expect("Can't serialize cross signing user-signing key"),
+        }
+    }
+}
+
+pub struct BootstrapCrossSigningResult {
+    pub upload_signing_keys_request: UploadSigningKeysRequest,
+    pub signature_request: SignatureUploadRequest,
+}
+
+impl From<(RustUploadSigningKeysRequest, RustSignatureUploadRequest)>
+    for BootstrapCrossSigningResult
+{
+    fn from(requests: (RustUploadSigningKeysRequest, RustSignatureUploadRequest)) -> Self {
+        Self {
+            upload_signing_keys_request: requests.0.into(),
+            signature_request: requests.1.into(),
+        }
+    }
+}
+
+pub enum OutgoingVerificationRequest {
+    ToDevice { request_id: String, event_type: String, body: String },
+    InRoom { request_id: String, room_id: String, event_type: String, content: String },
+}
+
+impl From<SdkVerificationRequest> for OutgoingVerificationRequest {
+    fn from(r: SdkVerificationRequest) -> Self {
+        match r {
+            SdkVerificationRequest::ToDevice(r) => r.into(),
+            SdkVerificationRequest::InRoom(r) => Self::InRoom {
+                request_id: r.txn_id.to_string(),
+                room_id: r.room_id.to_string(),
+                content: serde_json::to_string(&r.content)
+                    .expect("Can't serialize message content"),
+                event_type: r.content.event_type().to_string(),
+            },
+        }
+    }
+}
+
+impl From<ToDeviceRequest> for OutgoingVerificationRequest {
+    fn from(r: ToDeviceRequest) -> Self {
+        Self::ToDevice {
+            request_id: r.txn_id.to_string(),
+            event_type: r.event_type.to_string(),
+            body: serde_json::to_string(&r.messages).expect("Can't serialize to-device body"),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum Request {
+    ToDevice { request_id: String, event_type: String, body: String },
+    KeysUpload { request_id: String, body: String },
+    KeysQuery { request_id: String, users: Vec<String> },
+    KeysClaim { request_id: String, one_time_keys: HashMap<String, HashMap<String, String>> },
+    RoomMessage { request_id: String, room_id: String, event_type: String, content: String },
+    SignatureUpload { request_id: String, body: String },
+    KeysBackup { request_id: String, version: String, rooms: String },
+}
+
+impl From<OutgoingRequest> for Request {
+    fn from(r: OutgoingRequest) -> Self {
+        use matrix_sdk_crypto::OutgoingRequests::*;
+
+        match r.request() {
+            KeysUpload(u) => {
+                let body = json!({
+                    "device_keys": u.device_keys,
+                    "one_time_keys": u.one_time_keys,
+                });
+
+                Request::KeysUpload {
+                    request_id: r.request_id().to_string(),
+                    body: serde_json::to_string(&body)
+                        .expect("Can't serialize keys upload request"),
+                }
+            }
+            KeysQuery(k) => {
+                let users: Vec<String> = k.device_keys.keys().map(|u| u.to_string()).collect();
+                Request::KeysQuery { request_id: r.request_id().to_string(), users }
+            }
+            ToDeviceRequest(t) => Request::from(t),
+            SignatureUpload(t) => Request::SignatureUpload {
+                request_id: r.request_id().to_string(),
+                body: serde_json::to_string(&t.signed_keys)
+                    .expect("Can't serialize signature upload request"),
+            },
+            RoomMessage(r) => Request::from(r),
+            KeysClaim(c) => (r.request_id().to_owned(), c.clone()).into(),
+            KeysBackup(b) => Request::KeysBackup {
+                request_id: r.request_id().to_string(),
+                version: b.version.to_owned(),
+                rooms: serde_json::to_string(&b.rooms)
+                    .expect("Can't serialize keys backup request"),
+            },
+        }
+    }
+}
+
+impl From<ToDeviceRequest> for Request {
+    fn from(r: ToDeviceRequest) -> Self {
+        Request::ToDevice {
+            request_id: r.txn_id.to_string(),
+            event_type: r.event_type.to_string(),
+            body: serde_json::to_string(&r.messages).expect("Can't serialize to-device body"),
+        }
+    }
+}
+
+impl From<(OwnedTransactionId, KeysClaimRequest)> for Request {
+    fn from(request_tuple: (OwnedTransactionId, KeysClaimRequest)) -> Self {
+        let (request_id, request) = request_tuple;
+
+        Request::KeysClaim {
+            request_id: request_id.to_string(),
+            one_time_keys: request
+                .one_time_keys
+                .into_iter()
+                .map(|(u, d)| {
+                    (
+                        u.to_string(),
+                        d.into_iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(),
+                    )
+                })
+                .collect(),
+        }
+    }
+}
+
+impl From<&ToDeviceRequest> for Request {
+    fn from(r: &ToDeviceRequest) -> Self {
+        Request::ToDevice {
+            request_id: r.txn_id.to_string(),
+            event_type: r.event_type.to_string(),
+            body: serde_json::to_string(&r.messages).expect("Can't serialize to-device body"),
+        }
+    }
+}
+
+impl From<&RoomMessageRequest> for Request {
+    fn from(r: &RoomMessageRequest) -> Self {
+        Self::RoomMessage {
+            request_id: r.txn_id.to_string(),
+            room_id: r.room_id.to_string(),
+            event_type: r.content.event_type().to_string(),
+            content: serde_json::to_string(&r.content).expect("Can't serialize message content"),
+        }
+    }
+}
+
+pub(crate) fn response_from_string(body: &str) -> Response<Vec<u8>> {
+    Response::builder()
+        .status(200)
+        .body(body.as_bytes().to_vec())
+        .expect("Can't create HTTP response")
+}
+
+pub enum RequestType {
+    KeysQuery,
+    KeysClaim,
+    KeysUpload,
+    ToDevice,
+    SignatureUpload,
+    KeysBackup,
+    RoomMessage,
+}
+
+pub struct DeviceLists {
+    pub changed: Vec<String>,
+    pub left: Vec<String>,
+}
+
+impl From<DeviceLists> for RumaDeviceLists {
+    fn from(d: DeviceLists) -> Self {
+        assign!(RumaDeviceLists::new(), {
+            changed: d
+                .changed
+                .into_iter()
+                .filter_map(|u| UserId::parse(u).ok())
+                .collect(),
+            left: d
+                .left
+                .into_iter()
+                .filter_map(|u| UserId::parse(u).ok())
+                .collect(),
+        })
+    }
+}
+
+pub struct KeysImportResult {
+    /// The number of room keys that were imported.
+    pub imported: i64,
+    /// The total number of room keys that were found in the export.
+    pub total: i64,
+    /// The map of keys that were imported.
+    ///
+    /// It's a map from room id to a map of the sender key to a list of session
+    /// ids.
+    pub keys: HashMap<String, HashMap<String, Vec<String>>>,
+}
+
+pub(crate) enum OwnedResponse {
+    KeysClaim(KeysClaimResponse),
+    KeysUpload(KeysUploadResponse),
+    KeysQuery(KeysQueryResponse),
+    ToDevice(ToDeviceResponse),
+    SignatureUpload(SignatureUploadResponse),
+    KeysBackup(KeysBackupResponse),
+    RoomMessage(RoomMessageResponse),
+}
+
+impl From<KeysClaimResponse> for OwnedResponse {
+    fn from(response: KeysClaimResponse) -> Self {
+        OwnedResponse::KeysClaim(response)
+    }
+}
+
+impl From<KeysQueryResponse> for OwnedResponse {
+    fn from(response: KeysQueryResponse) -> Self {
+        OwnedResponse::KeysQuery(response)
+    }
+}
+
+impl From<KeysUploadResponse> for OwnedResponse {
+    fn from(response: KeysUploadResponse) -> Self {
+        OwnedResponse::KeysUpload(response)
+    }
+}
+
+impl From<ToDeviceResponse> for OwnedResponse {
+    fn from(response: ToDeviceResponse) -> Self {
+        OwnedResponse::ToDevice(response)
+    }
+}
+
+impl From<SignatureUploadResponse> for OwnedResponse {
+    fn from(response: SignatureUploadResponse) -> Self {
+        Self::SignatureUpload(response)
+    }
+}
+
+impl From<KeysBackupResponse> for OwnedResponse {
+    fn from(r: KeysBackupResponse) -> Self {
+        Self::KeysBackup(r)
+    }
+}
+
+impl From<RoomMessageResponse> for OwnedResponse {
+    fn from(response: RoomMessageResponse) -> Self {
+        OwnedResponse::RoomMessage(response)
+    }
+}
+
+impl<'a> From<&'a OwnedResponse> for IncomingResponse<'a> {
+    fn from(r: &'a OwnedResponse) -> Self {
+        match r {
+            OwnedResponse::KeysClaim(r) => IncomingResponse::KeysClaim(r),
+            OwnedResponse::KeysQuery(r) => IncomingResponse::KeysQuery(r),
+            OwnedResponse::KeysUpload(r) => IncomingResponse::KeysUpload(r),
+            OwnedResponse::ToDevice(r) => IncomingResponse::ToDevice(r),
+            OwnedResponse::SignatureUpload(r) => IncomingResponse::SignatureUpload(r),
+            OwnedResponse::KeysBackup(r) => IncomingResponse::KeysBackup(r),
+            OwnedResponse::RoomMessage(r) => IncomingResponse::RoomMessage(r),
+        }
+    }
+}

crates/matrix-sdk-crypto-ffi/src/users.rs

@@ -0,0 +1,60 @@
+use matrix_sdk_crypto::{types::CrossSigningKey, UserIdentities};
+
+use crate::CryptoStoreError;
+
+/// Enum representing cross signing identities of our own user or some other
+/// user.
+pub enum UserIdentity {
+    /// Our own user identity.
+    Own {
+        /// The unique id of our own user.
+        user_id: String,
+        /// Does our own user identity trust our own device.
+        trusts_our_own_device: bool,
+        /// The public master key of our identity.
+        master_key: String,
+        /// The public user-signing key of our identity.
+        user_signing_key: String,
+        /// The public self-signing key of our identity.
+        self_signing_key: String,
+    },
+    /// The user identity of other users.
+    Other {
+        /// The unique id of the user.
+        user_id: String,
+        /// The public master key of the identity.
+        master_key: String,
+        /// The public self-signing key of our identity.
+        self_signing_key: String,
+    },
+}
+
+impl UserIdentity {
+    pub(crate) async fn from_rust(i: UserIdentities) -> Result<Self, CryptoStoreError> {
+        Ok(match i {
+            UserIdentities::Own(i) => {
+                let master: CrossSigningKey = i.master_key().as_ref().to_owned();
+                let user_signing: CrossSigningKey = i.user_signing_key().as_ref().to_owned();
+                let self_signing: CrossSigningKey = i.self_signing_key().as_ref().to_owned();
+
+                UserIdentity::Own {
+                    user_id: i.user_id().to_string(),
+                    trusts_our_own_device: i.trusts_our_own_device().await?,
+                    master_key: serde_json::to_string(&master)?,
+                    user_signing_key: serde_json::to_string(&user_signing)?,
+                    self_signing_key: serde_json::to_string(&self_signing)?,
+                }
+            }
+            UserIdentities::Other(i) => {
+                let master: CrossSigningKey = i.master_key().as_ref().to_owned();
+                let self_signing: CrossSigningKey = i.self_signing_key().as_ref().to_owned();
+
+                UserIdentity::Other {
+                    user_id: i.user_id().to_string(),
+                    master_key: serde_json::to_string(&master)?,
+                    self_signing_key: serde_json::to_string(&self_signing)?,
+                }
+            }
+        })
+    }
+}

crates/matrix-sdk-crypto-ffi/src/verification.rs

@@ -0,0 +1,232 @@
+use matrix_sdk_crypto::{
+    CancelInfo as RustCancelInfo, QrVerification as InnerQr, Sas as InnerSas,
+    VerificationRequest as InnerVerificationRequest,
+};
+
+use crate::{OutgoingVerificationRequest, SignatureUploadRequest};
+
+/// Enum representing the different verification flows we support.
+pub enum Verification {
+    /// The `m.sas.v1` verification flow.
+    SasV1 {
+        #[allow(missing_docs)]
+        sas: Sas,
+    },
+    /// The `m.qr_code.scan.v1`, `m.qr_code.show.v1`, and `m.reciprocate.v1`
+    /// verification flow.
+    QrCodeV1 {
+        #[allow(missing_docs)]
+        qrcode: QrCode,
+    },
+}
+
+/// The `m.sas.v1` verification flow.
+pub struct Sas {
+    /// The other user that is participating in the verification flow
+    pub other_user_id: String,
+    /// The other user's device that is participating in the verification flow
+    pub other_device_id: String,
+    /// The unique ID of this verification flow, will be a random string for
+    /// to-device events or a event ID for in-room events.
+    pub flow_id: String,
+    /// The room ID where this verification is happening, will be `None` if the
+    /// verification is going through to-device messages
+    pub room_id: Option<String>,
+    /// Did we initiate the verification flow
+    pub we_started: bool,
+    /// Has the non-initiating side accepted the verification flow
+    pub has_been_accepted: bool,
+    /// Can the short auth string be presented
+    pub can_be_presented: bool,
+    /// Does the flow support the emoji representation of the short auth string
+    pub supports_emoji: bool,
+    /// Have we confirmed that the short auth strings match
+    pub have_we_confirmed: bool,
+    /// Has the verification completed successfully
+    pub is_done: bool,
+    /// Has the flow been cancelled
+    pub is_cancelled: bool,
+    /// Information about the cancellation of the flow, will be `None` if the
+    /// flow hasn't been cancelled
+    pub cancel_info: Option<CancelInfo>,
+}
+
+/// The `m.qr_code.scan.v1`, `m.qr_code.show.v1`, and `m.reciprocate.v1`
+/// verification flow.
+pub struct QrCode {
+    /// The other user that is participating in the verification flow
+    pub other_user_id: String,
+    /// The other user's device that is participating in the verification flow
+    pub other_device_id: String,
+    /// The unique ID of this verification flow, will be a random string for
+    /// to-device events or a event ID for in-room events.
+    pub flow_id: String,
+    /// The room ID where this verification is happening, will be `None` if the
+    /// verification is going through to-device messages
+    pub room_id: Option<String>,
+    /// Did we initiate the verification flow
+    pub we_started: bool,
+    /// Has the QR code been scanned by the other side
+    pub other_side_scanned: bool,
+    /// Has the scanning of the QR code been confirmed by us
+    pub has_been_confirmed: bool,
+    /// Did we scan the QR code and sent out a reciprocation
+    pub reciprocated: bool,
+    /// Has the verification completed successfully
+    pub is_done: bool,
+    /// Has the flow been cancelled
+    pub is_cancelled: bool,
+    /// Information about the cancellation of the flow, will be `None` if the
+    /// flow hasn't been cancelled
+    pub cancel_info: Option<CancelInfo>,
+}
+
+impl From<InnerQr> for QrCode {
+    fn from(qr: InnerQr) -> Self {
+        Self {
+            other_user_id: qr.other_user_id().to_string(),
+            flow_id: qr.flow_id().as_str().to_owned(),
+            is_cancelled: qr.is_cancelled(),
+            is_done: qr.is_done(),
+            cancel_info: qr.cancel_info().map(|c| c.into()),
+            reciprocated: qr.reciprocated(),
+            we_started: qr.we_started(),
+            other_side_scanned: qr.has_been_scanned(),
+            has_been_confirmed: qr.has_been_confirmed(),
+            other_device_id: qr.other_device_id().to_string(),
+            room_id: qr.room_id().map(|r| r.to_string()),
+        }
+    }
+}
+
+/// Information on why a verification flow has been cancelled and by whom.
+pub struct CancelInfo {
+    /// The textual representation of the cancel reason
+    pub reason: String,
+    /// The code describing the cancel reason
+    pub cancel_code: String,
+    /// Was the verification flow cancelled by us
+    pub cancelled_by_us: bool,
+}
+
+impl From<RustCancelInfo> for CancelInfo {
+    fn from(c: RustCancelInfo) -> Self {
+        Self {
+            reason: c.reason().to_owned(),
+            cancel_code: c.cancel_code().to_string(),
+            cancelled_by_us: c.cancelled_by_us(),
+        }
+    }
+}
+
+/// A result type for starting SAS verifications.
+pub struct StartSasResult {
+    /// The SAS verification object that got created.
+    pub sas: Sas,
+    /// The request that needs to be sent out to notify the other side that a
+    /// SAS verification should start.
+    pub request: OutgoingVerificationRequest,
+}
+
+/// A result type for scanning QR codes.
+pub struct ScanResult {
+    /// The QR code verification object that got created.
+    pub qr: QrCode,
+    /// The request that needs to be sent out to notify the other side that a
+    /// QR code verification should start.
+    pub request: OutgoingVerificationRequest,
+}
+
+impl From<InnerSas> for Sas {
+    fn from(sas: InnerSas) -> Self {
+        Self {
+            other_user_id: sas.other_user_id().to_string(),
+            other_device_id: sas.other_device_id().to_string(),
+            flow_id: sas.flow_id().as_str().to_owned(),
+            is_cancelled: sas.is_cancelled(),
+            is_done: sas.is_done(),
+            can_be_presented: sas.can_be_presented(),
+            supports_emoji: sas.supports_emoji(),
+            have_we_confirmed: sas.have_we_confirmed(),
+            we_started: sas.we_started(),
+            room_id: sas.room_id().map(|r| r.to_string()),
+            has_been_accepted: sas.has_been_accepted(),
+            cancel_info: sas.cancel_info().map(|c| c.into()),
+        }
+    }
+}
+
+/// A result type for requesting verifications.
+pub struct RequestVerificationResult {
+    /// The verification request object that got created.
+    pub verification: VerificationRequest,
+    /// The request that needs to be sent out to notify the other side that
+    /// we're requesting verification to begin.
+    pub request: OutgoingVerificationRequest,
+}
+
+/// A result type for confirming verifications.
+pub struct ConfirmVerificationResult {
+    /// The requests that needs to be sent out to notify the other side that we
+    /// confirmed the verification.
+    pub requests: Vec<OutgoingVerificationRequest>,
+    /// A request that will upload signatures of the verified device or user, if
+    /// the verification is completed and we're able to sign devices or users
+    pub signature_request: Option<SignatureUploadRequest>,
+}
+
+/// The verificatoin request object which then can transition into some concrete
+/// verification method
+pub struct VerificationRequest {
+    /// The other user that is participating in the verification flow
+    pub other_user_id: String,
+    /// The other user's device that is participating in the verification flow
+    pub other_device_id: Option<String>,
+    /// The unique ID of this verification flow, will be a random string for
+    /// to-device events or a event ID for in-room events.
+    pub flow_id: String,
+    /// The room ID where this verification is happening, will be `None` if the
+    /// verification is going through to-device messages
+    pub room_id: Option<String>,
+    /// Did we initiate the verification flow
+    pub we_started: bool,
+    /// Did both parties aggree to verification
+    pub is_ready: bool,
+    /// Did another device respond to the verification request
+    pub is_passive: bool,
+    /// Has the verification completed successfully
+    pub is_done: bool,
+    /// Has the flow been cancelled
+    pub is_cancelled: bool,
+    /// The list of verification methods that the other side advertised as
+    /// supported
+    pub their_methods: Option<Vec<String>>,
+    /// The list of verification methods that we advertised as supported
+    pub our_methods: Option<Vec<String>>,
+    /// Information about the cancellation of the flow, will be `None` if the
+    /// flow hasn't been cancelled
+    pub cancel_info: Option<CancelInfo>,
+}
+
+impl From<InnerVerificationRequest> for VerificationRequest {
+    fn from(v: InnerVerificationRequest) -> Self {
+        Self {
+            other_user_id: v.other_user().to_string(),
+            other_device_id: v.other_device_id().map(|d| d.to_string()),
+            flow_id: v.flow_id().as_str().to_owned(),
+            is_cancelled: v.is_cancelled(),
+            is_done: v.is_done(),
+            is_ready: v.is_ready(),
+            room_id: v.room_id().map(|r| r.to_string()),
+            we_started: v.we_started(),
+            is_passive: v.is_passive(),
+            cancel_info: v.cancel_info().map(|c| c.into()),
+            their_methods: v
+                .their_supported_methods()
+                .map(|v| v.into_iter().map(|m| m.to_string()).collect()),
+            our_methods: v
+                .our_supported_methods()
+                .map(|v| v.into_iter().map(|m| m.to_string()).collect()),
+        }
+    }
+}

crates/matrix-sdk-crypto/Cargo.toml

@@ -0,0 +1,78 @@
+[package]
+authors = ["Damir Jelić <poljar@termina.org.uk>"]
+description = "Matrix encryption library"
+edition = "2021"
+homepage = "https://github.com/matrix-org/matrix-rust-sdk"
+keywords = ["matrix", "chat", "messaging", "ruma", "nio"]
+license = "Apache-2.0"
+name = "matrix-sdk-crypto"
+readme = "README.md"
+repository = "https://github.com/matrix-org/matrix-rust-sdk"
+rust-version = "1.60"
+version = "0.5.0"
+
+[package.metadata.docs.rs]
+features = ["docsrs"]
+rustdoc-args = ["--cfg", "docsrs"]
+
+[features]
+default = []
+qrcode = ["matrix-sdk-qrcode"]
+backups_v1 = ["olm-rs", "bs58"]
+docsrs = []
+
+# Testing helpers for implementations based upon this
+testing = ["http"]
+
+[dependencies]
+aes = "0.8.1"
+atomic = "0.5.1"
+async-trait = "0.1.53"
+base64 = "0.13.0"
+bs58 = { version = "0.4.0", optional = true }
+byteorder = "1.4.3"
+ctr = "0.9.1"
+dashmap = "5.2.0"
+futures-util = { version = "0.3.21", default-features = false, features = ["alloc"] }
+hmac = "0.12.1"
+http = { version = "0.2.6", optional = true } # feature = testing only
+matrix-sdk-qrcode = { version = "0.3.0", path = "../matrix-sdk-qrcode", optional = true }
+matrix-sdk-common = { version = "0.5.0", path = "../matrix-sdk-common" }
+olm-rs = { version = "2.2.0", features = ["serde"], optional = true }
+pbkdf2 = { version = "0.11.0", default-features = false }
+rand = "0.8.5"
+serde = { version = "1.0.136", features = ["derive", "rc"] }
+serde_json = "1.0.79"
+sha2 = "0.10.2"
+thiserror = "1.0.30"
+tracing = "0.1.34"
+zeroize = { version = "1.3.0", features = ["zeroize_derive"] }
+
+[target.'cfg(target_arch = "wasm32")'.dependencies.ruma]
+version = "0.6.1"
+features = ["client-api-c", "js", "rand", "unstable-msc2676", "unstable-msc2677"]
+
+[target.'cfg(target_arch = "wasm32")'.dependencies.vodozemac]
+version = "0.2.0"
+features = ["js"]
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies.ruma]
+version = "0.6.1"
+features = ["client-api-c", "rand", "unstable-msc2676", "unstable-msc2677"]
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies.vodozemac]
+version = "0.2.0"
+
+[dev-dependencies]
+futures = { version = "0.3.21", default-features = false, features = ["executor"] }
+http = "0.2.6"
+indoc = "1.0.4"
+matches = "0.1.9"
+matrix-sdk-test = { version = "0.5.0", path = "../matrix-sdk-test" }
+proptest = { version = "1.0.0", default-features = false, features = ["std"] }
+# required for async_test macro
+tokio = { version = "1.7.0", default-features = false, features = ["macros", "rt-multi-thread"] }
+
+[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
+getrandom = { version = "0.2.6", features = ["js"] }
+wasm-bindgen-test = "0.3.24"

crates/matrix-sdk-crypto/README.md

@@ -0,0 +1,72 @@
+A no-network-IO implementation of a state machine that handles E2EE for
+[Matrix] clients.
+
+# Usage
+
+If you're just trying to write a Matrix client or bot in Rust, you're probably
+looking for [matrix-sdk] instead.
+
+However, if you're looking to add E2EE to an existing Matrix client or library,
+read on.
+
+The state machine works in a push/pull manner:
+
+- you push state changes and events retrieved from a Matrix homeserver /sync
+  response into the state machine
+- you pull requests that you'll need to send back to the homeserver out of the
+  state machine
+
+```rust,no_run
+use std::{collections::BTreeMap, convert::TryFrom};
+
+use matrix_sdk_crypto::{OlmMachine, OlmError};
+use ruma::{
+    api::client::sync::sync_events::v3::{ToDevice, DeviceLists},
+    device_id, user_id,
+};
+
+#[tokio::main]
+async fn main() -> Result<(), OlmError> {
+    let alice = user_id!("@alice:example.org");
+    let machine = OlmMachine::new(&alice, device_id!("DEVICEID")).await;
+
+    let to_device_events = ToDevice::default();
+    let changed_devices = DeviceLists::default();
+    let one_time_key_counts = BTreeMap::default();
+    let unused_fallback_keys = Some(Vec::new());
+
+    // Push changes that the server sent to us in a sync response.
+    let decrypted_to_device = machine.receive_sync_changes(
+        to_device_events,
+        &changed_devices,
+        &one_time_key_counts,
+        unused_fallback_keys.as_deref(),
+    ).await?;
+
+    // Pull requests that we need to send out.
+    let outgoing_requests = machine.outgoing_requests().await?;
+
+    // Send the requests here out and call machine.mark_request_as_sent().
+
+    Ok(())
+}
+```
+
+[Matrix]: https://matrix.org/
+[matrix-sdk]: https://github.com/matrix-org/matrix-rust-sdk/
+
+# Room key sharing algorithm
+
+The decision tree below visualizes the way this crate decides whether a room
+key will be shared with a requester upon a key request.
+
+![](https://raw.githubusercontent.com/matrix-org/matrix-rust-sdk/main/contrib/key-sharing-algorithm/model.png)
+
+
+# Crate Feature Flags
+
+The following crate feature flags are available:
+
+* `qrcode`: Enbles QRcode generation and reading code
+
+* `testing`: provides facilities and functions for tests, in particular for integration testing store implementations. ATTENTION: do not ever use outside of tests, we do not provide any stability warantees on these, these are merely helpers. If you find you _need_ any function provided here outside of tests, please open a Github Issue and inform us about your use case for us to consider.

crates/matrix-sdk-crypto/src/backups/keys/backup.rs

@@ -0,0 +1,151 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{
+    collections::BTreeMap,
+    sync::{Arc, Mutex},
+};
+
+use olm_rs::pk::OlmPkEncryption;
+use ruma::{
+    api::client::backup::{KeyBackupData, KeyBackupDataInit, SessionDataInit},
+    serde::Base64,
+    OwnedDeviceKeyId, OwnedUserId,
+};
+use zeroize::Zeroizing;
+
+use super::recovery::DecodeError;
+use crate::olm::InboundGroupSession;
+
+#[derive(Debug)]
+struct InnerBackupKey {
+    key: [u8; MegolmV1BackupKey::KEY_SIZE],
+    signatures: BTreeMap<OwnedUserId, BTreeMap<OwnedDeviceKeyId, String>>,
+    version: Mutex<Option<String>>,
+}
+
+/// The public part of a backup key.
+#[derive(Clone)]
+pub struct MegolmV1BackupKey {
+    inner: Arc<InnerBackupKey>,
+}
+
+impl std::fmt::Debug for MegolmV1BackupKey {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter
+            .debug_struct("MegolmV1BackupKey")
+            .field("key", &self.to_base64())
+            .field("version", &self.backup_version())
+            .finish()
+    }
+}
+
+impl MegolmV1BackupKey {
+    const KEY_SIZE: usize = 32;
+
+    pub(super) fn new(public_key: &str, version: Option<String>) -> Self {
+        let key = Self::from_base64(public_key).expect("Invalid backup key");
+
+        if let Some(version) = version {
+            key.set_version(version)
+        }
+
+        key
+    }
+
+    /// Get the full name of the backup algorithm this backup key supports.
+    pub fn backup_algorithm(&self) -> &str {
+        "m.megolm_backup.v1.curve25519-aes-sha2"
+    }
+
+    /// Get all the signatures of this `MegolmV1BackupKey`.
+    pub fn signatures(&self) -> BTreeMap<OwnedUserId, BTreeMap<OwnedDeviceKeyId, String>> {
+        self.inner.signatures.to_owned()
+    }
+
+    /// Try to create a new `MegolmV1BackupKey` from a base 64 encoded string.
+    pub fn from_base64(public_key: &str) -> Result<Self, DecodeError> {
+        let mut key = [0u8; Self::KEY_SIZE];
+        let decoded_key = crate::utilities::decode(public_key)?;
+
+        if decoded_key.len() != Self::KEY_SIZE {
+            Err(DecodeError::Length(Self::KEY_SIZE, decoded_key.len()))
+        } else {
+            key.copy_from_slice(&decoded_key);
+
+            let inner =
+                InnerBackupKey { key, signatures: Default::default(), version: Mutex::new(None) };
+
+            Ok(MegolmV1BackupKey { inner: inner.into() })
+        }
+    }
+
+    /// Convert the [`MegolmV1BackupKey`] to a base 64 encoded string.
+    pub fn to_base64(&self) -> String {
+        crate::utilities::encode(&self.inner.key)
+    }
+
+    /// Get the backup version that this key is used with, if any.
+    pub fn backup_version(&self) -> Option<String> {
+        self.inner.version.lock().unwrap().clone()
+    }
+
+    /// Set the backup version that this `MegolmV1BackupKey` will be used with.
+    ///
+    /// The key won't be able to encrypt room keys unless a version has been
+    /// set.
+    pub fn set_version(&self, version: String) {
+        *self.inner.version.lock().unwrap() = Some(version);
+    }
+
+    pub(crate) async fn encrypt(&self, session: InboundGroupSession) -> KeyBackupData {
+        let pk = OlmPkEncryption::new(&self.to_base64());
+
+        // It's ok to truncate here, there's a semantic difference only between
+        // 0 and 1+ anyways.
+        let forwarded_count = (session.forwarding_key_chain().len() as u32).into();
+        let first_message_index = session.first_known_index().into();
+
+        // Convert our key to the backup representation.
+        let key = session.to_backup().await;
+
+        // The key gets zeroized in `BackedUpRoomKey` but we're creating a copy
+        // here that won't, so let's wrap it up in a `Zeroizing` struct.
+        let key =
+            Zeroizing::new(serde_json::to_string(&key).expect("Can't serialize exported room key"));
+
+        let message = pk.encrypt(&key);
+
+        let session_data = SessionDataInit {
+            ephemeral: Base64::parse(message.ephemeral_key)
+                .expect("Can't decode the base64 encoded ephemeral backup key"),
+            ciphertext: Base64::parse(message.ciphertext)
+                .expect("Can't decode a base64 encoded libolm ciphertext"),
+            mac: Base64::parse(message.mac).expect("Can't decode a base64 encoded MAC"),
+        }
+        .into();
+
+        KeyBackupDataInit {
+            first_message_index,
+            forwarded_count,
+            // TODO is this actually used anywhere? seems to be completely
+            // useless and requires us to get the Device out of the store?
+            // Also should this be checked at the time of the backup or at the
+            // time of the room key receival?
+            is_verified: false,
+            session_data,
+        }
+        .into()
+    }
+}

crates/matrix-sdk-crypto/src/backups/keys/mod.rs

@@ -0,0 +1,54 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Module for the keys that are used to backup room keys.
+//!
+//! The backup key is split into two parts:
+//!
+//! ```text
+//!                     ┌───────────────────────────┐
+//!                     │  RecoveryKey | BackupKey  │
+//!                     └───────────────────────────┘
+//! ```
+//!
+//! 1. RecoveryKey, a private Curve25519 key that is used to decrypt backed up
+//!    room keys.
+//! 2. BackupKey, the public part of the Curve25519 RecoveryKey, this one is
+//!    used to encrypt room keys that get backed up.
+//!
+//! The `RecoveryKey` can be derived from a passphrase or randomly generated.
+//! To allow other devices access to this key you'll either have to re-enter the
+//! passphrase or the key as a base58 encoded string or received from another
+//! device using the secret sharing mechanism.
+//!
+//! In practice deriving a recovery key from a passphrase isn't done, and is
+//! **not** supported by the spec. Instead the secret storage key that encrypts
+//! secrets and puts them into your global account data gets derived from a
+//! passphrase and presented as a base58 encoded string, confusingly also called
+//! recovery key.
+//!
+//! The `RecoveryKey` can also be uploaded to your account data as an
+//! `m.megolm.v1` event, which gets encrypted by the SSSS key.
+//!
+//! The `MegolmV1BackupKey` is used to encrypt individual room keys so they can
+//! be uploaded to the homeserver.
+//!
+//! The `MegolmV1BackupKey` is a public key and is uploaded to the server using
+//! the `/room_keys/version` API endpoint.
+
+mod backup;
+mod recovery;
+
+pub use backup::MegolmV1BackupKey;
+pub use recovery::DecodeError;

crates/matrix-sdk-crypto/src/backups/keys/recovery.rs

@@ -0,0 +1,263 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{
+    convert::TryFrom,
+    io::{Cursor, Read},
+};
+
+use bs58;
+use olm_rs::{
+    errors::OlmPkDecryptionError,
+    pk::{OlmPkDecryption, PkMessage},
+};
+use thiserror::Error;
+use zeroize::Zeroizing;
+
+use super::MegolmV1BackupKey;
+use crate::store::RecoveryKey;
+
+/// Error type for the decoding of a RecoveryKey.
+#[derive(Debug, Error)]
+pub enum DecodeError {
+    /// The decoded recovery key has an invalid prefix.
+    #[error("The decoded recovery key has an invalid prefix: expected {0:?}, got {1:?}")]
+    Prefix([u8; 2], [u8; 2]),
+    /// The parity byte of the recovery key didn't match.
+    #[error("The parity byte of the recovery key doesn't match: expected {0:?}, got {1:?}")]
+    Parity(u8, u8),
+    /// The recovery key has an invalid length.
+    #[error("The decoded recovery key has a invalid length: expected {0}, got {1}")]
+    Length(usize, usize),
+    /// The recovry key isn't valid base58.
+    #[error(transparent)]
+    Base58(#[from] bs58::decode::Error),
+    /// The  recovery key isn't valid base64.
+    #[error(transparent)]
+    Base64(#[from] base64::DecodeError),
+    /// The recovery key is too short, we couldn't read enough data.
+    #[error(transparent)]
+    Io(#[from] std::io::Error),
+}
+
+#[derive(Debug, Error)]
+pub enum UnpicklingError {
+    #[error(transparent)]
+    Json(#[from] serde_json::Error),
+    // #[error("Couldn't decrypt the pickle: {0}")]
+    // Decryption(String),
+    #[error(transparent)]
+    Decode(#[from] DecodeError),
+}
+
+impl TryFrom<String> for RecoveryKey {
+    type Error = DecodeError;
+
+    fn try_from(value: String) -> Result<Self, Self::Error> {
+        Self::from_base58(&value)
+    }
+}
+
+impl std::fmt::Display for RecoveryKey {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let string = Zeroizing::new(self.to_base58());
+
+        let string = Zeroizing::new(
+            string
+                .chars()
+                .collect::<Vec<char>>()
+                .chunks(Self::DISPLAY_CHUNK_SIZE)
+                .map(|c| c.iter().collect::<String>())
+                .collect::<Vec<_>>()
+                .join(" "),
+        );
+
+        write!(f, "{}", string.as_str())
+    }
+}
+
+impl RecoveryKey {
+    const PREFIX: [u8; 2] = [0x8b, 0x01];
+    const PREFIX_PARITY: u8 = Self::PREFIX[0] ^ Self::PREFIX[1];
+    const DISPLAY_CHUNK_SIZE: usize = 4;
+
+    fn parity_byte(bytes: &[u8]) -> u8 {
+        bytes.iter().fold(Self::PREFIX_PARITY, |acc, x| acc ^ x)
+    }
+
+    /// Create a new recovery key from the given byte array.
+    ///
+    /// **Warning**: You need to make sure that the byte array contains correct
+    /// random data, either by using a random number generator or by using an
+    /// exported version of a previously created [`RecoveryKey`].
+    pub fn from_bytes(key: &[u8; Self::KEY_SIZE]) -> Self {
+        let mut inner = Box::new([0u8; Self::KEY_SIZE]);
+        inner.copy_from_slice(key);
+
+        Self::from_boxed_bytes(inner)
+    }
+
+    fn from_boxed_bytes(key: Box<[u8; Self::KEY_SIZE]>) -> Self {
+        Self { inner: key }
+    }
+
+    /// Get the recovery key as a raw byte representation.
+    pub fn as_bytes(&self) -> &[u8; Self::KEY_SIZE] {
+        &self.inner
+    }
+
+    /// Try to create a [`RecoveryKey`] from a base64 export of a `RecoveryKey`.
+    pub fn from_base64(key: &str) -> Result<Self, DecodeError> {
+        let decoded = Zeroizing::new(crate::utilities::decode(key)?);
+
+        if decoded.len() != Self::KEY_SIZE {
+            Err(DecodeError::Length(Self::KEY_SIZE, decoded.len()))
+        } else {
+            let mut key = Box::new([0u8; Self::KEY_SIZE]);
+            key.copy_from_slice(&decoded);
+
+            Ok(Self::from_boxed_bytes(key))
+        }
+    }
+
+    /// Try to create a [`RecoveryKey`] from a base58 export of a `RecoveryKey`.
+    pub fn from_base58(value: &str) -> Result<Self, DecodeError> {
+        // Remove any whitespace we might have
+        let value: String = value.chars().filter(|c| !c.is_whitespace()).collect();
+
+        let decoded = bs58::decode(value).with_alphabet(bs58::Alphabet::BITCOIN).into_vec()?;
+        let mut decoded = Cursor::new(decoded);
+
+        let mut prefix = [0u8; 2];
+        let mut key = Box::new([0u8; Self::KEY_SIZE]);
+        let mut expected_parity = [0u8; 1];
+
+        decoded.read_exact(&mut prefix)?;
+        decoded.read_exact(&mut *key)?;
+        decoded.read_exact(&mut expected_parity)?;
+
+        let expected_parity = expected_parity[0];
+        let parity = Self::parity_byte(key.as_ref());
+
+        let _ = Zeroizing::new(decoded.into_inner());
+
+        if prefix != Self::PREFIX {
+            Err(DecodeError::Prefix(Self::PREFIX, prefix))
+        } else if expected_parity != parity {
+            Err(DecodeError::Parity(expected_parity, parity))
+        } else {
+            Ok(Self::from_boxed_bytes(key))
+        }
+    }
+
+    /// Export the `RecoveryKey` as a base58 encoded string.
+    pub fn to_base58(&self) -> String {
+        let bytes = Zeroizing::new(
+            [
+                Self::PREFIX.as_ref(),
+                self.inner.as_ref(),
+                [Self::parity_byte(self.inner.as_ref())].as_ref(),
+            ]
+            .concat(),
+        );
+
+        bs58::encode(bytes.as_slice()).with_alphabet(bs58::Alphabet::BITCOIN).into_string()
+    }
+
+    fn get_pk_decrytpion(&self) -> OlmPkDecryption {
+        OlmPkDecryption::from_bytes(self.inner.as_ref())
+            .expect("Can't generate a libom PkDecryption object from our private key")
+    }
+
+    /// Extract the megolm.v1 public key from this `RecoveryKey`.
+    pub fn megolm_v1_public_key(&self) -> MegolmV1BackupKey {
+        let pk = self.get_pk_decrytpion();
+        let public_key = MegolmV1BackupKey::new(pk.public_key(), None);
+        public_key
+    }
+
+    /// Try to decrypt the given ciphertext using this `RecoveryKey`.
+    ///
+    /// This will use the [`m.megolm_backup.v1.curve25519-aes-sha2`] algorithm
+    /// to decrypt the given ciphertext.
+    ///
+    /// [`m.megolm_backup.v1.curve25519-aes-sha2`]:
+    /// https://spec.matrix.org/unstable/client-server-api/#backup-algorithm-mmegolm_backupv1curve25519-aes-sha2
+    pub fn decrypt_v1(
+        &self,
+        mac: String,
+        ephemeral_key: String,
+        ciphertext: String,
+    ) -> Result<String, OlmPkDecryptionError> {
+        let message = PkMessage::new(mac, ephemeral_key, ciphertext);
+        let pk = self.get_pk_decrytpion();
+
+        pk.decrypt(message)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{DecodeError, RecoveryKey};
+
+    const TEST_KEY: [u8; 32] = [
+        0x77, 0x07, 0x6D, 0x0A, 0x73, 0x18, 0xA5, 0x7D, 0x3C, 0x16, 0xC1, 0x72, 0x51, 0xB2, 0x66,
+        0x45, 0xDF, 0x4C, 0x2F, 0x87, 0xEB, 0xC0, 0x99, 0x2A, 0xB1, 0x77, 0xFB, 0xA5, 0x1D, 0xB9,
+        0x2C, 0x2A,
+    ];
+
+    #[test]
+    fn base64_decoding() -> Result<(), DecodeError> {
+        let key = RecoveryKey::new().expect("Can't create a new recovery key");
+
+        let base64 = key.to_base64();
+        let decoded_key = RecoveryKey::from_base64(&base64)?;
+        assert_eq!(key.inner, decoded_key.inner, "The decode key does't match the original");
+
+        RecoveryKey::from_base64("i").expect_err("The recovery key is too short");
+
+        Ok(())
+    }
+
+    #[test]
+    fn base58_decoding() -> Result<(), DecodeError> {
+        let key = RecoveryKey::new().expect("Can't create a new recovery key");
+
+        let base64 = key.to_base58();
+        let decoded_key = RecoveryKey::from_base58(&base64)?;
+        assert_eq!(key.inner, decoded_key.inner, "The decode key does't match the original");
+
+        let test_key =
+            RecoveryKey::from_base58("EsTcLW2KPGiFwKEA3As5g5c4BXwkqeeJZJV8Q9fugUMNUE4d")?;
+        assert_eq!(
+            test_key.as_bytes(),
+            &TEST_KEY,
+            "The decoded recovery key doesn't match the test key"
+        );
+
+        let test_key = RecoveryKey::from_base58(
+            "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d",
+        )?;
+        assert_eq!(
+            test_key.as_bytes(),
+            &TEST_KEY,
+            "The decoded recovery key doesn't match the test key"
+        );
+
+        RecoveryKey::from_base58("EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4e")
+            .expect_err("Can't create a recovery key if the parity byte is invalid");
+
+        Ok(())
+    }
+}

crates/matrix-sdk-crypto/src/backups/mod.rs

@@ -0,0 +1,466 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Server-side backup support for room keys
+//!
+//! This module largely implements support for server-side backups using the
+//! `m.megolm_backup.v1.curve25519-aes-sha2` backup algorithm.
+//!
+//! Due to various flaws in this backup algorithm it is **not** recommended to
+//! use this module or any of its functionality. The module is only provided for
+//! backwards compatibility.
+//!
+//! [spec]: https://spec.matrix.org/unstable/client-server-api/#server-side-key-backups
+
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    sync::Arc,
+};
+
+use matrix_sdk_common::locks::RwLock;
+use ruma::{
+    api::client::backup::RoomKeyBackup, serde::Raw, DeviceKeyAlgorithm, OwnedDeviceKeyId,
+    OwnedRoomId, OwnedTransactionId, OwnedUserId, TransactionId,
+};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use tracing::{debug, info, instrument, trace, warn};
+
+use crate::{
+    olm::{Account, InboundGroupSession},
+    store::{BackupKeys, Changes, RecoveryKey, RoomKeyCounts, Store},
+    CryptoStoreError, KeysBackupRequest, OutgoingRequest,
+};
+
+mod keys;
+
+pub use keys::{DecodeError, MegolmV1BackupKey};
+pub use olm_rs::errors::OlmPkDecryptionError;
+
+/// A state machine that handles backing up room keys.
+///
+/// The state machine can be activated using the
+/// [`BackupMachine::enable_backup_v1`] method. After the state machine has been
+/// enabled a request that will upload encrypted room keys can be generated
+/// using the [`BackupMachine::backup`] method.
+#[derive(Debug, Clone)]
+pub struct BackupMachine {
+    account: Account,
+    store: Store,
+    backup_key: Arc<RwLock<Option<MegolmV1BackupKey>>>,
+    pending_backup: Arc<RwLock<Option<PendingBackup>>>,
+}
+
+#[derive(Debug, Clone)]
+struct PendingBackup {
+    request_id: OwnedTransactionId,
+    request: KeysBackupRequest,
+    sessions: BTreeMap<OwnedRoomId, BTreeMap<String, BTreeSet<String>>>,
+}
+
+impl PendingBackup {
+    fn session_was_part_of_the_backup(&self, session: &InboundGroupSession) -> bool {
+        self.sessions
+            .get(session.room_id())
+            .and_then(|r| r.get(session.sender_key()).map(|s| s.contains(session.session_id())))
+            .unwrap_or(false)
+    }
+}
+
+impl From<PendingBackup> for OutgoingRequest {
+    fn from(b: PendingBackup) -> Self {
+        OutgoingRequest { request_id: b.request_id, request: Arc::new(b.request.into()) }
+    }
+}
+
+impl BackupMachine {
+    const BACKUP_BATCH_SIZE: usize = 100;
+
+    pub(crate) fn new(
+        account: Account,
+        store: Store,
+        backup_key: Option<MegolmV1BackupKey>,
+    ) -> Self {
+        Self {
+            account,
+            store,
+            backup_key: RwLock::new(backup_key).into(),
+            pending_backup: RwLock::new(None).into(),
+        }
+    }
+
+    /// Are we able to back up room keys to the server?
+    pub async fn enabled(&self) -> bool {
+        self.backup_key.read().await.as_ref().map(|b| b.backup_version().is_some()).unwrap_or(false)
+    }
+
+    /// Verify some backup auth data that we downloaded from the server.
+    ///
+    /// The auth data should be fetched from the server using the
+    /// [`/room_keys/version`] endpoint.
+    ///
+    /// Ruma models this using the [`BackupAlgorithm`] struct, but care needs to
+    /// be taken if that struct is used. Some clients might use unspecced fields
+    /// and the given Ruma struct will lose the unspecced fields after a
+    /// serialization cycle.
+    ///
+    /// [`BackupAlgorithm`]: ruma::api::client::backup::BackupAlgorithm
+    /// [`/room_keys/version`]: https://spec.matrix.org/unstable/client-server-api/#get_matrixclientv3room_keysversion
+    pub async fn verify_backup(
+        &self,
+        mut serialized_auth_data: Value,
+    ) -> Result<bool, CryptoStoreError> {
+        #[derive(Debug, Serialize, Deserialize)]
+        struct AuthData {
+            public_key: String,
+            #[serde(default)]
+            signatures: BTreeMap<OwnedUserId, BTreeMap<OwnedDeviceKeyId, String>>,
+            #[serde(flatten)]
+            extra: BTreeMap<String, Value>,
+        }
+
+        let auth_data: AuthData = serde_json::from_value(serialized_auth_data.clone())?;
+
+        trace!(?auth_data, "Verifying backup auth data");
+
+        Ok(if let Some(signatures) = auth_data.signatures.get(self.store.user_id()) {
+            for device_key_id in signatures.keys() {
+                if device_key_id.algorithm() == DeviceKeyAlgorithm::Ed25519 {
+                    if device_key_id.device_id() == self.account.device_id() {
+                        let result = self.account.is_signed(&mut serialized_auth_data);
+
+                        trace!(?result, "Checking auth data signature of our own device");
+
+                        if result.is_ok() {
+                            return Ok(true);
+                        }
+                    } else {
+                        let device = self
+                            .store
+                            .get_device(self.store.user_id(), device_key_id.device_id())
+                            .await?;
+
+                        trace!(
+                            device_id = device_key_id.device_id().as_str(),
+                            "Checking backup auth data for device"
+                        );
+
+                        if let Some(device) = device {
+                            if device.verified()
+                                && device.is_signed_by_device(&mut serialized_auth_data).is_ok()
+                            {
+                                return Ok(true);
+                            }
+                        } else {
+                            trace!(
+                                device_id = device_key_id.device_id().as_str(),
+                                "Device not found, can't check signature"
+                            );
+                        }
+                    }
+                }
+            }
+
+            false
+        } else {
+            false
+        })
+    }
+
+    /// Activate the given backup key to be used to encrypt and backup room
+    /// keys.
+    ///
+    /// This will use the [`m.megolm_backup.v1.curve25519-aes-sha2`] algorithm
+    /// to encrypt the room keys.
+    ///
+    /// [`m.megolm_backup.v1.curve25519-aes-sha2`]:
+    /// https://spec.matrix.org/unstable/client-server-api/#backup-algorithm-mmegolm_backupv1curve25519-aes-sha2
+    pub async fn enable_backup_v1(&self, key: MegolmV1BackupKey) -> Result<(), CryptoStoreError> {
+        if key.backup_version().is_some() {
+            *self.backup_key.write().await = Some(key.clone());
+            info!(backup_key = ?key, "Activated a backup");
+        } else {
+            warn!(backup_key = ?key, "Tried to activate a backup without having the backup key uploaded");
+        }
+
+        Ok(())
+    }
+
+    /// Get the number of backed up room keys and the total number of room keys.
+    pub async fn room_key_counts(&self) -> Result<RoomKeyCounts, CryptoStoreError> {
+        self.store.inbound_group_session_counts().await
+    }
+
+    /// Disable and reset our backup state.
+    ///
+    /// This will remove any pending backup request, remove the backup key and
+    /// reset the backup state of each room key we have.
+    #[instrument(skip(self))]
+    pub async fn disable_backup(&self) -> Result<(), CryptoStoreError> {
+        debug!("Disabling key backup and resetting backup state for room keys");
+
+        self.backup_key.write().await.take();
+        self.pending_backup.write().await.take();
+
+        self.store.reset_backup_state().await?;
+
+        debug!("Done disabling backup");
+
+        Ok(())
+    }
+
+    /// Store the recovery key in the cryptostore.
+    ///
+    /// This is useful if the client wants to support gossiping of the backup
+    /// key.
+    pub async fn save_recovery_key(
+        &self,
+        recovery_key: Option<RecoveryKey>,
+        version: Option<String>,
+    ) -> Result<(), CryptoStoreError> {
+        let changes = Changes { recovery_key, backup_version: version, ..Default::default() };
+        self.store.save_changes(changes).await
+    }
+
+    /// Get the backup keys we have saved in our crypto store.
+    pub async fn get_backup_keys(&self) -> Result<BackupKeys, CryptoStoreError> {
+        self.store.load_backup_keys().await
+    }
+
+    /// Encrypt a batch of room keys and return a request that needs to be sent
+    /// out to backup the room keys.
+    pub async fn backup(&self) -> Result<Option<OutgoingRequest>, CryptoStoreError> {
+        let mut request = self.pending_backup.write().await;
+
+        if let Some(request) = &*request {
+            trace!("Backing up, returning an existing request");
+
+            Ok(Some(request.clone().into()))
+        } else {
+            trace!("Backing up, creating a new request");
+
+            let new_request = self.backup_helper().await?;
+            *request = new_request.clone();
+
+            Ok(new_request.map(|r| r.into()))
+        }
+    }
+
+    pub(crate) async fn mark_request_as_sent(
+        &self,
+        request_id: &TransactionId,
+    ) -> Result<(), CryptoStoreError> {
+        let mut request = self.pending_backup.write().await;
+
+        if let Some(r) = &*request {
+            if r.request_id == request_id {
+                let sessions: Vec<_> = self
+                    .store
+                    .get_inbound_group_sessions()
+                    .await?
+                    .into_iter()
+                    .filter(|s| r.session_was_part_of_the_backup(s))
+                    .collect();
+
+                for session in &sessions {
+                    session.mark_as_backed_up()
+                }
+
+                trace!(request_id = ?r.request_id, keys = ?r.sessions, "Marking room keys as backed up");
+
+                let changes = Changes { inbound_group_sessions: sessions, ..Default::default() };
+                self.store.save_changes(changes).await?;
+
+                let counts = self.store.inbound_group_session_counts().await?;
+
+                trace!(
+                    room_key_counts = ?counts,
+                    request_id = ?r.request_id, keys = ?r.sessions, "Marked room keys as backed up"
+                );
+
+                *request = None;
+            } else {
+                warn!(
+                    expected = r.request_id.to_string().as_str(),
+                    got = request_id.to_string().as_str(),
+                    "Tried to mark a pending backup as sent but the request id didn't match"
+                )
+            }
+        } else {
+            warn!(
+                request_id = request_id.to_string().as_str(),
+                "Tried to mark a pending backup as sent but there isn't a backup pending"
+            );
+        }
+
+        Ok(())
+    }
+
+    async fn backup_helper(&self) -> Result<Option<PendingBackup>, CryptoStoreError> {
+        if let Some(backup_key) = &*self.backup_key.read().await {
+            if let Some(version) = backup_key.backup_version() {
+                let sessions =
+                    self.store.inbound_group_sessions_for_backup(Self::BACKUP_BATCH_SIZE).await?;
+
+                if !sessions.is_empty() {
+                    let key_count = sessions.len();
+                    let (backup, session_record) = Self::backup_keys(sessions, backup_key).await;
+
+                    info!(
+                        key_count = key_count,
+                        keys = ?session_record,
+                        ?backup_key,
+                        "Successfully created a room keys backup request"
+                    );
+
+                    let request = PendingBackup {
+                        request_id: TransactionId::new(),
+                        request: KeysBackupRequest { version, rooms: backup },
+                        sessions: session_record,
+                    };
+
+                    Ok(Some(request))
+                } else {
+                    trace!(?backup_key, "No room keys need to be backed up");
+                    Ok(None)
+                }
+            } else {
+                warn!("Trying to backup room keys but the backup key wasn't uploaded");
+                Ok(None)
+            }
+        } else {
+            warn!("Trying to backup room keys but no backup key was found");
+            Ok(None)
+        }
+    }
+
+    /// Backup all the non-backed up room keys we know about
+    async fn backup_keys(
+        sessions: Vec<InboundGroupSession>,
+        backup_key: &MegolmV1BackupKey,
+    ) -> (
+        BTreeMap<OwnedRoomId, RoomKeyBackup>,
+        BTreeMap<OwnedRoomId, BTreeMap<String, BTreeSet<String>>>,
+    ) {
+        let mut backup: BTreeMap<OwnedRoomId, RoomKeyBackup> = BTreeMap::new();
+        let mut session_record: BTreeMap<OwnedRoomId, BTreeMap<String, BTreeSet<String>>> =
+            BTreeMap::new();
+
+        for session in sessions {
+            let room_id = session.room_id().to_owned();
+            let session_id = session.session_id().to_owned();
+            let sender_key = session.sender_key().to_owned();
+            let session = backup_key.encrypt(session).await;
+
+            session_record
+                .entry(room_id.to_owned())
+                .or_default()
+                .entry(sender_key)
+                .or_default()
+                .insert(session_id.clone());
+
+            let session = Raw::new(&session).expect("Can't serialize a backed up room key");
+
+            backup
+                .entry(room_id)
+                .or_insert_with(|| RoomKeyBackup::new(BTreeMap::new()))
+                .sessions
+                .insert(session_id, session);
+        }
+
+        (backup, session_record)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use matrix_sdk_test::async_test;
+    use ruma::{device_id, room_id, user_id, DeviceId, RoomId, UserId};
+
+    use crate::{store::RecoveryKey, OlmError, OlmMachine};
+
+    fn alice_id() -> &'static UserId {
+        user_id!("@alice:example.org")
+    }
+
+    fn alice_device_id() -> &'static DeviceId {
+        device_id!("JLAFKJWSCS")
+    }
+
+    fn room_id() -> &'static RoomId {
+        room_id!("!test:localhost")
+    }
+
+    fn room_id2() -> &'static RoomId {
+        room_id!("!test2:localhost")
+    }
+
+    async fn backup_flow(machine: OlmMachine) -> Result<(), OlmError> {
+        let backup_machine = machine.backup_machine();
+        let counts = backup_machine.store.inbound_group_session_counts().await?;
+
+        assert_eq!(counts.total, 0, "Initially no keys exist");
+        assert_eq!(counts.backed_up, 0, "Initially no backed up keys exist");
+
+        machine.create_outbound_group_session_with_defaults(room_id()).await?;
+        machine.create_outbound_group_session_with_defaults(room_id2()).await?;
+
+        let counts = backup_machine.store.inbound_group_session_counts().await?;
+        assert_eq!(counts.total, 2, "Two room keys need to exist in the store");
+        assert_eq!(counts.backed_up, 0, "No room keys have been backed up yet");
+
+        let recovery_key = RecoveryKey::new().expect("Can't create new recovery key");
+        let backup_key = recovery_key.megolm_v1_public_key();
+        backup_key.set_version("1".to_owned());
+
+        backup_machine.enable_backup_v1(backup_key).await?;
+
+        let request =
+            backup_machine.backup().await?.expect("Created a backup request successfully");
+        assert_eq!(
+            Some(&*request.request_id),
+            backup_machine.backup().await?.as_ref().map(|r| &*r.request_id),
+            "Calling backup again without uploading creates the same backup request"
+        );
+
+        backup_machine.mark_request_as_sent(&request.request_id).await?;
+
+        let counts = backup_machine.store.inbound_group_session_counts().await?;
+        assert_eq!(counts.total, 2);
+        assert_eq!(counts.backed_up, 2, "All room keys have been backed up");
+
+        assert!(
+            backup_machine.backup().await?.is_none(),
+            "No room keys need to be backed up, no request needs to be created"
+        );
+
+        backup_machine.disable_backup().await?;
+
+        let counts = backup_machine.store.inbound_group_session_counts().await?;
+        assert_eq!(counts.total, 2);
+        assert_eq!(
+            counts.backed_up, 0,
+            "Disabling the backup resets the backup flag on the room keys"
+        );
+
+        Ok(())
+    }
+
+    #[async_test]
+    async fn memory_store_backups() -> Result<(), OlmError> {
+        let machine = OlmMachine::new(alice_id(), alice_device_id()).await;
+
+        backup_flow(machine).await
+    }
+}

crates/matrix-sdk-crypto/src/error.rs

@@ -12,8 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use olm_rs::errors::{OlmGroupSessionError, OlmSessionError};
-use ruma::{identifiers::Error as IdentifierError, DeviceId, UserId};
+use ruma::{IdParseError, OwnedDeviceId, OwnedRoomId, OwnedUserId};
 use serde_json::Error as SerdeError;
 use thiserror::Error;
 
@@ -34,13 +33,13 @@ pub enum OlmError {
     #[error(transparent)]
     JsonError(#[from] SerdeError),
 
-    /// The underlying Olm session operation returned an error.
-    #[error("can't finish Olm Session operation {0}")]
-    OlmSession(#[from] OlmSessionError),
+    /// The event could not have been decrypted.
+    #[error(transparent)]
+    Decryption(#[from] vodozemac::olm::DecryptionError),
 
-    /// The underlying group session operation returned an error.
-    #[error("can't finish Olm Session operation {0}")]
-    OlmGroupSession(#[from] OlmGroupSessionError),
+    /// The received room key couldn't be converted into a valid Megolm session.
+    #[error(transparent)]
+    SessionCreation(#[from] vodozemac::megolm::SessionKeyDecodeError),
 
     /// The storage layer returned an error.
     #[error("failed to read or write to the crypto store {0}")]
@@ -50,12 +49,12 @@ pub enum OlmError {
     #[error(
         "decryption failed likely because an Olm session from {0} with sender key {1} was wedged"
     )]
-    SessionWedged(UserId, String),
+    SessionWedged(OwnedUserId, String),
 
     /// An Olm message got replayed while the Olm ratchet has already moved
     /// forward.
     #[error("decryption failed because an Olm message from {0} with sender key {1} was replayed")]
-    ReplayedMessage(UserId, String),
+    ReplayedMessage(OwnedUserId, String),
 
     /// Encryption failed because the device does not have a valid Olm session
     /// with us.
@@ -77,19 +76,23 @@ pub enum MegolmError {
     #[error(transparent)]
     JsonError(#[from] SerdeError),
 
-    /// Decryption failed because the session needed to decrypt the event is
-    /// missing.
-    #[error("decryption failed because the session to decrypt the message is missing")]
-    MissingSession,
+    /// Decryption failed because we're missing the room key that was to encrypt
+    /// the event.
+    #[error("decryption failed because the room key is missing")]
+    MissingRoomKey,
 
-    /// The underlying group session operation returned an error.
-    #[error("can't finish Olm group session operation {0}")]
-    OlmGroupSession(#[from] OlmGroupSessionError),
+    /// The encrypted megolm message couldn't be decoded.
+    #[error(transparent)]
+    Decode(#[from] vodozemac::DecodeError),
 
     /// The room where a group session should be shared is not encrypted.
     #[error("The room where a group session should be shared is not encrypted")]
     EncryptionNotEnabled,
 
+    /// The event could not have been decrypted.
+    #[error(transparent)]
+    Decryption(#[from] vodozemac::megolm::DecryptionError),
+
     /// The storage layer returned an error.
     #[error(transparent)]
     Store(#[from] CryptoStoreError),
@@ -97,8 +100,8 @@ pub enum MegolmError {
 
 #[derive(Error, Debug)]
 pub enum EventError {
-    #[error("the Olm message has a unsupported type")]
-    UnsupportedOlmType,
+    #[error("the Olm message has a unsupported type, got {0}, expected 0 or 1")]
+    UnsupportedOlmType(u64),
 
     #[error("the Encrypted message has been encrypted with a unsupported algorithm.")]
     UnsupportedAlgorithm,
@@ -118,21 +121,23 @@ pub enum EventError {
     #[error("the Encrypted message is missing the field {0}")]
     MissingField(String),
 
-    #[error("the sender of the plaintext doesn't match the sender of the encrypted message.")]
-    MissmatchedSender,
+    #[error(
+        "the sender of the plaintext doesn't match the sender of the encrypted \
+        message, got {0}, expected {0}"
+    )]
+    MismatchedSender(OwnedUserId, OwnedUserId),
 
-    #[error("the keys of the message don't match the keys in our database.")]
-    MissmatchedKeys,
-}
+    #[error(
+        "the public that was part of the message doesn't match to the key we \
+        have stored, expected {0}, got {0}"
+    )]
+    MismatchedKeys(String, String),
 
-#[derive(Error, Debug)]
-pub enum SessionUnpicklingError {
-    /// The underlying Olm session operation returned an error.
-    #[error("can't finish Olm Session operation {0}")]
-    OlmSession(#[from] OlmSessionError),
-    /// The Session timestamp was invalid.
-    #[error("can't load session timestamps")]
-    SessionTimestampError,
+    #[error(
+        "the room id of the room key doesn't match the room id of the \
+        decrypted event: expected {0}, got {:1}"
+    )]
+    MismatchedRoom(OwnedRoomId, Option<OwnedRoomId>),
 }
 
 /// Error type describin different errors that happen when we check or create
@@ -145,7 +150,7 @@ pub enum SignatureError {
 
     /// The ID of the signing key isn't a valid key ID.
     #[error("the ID of the signing key is invalid")]
-    InvalidKeyId(#[from] IdentifierError),
+    InvalidKeyId(#[from] IdParseError),
 
     /// The signing key that should create or check a signature is missing.
     #[error("the signing key is missing from the object that signed the message")]
@@ -154,7 +159,7 @@ pub enum SignatureError {
     /// The user id of signing key differs from the user id that provided the
     /// signature.
     #[error("the user id of the signing key differs user id that provided the signature")]
-    UserIdMissmatch,
+    UserIdMismatch,
 
     /// The provided JSON value that was signed and the signature should be
     /// checked isn't a valid JSON object.
@@ -167,8 +172,12 @@ pub enum SignatureError {
     NoSignatureFound,
 
     /// The signature couldn't be verified.
-    #[error("the signature didn't match the provided key")]
-    VerificationError,
+    #[error(transparent)]
+    VerificationError(#[from] vodozemac::SignatureError),
+
+    /// The public key isn't a valid ed25519 key.
+    #[error(transparent)]
+    InvalidKey(#[from] vodozemac::KeyError),
 
     /// The signed object couldn't be deserialized.
     #[error(transparent)]
@@ -176,29 +185,33 @@ pub enum SignatureError {
 }
 
 #[derive(Error, Debug)]
-pub(crate) enum SessionCreationError {
+pub enum SessionCreationError {
     #[error(
         "Failed to create a new Olm session for {0} {1}, the requested \
         one-time key isn't a signed curve key"
     )]
-    OneTimeKeyNotSigned(UserId, Box<DeviceId>),
+    OneTimeKeyNotSigned(OwnedUserId, OwnedDeviceId),
     #[error(
         "Tried to create a new Olm session for {0} {1}, but the signed \
         one-time key is missing"
     )]
-    OneTimeKeyMissing(UserId, Box<DeviceId>),
+    OneTimeKeyMissing(OwnedUserId, OwnedDeviceId),
     #[error(
         "Tried to create a new Olm session for {0} {1}, but the one-time \
         key algorithm is unsupported"
     )]
-    OneTimeKeyUnknown(UserId, Box<DeviceId>),
+    OneTimeKeyUnknown(OwnedUserId, OwnedDeviceId),
     #[error("Failed to verify the one-time key signatures for {0} {1}: {2:?}")]
-    InvalidSignature(UserId, Box<DeviceId>, SignatureError),
+    InvalidSignature(OwnedUserId, OwnedDeviceId, SignatureError),
     #[error(
         "Tried to create an Olm session for {0} {1}, but the device is missing \
         a curve25519 key"
     )]
-    DeviceMissingCurveKey(UserId, Box<DeviceId>),
-    #[error("Error creating new Olm session for {0} {1}: {2:?}")]
-    OlmError(UserId, Box<DeviceId>, OlmSessionError),
+    DeviceMissingCurveKey(OwnedUserId, OwnedDeviceId),
+    #[error("Error deserializing the one-time key: {0}")]
+    InvalidJson(#[from] serde_json::Error),
+    #[error("The given curve25519 key is not a valid key")]
+    InvalidCurveKey(#[from] vodozemac::KeyError),
+    #[error(transparent)]
+    InboundCreation(#[from] vodozemac::olm::SessionCreationError),
 }

crates/matrix-sdk-crypto/src/file_encryption/attachments.rs

@@ -18,26 +18,29 @@ use std::{
 };
 
 use aes::{
-    cipher::{generic_array::GenericArray, FromBlockCipher, NewBlockCipher, StreamCipher},
-    Aes256, Aes256Ctr,
+    cipher::{generic_array::GenericArray, KeyIvInit, StreamCipher},
+    Aes256,
 };
 use base64::DecodeError;
-use getrandom::getrandom;
-use ruma::events::room::{EncryptedFile, JsonWebKey, JsonWebKeyInit};
+use rand::{thread_rng, RngCore};
+use ruma::{
+    events::room::{EncryptedFile, JsonWebKey, JsonWebKeyInit},
+    serde::Base64,
+};
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use thiserror::Error;
-use zeroize::Zeroizing;
-
-use crate::utilities::{decode, decode_url_safe, encode, encode_url_safe};
+use zeroize::Zeroize;
 
 const IV_SIZE: usize = 16;
 const KEY_SIZE: usize = 32;
 const VERSION: &str = "v2";
 
+type Aes256Ctr = ctr::Ctr128BE<Aes256>;
+
 /// A wrapper that transparently encrypts anything that implements `Read` as an
 /// Matrix attachment.
-pub struct AttachmentDecryptor<'a, R: 'a + Read> {
+pub struct AttachmentDecryptor<'a, R: Read> {
     inner: &'a mut R,
     expected_hash: Vec<u8>,
     sha: Sha256,
@@ -125,37 +128,45 @@ impl<'a, R: Read + 'a> AttachmentDecryptor<'a, R> {
     /// ```
     pub fn new(
         input: &'a mut R,
-        info: EncryptionInfo,
+        info: MediaEncryptionInfo,
     ) -> Result<AttachmentDecryptor<'a, R>, DecryptorError> {
         if info.version != VERSION {
             return Err(DecryptorError::UnknownVersion);
         }
 
-        let hash = decode(info.hashes.get("sha256").ok_or(DecryptorError::MissingHash)?)?;
-        let key = Zeroizing::from(decode_url_safe(info.web_key.k)?);
-        let iv = decode(info.iv)?;
+        let hash =
+            info.hashes.get("sha256").ok_or(DecryptorError::MissingHash)?.as_bytes().to_owned();
+        let mut key = info.web_key.k.into_inner();
+        let iv = info.iv.into_inner();
+
+        if key.len() != KEY_SIZE {
+            return Err(DecryptorError::KeyNonceLength);
+        }
+
+        let key_array = GenericArray::from_slice(&key);
         let iv = GenericArray::from_exact_iter(iv).ok_or(DecryptorError::KeyNonceLength)?;
 
         let sha = Sha256::default();
-        let aes = Aes256::new_from_slice(&key).map_err(|_| DecryptorError::KeyNonceLength)?;
-        let aes = Aes256Ctr::from_block_cipher(aes, &iv);
+
+        let aes = Aes256Ctr::new(key_array, &iv);
+        key.zeroize();
 
         Ok(AttachmentDecryptor { inner: input, expected_hash: hash, sha, aes })
     }
 }
 
 /// A wrapper that transparently encrypts anything that implements `Read`.
-pub struct AttachmentEncryptor<'a, R: Read + 'a> {
+pub struct AttachmentEncryptor<'a, R: Read + ?Sized> {
     finished: bool,
     inner: &'a mut R,
     web_key: JsonWebKey,
-    iv: String,
-    hashes: BTreeMap<String, String>,
+    iv: Base64,
+    hashes: BTreeMap<String, Base64>,
     aes: Aes256Ctr,
     sha: Sha256,
 }
 
-impl<'a, R: 'a + Read + std::fmt::Debug> std::fmt::Debug for AttachmentEncryptor<'a, R> {
+impl<'a, R: 'a + Read + std::fmt::Debug + ?Sized> std::fmt::Debug for AttachmentEncryptor<'a, R> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("AttachmentEncryptor")
             .field("inner", &self.inner)
@@ -164,13 +175,15 @@ impl<'a, R: 'a + Read + std::fmt::Debug> std::fmt::Debug for AttachmentEncryptor
     }
 }
 
-impl<'a, R: Read + 'a> Read for AttachmentEncryptor<'a, R> {
+impl<'a, R: Read + ?Sized + 'a> Read for AttachmentEncryptor<'a, R> {
     fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
         let read_bytes = self.inner.read(buf)?;
 
         if read_bytes == 0 {
             let hash = self.sha.finalize_reset();
-            self.hashes.entry("sha256".to_owned()).or_insert_with(|| encode(hash));
+            self.hashes
+                .entry("sha256".to_owned())
+                .or_insert_with(|| Base64::new(hash.as_slice().to_owned()));
             Ok(0)
         } else {
             self.aes.apply_keystream(&mut buf[0..read_bytes]);
@@ -181,7 +194,7 @@ impl<'a, R: Read + 'a> Read for AttachmentEncryptor<'a, R> {
     }
 }
 
-impl<'a, R: Read + 'a> AttachmentEncryptor<'a, R> {
+impl<'a, R: Read + ?Sized + 'a> AttachmentEncryptor<'a, R> {
     /// Wrap the given reader encrypting all the data we read from it.
     ///
     /// After all the reads are done, and all the data is encrypted that we wish
@@ -211,27 +224,29 @@ impl<'a, R: Read + 'a> AttachmentEncryptor<'a, R> {
     /// let key = encryptor.finish();
     /// ```
     pub fn new(reader: &'a mut R) -> Self {
-        let mut key = Zeroizing::new([0u8; KEY_SIZE]);
-        let mut iv = Zeroizing::new([0u8; IV_SIZE]);
+        let mut key = [0u8; KEY_SIZE];
+        let mut iv = [0u8; IV_SIZE];
 
-        getrandom(&mut *key).expect("Can't generate randomness");
+        let mut rng = thread_rng();
+
+        rng.fill_bytes(&mut key);
         // Only populate the first 8 bytes with randomness, the rest is 0
         // initialized for the counter.
-        getrandom(&mut iv[0..8]).expect("Can't generate randomness");
+        rng.fill_bytes(&mut iv[0..8]);
 
         let web_key = JsonWebKey::from(JsonWebKeyInit {
             kty: "oct".to_owned(),
             key_ops: vec!["encrypt".to_owned(), "decrypt".to_owned()],
             alg: "A256CTR".to_owned(),
-            k: encode_url_safe(&*key),
+            k: Base64::new(key.to_vec()),
             ext: true,
         });
-        let encoded_iv = encode(&*iv);
-        let iv = GenericArray::from_slice(&*iv);
-        let key = GenericArray::from_slice(&*key);
+        let encoded_iv = Base64::new((iv).to_vec());
 
-        let aes = Aes256::new(key);
-        let aes = Aes256Ctr::from_block_cipher(aes, iv);
+        let key_array = &key.into();
+
+        let aes = Aes256Ctr::new(key_array, &iv.into());
+        key.zeroize();
 
         AttachmentEncryptor {
             finished: false,
@@ -245,12 +260,14 @@ impl<'a, R: Read + 'a> AttachmentEncryptor<'a, R> {
     }
 
     /// Consume the encryptor and get the encryption key.
-    pub fn finish(mut self) -> EncryptionInfo {
+    pub fn finish(mut self) -> MediaEncryptionInfo {
         let hash = self.sha.finalize();
-        self.hashes.entry("sha256".to_owned()).or_insert_with(|| encode(hash));
+        self.hashes
+            .entry("sha256".to_owned())
+            .or_insert_with(|| Base64::new(hash.as_slice().to_owned()));
 
-        EncryptionInfo {
-            version: VERSION.to_string(),
+        MediaEncryptionInfo {
+            version: VERSION.to_owned(),
             hashes: self.hashes,
             iv: self.iv,
             web_key: self.web_key,
@@ -261,38 +278,38 @@ impl<'a, R: Read + 'a> AttachmentEncryptor<'a, R> {
 /// Struct holding all the information that is needed to decrypt an encrypted
 /// file.
 #[derive(Debug, Serialize, Deserialize)]
-pub struct EncryptionInfo {
+pub struct MediaEncryptionInfo {
     #[serde(rename = "v")]
     /// The version of the encryption scheme.
     pub version: String,
     /// The web key that was used to encrypt the file.
     pub web_key: JsonWebKey,
     /// The initialization vector that was used to encrypt the file.
-    pub iv: String,
+    pub iv: Base64,
     /// The hashes that can be used to check the validity of the file.
-    pub hashes: BTreeMap<String, String>,
+    pub hashes: BTreeMap<String, Base64>,
 }
 
-impl From<EncryptedFile> for EncryptionInfo {
+impl From<EncryptedFile> for MediaEncryptionInfo {
     fn from(file: EncryptedFile) -> Self {
         Self { version: file.v, web_key: file.key, iv: file.iv, hashes: file.hashes }
     }
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use std::io::{Cursor, Read};
 
     use serde_json::json;
 
-    use super::{AttachmentDecryptor, AttachmentEncryptor, EncryptionInfo};
+    use super::{AttachmentDecryptor, AttachmentEncryptor, MediaEncryptionInfo};
 
     const EXAMPLE_DATA: &[u8] = &[
         179, 154, 118, 127, 186, 127, 110, 33, 203, 33, 33, 134, 67, 100, 173, 46, 235, 27, 215,
         172, 36, 26, 75, 47, 33, 160,
     ];
 
-    fn example_key() -> EncryptionInfo {
+    fn example_key() -> MediaEncryptionInfo {
         let info = json!({
             "v": "v2",
             "web_key": {

crates/matrix-sdk-crypto/src/file_encryption/key_export.rs

@@ -15,22 +15,25 @@
 use std::io::{Cursor, Read, Seek, SeekFrom};
 
 use aes::{
-    cipher::{generic_array::GenericArray, FromBlockCipher, NewBlockCipher, StreamCipher},
-    Aes256, Aes256Ctr,
+    cipher::{generic_array::GenericArray, KeyIvInit, StreamCipher},
+    Aes256,
 };
 use byteorder::{BigEndian, ReadBytesExt};
-use getrandom::getrandom;
-use hmac::{Hmac, Mac, NewMac};
+use hmac::{Hmac, Mac};
 use pbkdf2::pbkdf2;
+use rand::{thread_rng, RngCore};
 use serde_json::Error as SerdeError;
 use sha2::{Sha256, Sha512};
 use thiserror::Error;
+use zeroize::Zeroize;
 
 use crate::{
     olm::ExportedRoomKey,
     utilities::{decode, encode, DecodeError},
 };
 
+type Aes256Ctr = ctr::Ctr128BE<Aes256>;
+
 const SALT_SIZE: usize = 16;
 const IV_SIZE: usize = 16;
 const MAC_SIZE: usize = 32;
@@ -76,14 +79,14 @@ pub enum KeyExportError {
 /// ```no_run
 /// # use std::io::Cursor;
 /// # use matrix_sdk_crypto::{OlmMachine, decrypt_key_export};
-/// # use ruma::user_id;
+/// # use ruma::{device_id, user_id};
 /// # use futures::executor::block_on;
 /// # let alice = user_id!("@alice:example.org");
-/// # let machine = OlmMachine::new(&alice, "DEVICEID".into());
 /// # block_on(async {
+/// # let machine = OlmMachine::new(&alice, device_id!("DEVICEID")).await;
 /// # let export = Cursor::new("".to_owned());
 /// let exported_keys = decrypt_key_export(export, "1234").unwrap();
-/// machine.import_keys(exported_keys, |_, _| {}).await.unwrap();
+/// machine.import_keys(exported_keys, false, |_, _| {}).await.unwrap();
 /// # });
 /// ```
 pub fn decrypt_key_export(
@@ -101,7 +104,13 @@ pub fn decrypt_key_export(
     let payload: String =
         x.lines().filter(|l| !(l.starts_with(HEADER) || l.starts_with(FOOTER))).collect();
 
-    Ok(serde_json::from_str(&decrypt_helper(&payload, passphrase)?)?)
+    let mut decrypted = decrypt_helper(&payload, passphrase)?;
+
+    let ret = serde_json::from_str(&decrypted);
+
+    decrypted.zeroize();
+
+    Ok(ret?)
 }
 
 /// Encrypt the list of exported room keys using the given passphrase.
@@ -127,13 +136,13 @@ pub fn decrypt_key_export(
 /// # Examples
 /// ```no_run
 /// # use matrix_sdk_crypto::{OlmMachine, encrypt_key_export};
-/// # use ruma::{user_id, room_id};
+/// # use ruma::{device_id, user_id, room_id};
 /// # use futures::executor::block_on;
 /// # let alice = user_id!("@alice:example.org");
-/// # let machine = OlmMachine::new(&alice, "DEVICEID".into());
 /// # block_on(async {
+/// # let machine = OlmMachine::new(&alice, device_id!("DEVICEID")).await;
 /// let room_id = room_id!("!test:localhost");
-/// let exported_keys = machine.export_keys(|s| s.room_id() == &room_id).await.unwrap();
+/// let exported_keys = machine.export_keys(|s| s.room_id() == room_id).await.unwrap();
 /// let encrypted_export = encrypt_key_export(&exported_keys, "1234", 1);
 /// # });
 /// ```
@@ -144,37 +153,41 @@ pub fn encrypt_key_export(
 ) -> Result<String, SerdeError> {
     let mut plaintext = serde_json::to_string(keys)?.into_bytes();
     let ciphertext = encrypt_helper(&mut plaintext, passphrase, rounds);
+
+    plaintext.zeroize();
+
     Ok([HEADER.to_owned(), ciphertext, FOOTER.to_owned()].join("\n"))
 }
 
-fn encrypt_helper(mut plaintext: &mut [u8], passphrase: &str, rounds: u32) -> String {
+fn encrypt_helper(plaintext: &mut [u8], passphrase: &str, rounds: u32) -> String {
     let mut salt = [0u8; SALT_SIZE];
     let mut iv = [0u8; IV_SIZE];
     let mut derived_keys = [0u8; KEY_SIZE * 2];
 
-    getrandom(&mut salt).expect("Can't generate randomness");
-    getrandom(&mut iv).expect("Can't generate randomness");
+    let mut rng = thread_rng();
+
+    rng.fill_bytes(&mut salt);
+    rng.fill_bytes(&mut iv);
 
     let mut iv = u128::from_be_bytes(iv);
     iv &= !(1 << 63);
+    let iv = iv.to_be_bytes();
 
     pbkdf2::<Hmac<Sha512>>(passphrase.as_bytes(), &salt, rounds, &mut derived_keys);
     let (key, hmac_key) = derived_keys.split_at(KEY_SIZE);
 
-    let key = GenericArray::from_slice(key);
-    let iv = iv.to_be_bytes();
-    let iv = GenericArray::from_slice(&iv);
+    // This is fine because the key is guaranteed to be 32 bytes, derive 64
+    // bytes and split at the middle.
+    let key_array = GenericArray::from_slice(key);
 
-    let aes = Aes256::new(key);
-    let mut aes = Aes256Ctr::from_block_cipher(aes, iv);
-
-    aes.apply_keystream(&mut plaintext);
+    let mut aes = Aes256Ctr::new(key_array, &iv.into());
+    aes.apply_keystream(plaintext);
 
     let mut payload: Vec<u8> = vec![];
 
     payload.extend(&VERSION.to_be_bytes());
     payload.extend(&salt);
-    payload.extend(&*iv);
+    payload.extend(&iv);
     payload.extend(&rounds.to_be_bytes());
     payload.extend_from_slice(plaintext);
 
@@ -184,6 +197,8 @@ fn encrypt_helper(mut plaintext: &mut [u8], passphrase: &str, rounds: u32) -> St
 
     payload.extend(mac.into_bytes());
 
+    derived_keys.zeroize();
+
     encode(payload)
 }
 
@@ -220,30 +235,56 @@ fn decrypt_helper(ciphertext: &str, passphrase: &str) -> Result<String, KeyExpor
 
     let mut hmac = Hmac::<Sha256>::new_from_slice(hmac_key).expect("Can't create an HMAC object");
     hmac.update(&decoded[0..ciphertext_end]);
-    hmac.verify(&mac).map_err(|_| KeyExportError::InvalidMac)?;
+    hmac.verify_slice(&mac).map_err(|_| KeyExportError::InvalidMac)?;
 
-    let key = GenericArray::from_slice(key);
-    let iv = GenericArray::from_slice(&iv);
+    // This is fine because the key is guaranteed to be 32 bytes, derive 64
+    // bytes and split at the middle.
+    let key_array = GenericArray::from_slice(key);
 
-    let mut ciphertext = &mut decoded[ciphertext_start..ciphertext_end];
-    let aes = Aes256::new(key);
-    let mut aes = Aes256Ctr::from_block_cipher(aes, iv);
-    aes.apply_keystream(&mut ciphertext);
+    let ciphertext = &mut decoded[ciphertext_start..ciphertext_end];
+    let mut aes = Aes256Ctr::new(key_array, &iv.into());
+    aes.apply_keystream(ciphertext);
 
-    Ok(String::from_utf8(ciphertext.to_owned())?)
+    let ret = String::from_utf8(ciphertext.to_owned());
+
+    derived_keys.zeroize();
+    ciphertext.zeroize();
+
+    Ok(ret?)
+}
+
+#[cfg(all(test, not(target_arch = "wasm32")))]
+mod proptests {
+    use proptest::prelude::*;
+
+    use super::{decrypt_helper, encrypt_helper};
+
+    proptest! {
+        #[test]
+        fn proptest_encrypt_cycle(plaintext in prop::string::string_regex(".*").unwrap()) {
+            let mut plaintext_bytes = plaintext.clone().into_bytes();
+
+            let ciphertext = encrypt_helper(&mut plaintext_bytes, "test", 1);
+            let decrypted = decrypt_helper(&ciphertext, "test").unwrap();
+
+            prop_assert!(plaintext == decrypted);
+        }
+    }
 }
 
 #[cfg(test)]
-mod test {
-    use std::io::Cursor;
+mod tests {
+    use std::{
+        collections::{BTreeMap, BTreeSet},
+        io::Cursor,
+    };
 
     use indoc::indoc;
     use matrix_sdk_test::async_test;
-    use proptest::prelude::*;
     use ruma::room_id;
 
     use super::{decode, decrypt_helper, decrypt_key_export, encrypt_helper, encrypt_key_export};
-    use crate::machine::test::get_prepared_machine;
+    use crate::{error::OlmResult, machine::tests::get_prepared_machine, RoomKeyImportResult};
 
     const PASSPHRASE: &str = "1234";
 
@@ -264,28 +305,16 @@ mod test {
         -----END MEGOLM SESSION DATA-----
     "};
 
-    fn export_wihtout_headers() -> String {
+    fn export_without_headers() -> String {
         TEST_EXPORT.lines().filter(|l| !l.starts_with("-----")).collect()
     }
 
     #[test]
     fn test_decode() {
-        let export = export_wihtout_headers();
+        let export = export_without_headers();
         assert!(decode(export).is_ok());
     }
 
-    proptest! {
-        #[test]
-        fn proptest_encrypt_cycle(plaintext in prop::string::string_regex(".*").unwrap()) {
-            let mut plaintext_bytes = plaintext.clone().into_bytes();
-
-            let ciphertext = encrypt_helper(&mut plaintext_bytes, "test", 1);
-            let decrypted = decrypt_helper(&ciphertext, "test").unwrap();
-
-            prop_assert!(plaintext == decrypted);
-        }
-    }
-
     #[test]
     fn test_encrypt_decrypt() {
         let data = "It's a secret to everybody";
@@ -302,16 +331,74 @@ mod test {
         let (machine, _) = get_prepared_machine().await;
         let room_id = room_id!("!test:localhost");
 
-        machine.create_outbound_group_session_with_defaults(&room_id).await.unwrap();
-        let export = machine.export_keys(|s| s.room_id() == &room_id).await.unwrap();
+        machine.create_outbound_group_session_with_defaults(room_id).await.unwrap();
+        let export = machine.export_keys(|s| s.room_id() == room_id).await.unwrap();
 
         assert!(!export.is_empty());
 
         let encrypted = encrypt_key_export(&export, "1234", 1).unwrap();
         let decrypted = decrypt_key_export(Cursor::new(encrypted), "1234").unwrap();
 
-        assert_eq!(export, decrypted);
-        assert_eq!(machine.import_keys(decrypted, |_, _| {}).await.unwrap(), (0, 1));
+        for (exported, decrypted) in export.iter().zip(decrypted.iter()) {
+            assert_eq!(exported.session_key.to_base64(), decrypted.session_key.to_base64());
+        }
+
+        assert_eq!(
+            machine.import_keys(decrypted, false, |_, _| {}).await.unwrap(),
+            RoomKeyImportResult::new(0, 1, BTreeMap::new())
+        );
+    }
+
+    #[async_test]
+    async fn test_importing_better_session() -> OlmResult<()> {
+        let (machine, _) = get_prepared_machine().await;
+        let room_id = room_id!("!test:localhost");
+        let session = machine.create_inbound_session(room_id).await?;
+
+        let export = vec![session.export_at_index(10).await];
+
+        let keys = RoomKeyImportResult::new(
+            1,
+            1,
+            BTreeMap::from([(
+                session.room_id().to_owned(),
+                BTreeMap::from([(
+                    session.sender_key().to_owned(),
+                    BTreeSet::from([session.session_id().to_owned()]),
+                )]),
+            )]),
+        );
+
+        assert_eq!(machine.import_keys(export, false, |_, _| {}).await?, keys,);
+
+        let export = vec![session.export_at_index(10).await];
+        assert_eq!(
+            machine.import_keys(export, false, |_, _| {}).await?,
+            RoomKeyImportResult::new(0, 1, BTreeMap::new())
+        );
+
+        let better_export = vec![session.export().await];
+
+        assert_eq!(machine.import_keys(better_export, false, |_, _| {}).await?, keys,);
+
+        let another_session = machine.create_inbound_session(room_id).await?;
+        let export = vec![another_session.export_at_index(10).await];
+
+        let keys = RoomKeyImportResult::new(
+            1,
+            1,
+            BTreeMap::from([(
+                another_session.room_id().to_owned(),
+                BTreeMap::from([(
+                    another_session.sender_key().to_owned(),
+                    BTreeSet::from([another_session.session_id().to_owned()]),
+                )]),
+            )]),
+        );
+
+        assert_eq!(machine.import_keys(export, false, |_, _| {}).await?, keys,);
+
+        Ok(())
     }
 
     #[test]

crates/matrix-sdk-crypto/src/file_encryption/mod.rs

@@ -1,5 +1,7 @@
 mod attachments;
 mod key_export;
 
-pub use attachments::{AttachmentDecryptor, AttachmentEncryptor, DecryptorError, EncryptionInfo};
+pub use attachments::{
+    AttachmentDecryptor, AttachmentEncryptor, DecryptorError, MediaEncryptionInfo,
+};
 pub use key_export::{decrypt_key_export, encrypt_key_export, KeyExportError};

crates/matrix-sdk-crypto/src/gossiping/mod.rs

@@ -18,17 +18,20 @@ use std::sync::Arc;
 
 use dashmap::{DashMap, DashSet};
 pub(crate) use machine::GossipMachine;
-use matrix_sdk_common::uuid::Uuid;
 use ruma::{
     events::{
-        room_key_request::{Action, RequestedKeyInfo, RoomKeyRequestToDeviceEventContent},
-        secret::request::{
-            RequestAction, RequestToDeviceEventContent as SecretRequestEventContent, SecretName,
+        room_key_request::{
+            Action, RequestedKeyInfo, ToDeviceRoomKeyRequestEvent,
+            ToDeviceRoomKeyRequestEventContent,
         },
-        AnyToDeviceEventContent, ToDeviceEvent,
+        secret::request::{
+            RequestAction, SecretName, ToDeviceSecretRequestEvent as SecretRequestEvent,
+            ToDeviceSecretRequestEventContent as SecretRequestEventContent,
+        },
+        AnyToDeviceEventContent,
     },
     to_device::DeviceIdOrAllDevices,
-    DeviceId, DeviceIdBox, UserId,
+    DeviceId, OwnedDeviceId, OwnedTransactionId, OwnedUserId, TransactionId, UserId,
 };
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
@@ -63,9 +66,9 @@ pub enum KeyForwardDecision {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct GossipRequest {
     /// The user we requested the secret from
-    pub request_recipient: UserId,
+    pub request_recipient: OwnedUserId,
     /// The unique id of the secret request.
-    pub request_id: Uuid,
+    pub request_id: OwnedTransactionId,
     /// The info of the requested secret.
     pub info: SecretInfo,
     /// Has the request been sent out.
@@ -75,12 +78,29 @@ pub struct GossipRequest {
 /// An enum over the various secret request types we can have.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum SecretInfo {
-    // Info for the `m.room_key_request` variant
+    /// Info for the `m.room_key_request` variant
     KeyRequest(RequestedKeyInfo),
-    // Info for the `m.secret.request` variant
+    /// Info for the `m.secret.request` variant
     SecretRequest(SecretName),
 }
 
+impl SecretInfo {
+    /// Serialize `SecretInfo` into `String` for usage as database keys and
+    /// comparison
+    pub fn as_key(&self) -> String {
+        match &self {
+            SecretInfo::KeyRequest(ref info) => format!(
+                "keyRequest:{:}:{:}:{:}:{:}",
+                info.room_id.as_str(),
+                info.sender_key,
+                info.session_id,
+                &info.algorithm
+            ),
+            SecretInfo::SecretRequest(ref sname) => format!("secretName:{:}", sname),
+        }
+    }
+}
+
 impl From<RequestedKeyInfo> for SecretInfo {
     fn from(i: RequestedKeyInfo) -> Self {
         Self::KeyRequest(i)
@@ -95,10 +115,10 @@ impl From<SecretName> for SecretInfo {
 
 impl GossipRequest {
     /// Create an ougoing secret request for the given secret.
-    pub(crate) fn from_secret_name(own_user_id: UserId, secret_name: SecretName) -> Self {
+    pub(crate) fn from_secret_name(own_user_id: OwnedUserId, secret_name: SecretName) -> Self {
         Self {
             request_recipient: own_user_id,
-            request_id: Uuid::new_v4(),
+            request_id: TransactionId::new(),
             info: secret_name.into(),
             sent_out: false,
         }
@@ -114,47 +134,47 @@ impl GossipRequest {
     fn to_request(&self, own_device_id: &DeviceId) -> OutgoingRequest {
         let content = match &self.info {
             SecretInfo::KeyRequest(r) => {
-                AnyToDeviceEventContent::RoomKeyRequest(RoomKeyRequestToDeviceEventContent::new(
+                AnyToDeviceEventContent::RoomKeyRequest(ToDeviceRoomKeyRequestEventContent::new(
                     Action::Request,
                     Some(r.clone()),
                     own_device_id.to_owned(),
-                    self.request_id.to_string(),
+                    self.request_id.clone(),
                 ))
             }
             SecretInfo::SecretRequest(s) => {
                 AnyToDeviceEventContent::SecretRequest(SecretRequestEventContent::new(
                     RequestAction::Request(s.clone()),
                     own_device_id.to_owned(),
-                    self.request_id.to_string(),
+                    self.request_id.clone(),
                 ))
             }
         };
 
-        let request = ToDeviceRequest::new_with_id(
+        let request = ToDeviceRequest::with_id(
             &self.request_recipient,
             DeviceIdOrAllDevices::AllDevices,
             content,
-            self.request_id,
+            self.request_id.clone(),
         );
 
-        OutgoingRequest { request_id: request.txn_id, request: Arc::new(request.into()) }
+        OutgoingRequest { request_id: request.txn_id.clone(), request: Arc::new(request.into()) }
     }
 
     fn to_cancellation(&self, own_device_id: &DeviceId) -> OutgoingRequest {
         let content = match self.info {
             SecretInfo::KeyRequest(_) => {
-                AnyToDeviceEventContent::RoomKeyRequest(RoomKeyRequestToDeviceEventContent::new(
+                AnyToDeviceEventContent::RoomKeyRequest(ToDeviceRoomKeyRequestEventContent::new(
                     Action::CancelRequest,
                     None,
                     own_device_id.to_owned(),
-                    self.request_id.to_string(),
+                    self.request_id.clone(),
                 ))
             }
             SecretInfo::SecretRequest(_) => {
                 AnyToDeviceEventContent::SecretRequest(SecretRequestEventContent::new(
                     RequestAction::RequestCancellation,
                     own_device_id.to_owned(),
-                    self.request_id.to_string(),
+                    self.request_id.clone(),
                 ))
             }
         };
@@ -165,7 +185,7 @@ impl GossipRequest {
             content,
         );
 
-        OutgoingRequest { request_id: request.txn_id, request: Arc::new(request.into()) }
+        OutgoingRequest { request_id: request.txn_id.clone(), request: Arc::new(request.into()) }
     }
 }
 
@@ -190,18 +210,18 @@ impl PartialEq for GossipRequest {
 
 #[derive(Debug, Clone)]
 enum RequestEvent {
-    KeyShare(ToDeviceEvent<RoomKeyRequestToDeviceEventContent>),
-    Secret(ToDeviceEvent<SecretRequestEventContent>),
+    KeyShare(ToDeviceRoomKeyRequestEvent),
+    Secret(SecretRequestEvent),
 }
 
-impl From<ToDeviceEvent<SecretRequestEventContent>> for RequestEvent {
-    fn from(e: ToDeviceEvent<SecretRequestEventContent>) -> Self {
+impl From<SecretRequestEvent> for RequestEvent {
+    fn from(e: SecretRequestEvent) -> Self {
         Self::Secret(e)
     }
 }
 
-impl From<ToDeviceEvent<RoomKeyRequestToDeviceEventContent>> for RequestEvent {
-    fn from(e: ToDeviceEvent<RoomKeyRequestToDeviceEventContent>) -> Self {
+impl From<ToDeviceRoomKeyRequestEvent> for RequestEvent {
+    fn from(e: ToDeviceRoomKeyRequestEvent) -> Self {
         Self::KeyShare(e)
     }
 }
@@ -229,7 +249,7 @@ impl RequestEvent {
         }
     }
 
-    fn request_id(&self) -> &str {
+    fn request_id(&self) -> &TransactionId {
         match self {
             RequestEvent::KeyShare(e) => &e.content.request_id,
             RequestEvent::Secret(e) => &e.content.request_id,
@@ -239,13 +259,17 @@ impl RequestEvent {
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 struct RequestInfo {
-    sender: UserId,
-    requesting_device_id: DeviceIdBox,
-    request_id: String,
+    sender: OwnedUserId,
+    requesting_device_id: OwnedDeviceId,
+    request_id: OwnedTransactionId,
 }
 
 impl RequestInfo {
-    fn new(sender: UserId, requesting_device_id: DeviceIdBox, request_id: String) -> Self {
+    fn new(
+        sender: OwnedUserId,
+        requesting_device_id: OwnedDeviceId,
+        request_id: OwnedTransactionId,
+    ) -> Self {
         Self { sender, requesting_device_id, request_id }
     }
 }
@@ -255,7 +279,8 @@ impl RequestInfo {
 #[derive(Debug, Clone)]
 struct WaitQueue {
     requests_waiting_for_session: Arc<DashMap<RequestInfo, RequestEvent>>,
-    requests_ids_waiting: Arc<DashMap<(UserId, DeviceIdBox), DashSet<String>>>,
+    #[allow(clippy::type_complexity)]
+    requests_ids_waiting: Arc<DashMap<(OwnedUserId, OwnedDeviceId), DashSet<OwnedTransactionId>>>,
 }
 
 impl WaitQueue {
@@ -282,7 +307,7 @@ impl WaitQueue {
         self.requests_waiting_for_session.insert(key, event);
 
         let key = (device.user_id().to_owned(), device.device_id().into());
-        self.requests_ids_waiting.entry(key).or_insert_with(DashSet::new).insert(request_id);
+        self.requests_ids_waiting.entry(key).or_default().insert(request_id);
     }
 
     fn remove(&self, user_id: &UserId, device_id: &DeviceId) -> Vec<(RequestInfo, RequestEvent)> {

crates/matrix-sdk-crypto/src/identities/device.rs

@@ -25,40 +25,37 @@ use std::{
 use atomic::Atomic;
 use matrix_sdk_common::locks::Mutex;
 use ruma::{
-    api::client::r0::keys::upload_signatures::Request as SignatureUploadRequest,
-    encryption::{DeviceKeys, SignedKey},
+    api::client::keys::upload_signatures::v3::Request as SignatureUploadRequest,
     events::{
-        forwarded_room_key::ForwardedRoomKeyToDeviceEventContent,
-        key::verification::VerificationMethod, room::encrypted::EncryptedToDeviceEventContent,
+        forwarded_room_key::ToDeviceForwardedRoomKeyEventContent,
+        key::verification::VerificationMethod, room::encrypted::ToDeviceRoomEncryptedEventContent,
         AnyToDeviceEventContent,
     },
-    DeviceId, DeviceIdBox, DeviceKeyAlgorithm, DeviceKeyId, EventEncryptionAlgorithm, UserId,
+    DeviceId, DeviceKeyAlgorithm, DeviceKeyId, EventEncryptionAlgorithm, OwnedDeviceId,
+    OwnedDeviceKeyId, OwnedUserId, UserId,
 };
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use serde_json::{json, Value};
 use tracing::warn;
+use vodozemac::{Curve25519PublicKey, Ed25519PublicKey};
 
 use super::{atomic_bool_deserializer, atomic_bool_serializer};
+#[cfg(any(test, feature = "testing"))]
+use crate::OlmMachine;
 use crate::{
     error::{EventError, OlmError, OlmResult, SignatureError},
     identities::{ReadOnlyOwnUserIdentity, ReadOnlyUserIdentities},
-    olm::{InboundGroupSession, PrivateCrossSigningIdentity, Session, Utility},
+    olm::{InboundGroupSession, Session, VerifyJson},
     store::{Changes, CryptoStore, DeviceChanges, Result as StoreResult},
+    types::{DeviceKey, DeviceKeys, SignedKey},
     verification::VerificationMachine,
-    OutgoingVerificationRequest, Sas, ToDeviceRequest, VerificationRequest,
+    OutgoingVerificationRequest, ReadOnlyAccount, Sas, ToDeviceRequest, VerificationRequest,
 };
-#[cfg(test)]
-use crate::{OlmMachine, ReadOnlyAccount};
 
 /// A read-only version of a `Device`.
 #[derive(Clone, Serialize, Deserialize)]
 pub struct ReadOnlyDevice {
-    user_id: Arc<UserId>,
-    device_id: Arc<DeviceId>,
-    algorithms: Arc<[EventEncryptionAlgorithm]>,
-    keys: Arc<BTreeMap<DeviceKeyId, String>>,
-    pub(crate) signatures: Arc<BTreeMap<UserId, BTreeMap<DeviceKeyId, String>>>,
-    display_name: Arc<Option<String>>,
+    pub(crate) inner: Arc<DeviceKeys>,
     #[serde(
         serialize_with = "atomic_bool_serializer",
         deserialize_with = "atomic_bool_deserializer"
@@ -74,9 +71,9 @@ pub struct ReadOnlyDevice {
 impl std::fmt::Debug for ReadOnlyDevice {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("ReadOnlyDevice")
-            .field("user_id", self.user_id())
+            .field("user_id", &self.user_id())
             .field("device_id", &self.device_id())
-            .field("display_name", self.display_name())
+            .field("display_name", &self.display_name())
             .field("keys", self.keys())
             .field("deleted", &self.deleted.load(Ordering::SeqCst))
             .field("trust_state", &self.trust_state)
@@ -181,8 +178,8 @@ impl Device {
 
     /// Get the Olm sessions that belong to this device.
     pub(crate) async fn get_sessions(&self) -> StoreResult<Option<Arc<Mutex<Vec<Session>>>>> {
-        if let Some(k) = self.get_key(DeviceKeyAlgorithm::Curve25519) {
-            self.verification_machine.store.get_sessions(k).await
+        if let Some(k) = self.curve25519_key() {
+            self.verification_machine.store.get_sessions(&k.to_base64()).await
         } else {
             Ok(None)
         }
@@ -221,13 +218,14 @@ impl Device {
         if self.user_id() == self.verification_machine.own_user_id() {
             Ok(self
                 .verification_machine
+                .store
                 .private_identity
                 .lock()
                 .await
                 .sign_device(&self.inner)
                 .await?)
         } else {
-            Err(SignatureError::UserIdMissmatch)
+            Err(SignatureError::UserIdMismatch)
         }
     }
 
@@ -258,7 +256,7 @@ impl Device {
     pub(crate) async fn encrypt(
         &self,
         content: AnyToDeviceEventContent,
-    ) -> OlmResult<(Session, EncryptedToDeviceEventContent)> {
+    ) -> OlmResult<(Session, ToDeviceRoomEncryptedEventContent)> {
         self.inner.encrypt(self.verification_machine.store.inner(), content).await
     }
 
@@ -268,14 +266,14 @@ impl Device {
         &self,
         session: InboundGroupSession,
         message_index: Option<u32>,
-    ) -> OlmResult<(Session, EncryptedToDeviceEventContent)> {
+    ) -> OlmResult<(Session, ToDeviceRoomEncryptedEventContent)> {
         let export = if let Some(index) = message_index {
             session.export_at_index(index).await
         } else {
             session.export().await
         };
 
-        let content: ForwardedRoomKeyToDeviceEventContent = if let Ok(c) = export.try_into() {
+        let content: ToDeviceForwardedRoomKeyEventContent = if let Ok(c) = export.try_into() {
             c
         } else {
             // TODO remove this panic.
@@ -295,8 +293,7 @@ impl Device {
 /// A read only view over all devices belonging to a user.
 #[derive(Debug)]
 pub struct UserDevices {
-    pub(crate) inner: HashMap<DeviceIdBox, ReadOnlyDevice>,
-    pub(crate) private_identity: Arc<Mutex<PrivateCrossSigningIdentity>>,
+    pub(crate) inner: HashMap<OwnedDeviceId, ReadOnlyDevice>,
     pub(crate) verification_machine: VerificationMachine,
     pub(crate) own_identity: Option<ReadOnlyOwnUserIdentity>,
     pub(crate) device_owner_identity: Option<ReadOnlyUserIdentities>,
@@ -313,15 +310,31 @@ impl UserDevices {
         })
     }
 
+    fn own_user_id(&self) -> &UserId {
+        self.verification_machine.own_user_id()
+    }
+
+    fn own_device_id(&self) -> &DeviceId {
+        self.verification_machine.own_device_id()
+    }
+
     /// Returns true if there is at least one devices of this user that is
     /// considered to be verified, false otherwise.
+    ///
+    /// This won't consider your own device as verified, as your own device is
+    /// always implicitly verified.
     pub fn is_any_verified(&self) -> bool {
-        self.inner.values().any(|d| d.verified(&self.own_identity, &self.device_owner_identity))
+        self.inner
+            .values()
+            .filter(|d| {
+                !(d.user_id() == self.own_user_id() && d.device_id() == self.own_device_id())
+            })
+            .any(|d| d.verified(&self.own_identity, &self.device_owner_identity))
     }
 
     /// Iterator over all the device ids of the user devices.
-    pub fn keys(&self) -> impl Iterator<Item = &DeviceIdBox> {
-        self.inner.keys()
+    pub fn keys(&self) -> impl Iterator<Item = &DeviceId> {
+        self.inner.keys().map(Deref::deref)
     }
 
     /// Iterator over all the devices of the user devices.
@@ -361,56 +374,67 @@ impl From<i64> for LocalTrust {
 }
 
 impl ReadOnlyDevice {
-    /// Create a new Device.
-    pub fn new(
-        user_id: UserId,
-        device_id: Box<DeviceId>,
-        display_name: Option<String>,
-        trust_state: LocalTrust,
-        algorithms: Vec<EventEncryptionAlgorithm>,
-        keys: BTreeMap<DeviceKeyId, String>,
-        signatures: BTreeMap<UserId, BTreeMap<DeviceKeyId, String>>,
-    ) -> Self {
+    /// Create a new Device, this constructor skips signature verification of
+    /// the keys, `TryFrom` should be used for completely new devices we
+    /// receive.
+    pub fn new(device_keys: DeviceKeys, trust_state: LocalTrust) -> Self {
         Self {
-            user_id: Arc::new(user_id),
-            device_id: device_id.into(),
-            display_name: Arc::new(display_name),
+            inner: device_keys.into(),
             trust_state: Arc::new(Atomic::new(trust_state)),
-            signatures: Arc::new(signatures),
-            algorithms: algorithms.into(),
-            keys: Arc::new(keys),
             deleted: Arc::new(AtomicBool::new(false)),
         }
     }
 
     /// The user id of the device owner.
     pub fn user_id(&self) -> &UserId {
-        &self.user_id
+        &self.inner.user_id
     }
 
     /// The unique ID of the device.
     pub fn device_id(&self) -> &DeviceId {
-        &self.device_id
+        &self.inner.device_id
     }
 
     /// Get the human readable name of the device.
-    pub fn display_name(&self) -> &Option<String> {
-        &self.display_name
+    pub fn display_name(&self) -> Option<&str> {
+        self.inner.unsigned.device_display_name.as_deref()
     }
 
     /// Get the key of the given key algorithm belonging to this device.
-    pub fn get_key(&self, algorithm: DeviceKeyAlgorithm) -> Option<&String> {
-        self.keys.get(&DeviceKeyId::from_parts(algorithm, &self.device_id))
+    pub fn get_key(&self, algorithm: DeviceKeyAlgorithm) -> Option<&DeviceKey> {
+        self.inner.keys.get(&DeviceKeyId::from_parts(algorithm, self.device_id()))
+    }
+
+    /// Get the Curve25519 key of the given device.
+    pub fn curve25519_key(&self) -> Option<Curve25519PublicKey> {
+        self.get_key(DeviceKeyAlgorithm::Curve25519).and_then(|k| {
+            if let DeviceKey::Curve25519(k) = k {
+                Some(*k)
+            } else {
+                None
+            }
+        })
+    }
+
+    /// Get the Ed25519 key of the given device.
+    pub fn ed25519_key(&self) -> Option<Ed25519PublicKey> {
+        self.get_key(DeviceKeyAlgorithm::Ed25519).and_then(|k| {
+            if let DeviceKey::Ed25519(k) = k {
+                Some(*k)
+            } else {
+                None
+            }
+        })
     }
 
     /// Get a map containing all the device keys.
-    pub fn keys(&self) -> &BTreeMap<DeviceKeyId, String> {
-        &self.keys
+    pub fn keys(&self) -> &BTreeMap<OwnedDeviceKeyId, DeviceKey> {
+        &self.inner.keys
     }
 
     /// Get a map containing all the device signatures.
-    pub fn signatures(&self) -> &BTreeMap<UserId, BTreeMap<DeviceKeyId, String>> {
-        &self.signatures
+    pub fn signatures(&self) -> &BTreeMap<OwnedUserId, BTreeMap<OwnedDeviceKeyId, String>> {
+        &self.inner.signatures
     }
 
     /// Get the trust state of the device.
@@ -440,7 +464,7 @@ impl ReadOnlyDevice {
 
     /// Get the list of algorithms this device supports.
     pub fn algorithms(&self) -> &[EventEncryptionAlgorithm] {
-        &self.algorithms
+        &self.inner.algorithms
     }
 
     /// Is the device deleted.
@@ -489,21 +513,22 @@ impl ReadOnlyDevice {
         &self,
         store: &dyn CryptoStore,
         content: AnyToDeviceEventContent,
-    ) -> OlmResult<(Session, EncryptedToDeviceEventContent)> {
-        let sender_key = if let Some(k) = self.get_key(DeviceKeyAlgorithm::Curve25519) {
+    ) -> OlmResult<(Session, ToDeviceRoomEncryptedEventContent)> {
+        let sender_key = if let Some(k) = self.curve25519_key() {
             k
         } else {
             warn!(
-                "Trying to encrypt a Megolm session for user {} on device {}, \
-                but the device doesn't have a curve25519 key",
-                self.user_id(),
-                self.device_id()
+                user_id = %self.user_id(),
+                device_id = %self.device_id(),
+                "Trying to encrypt a Megolm session, but the device doesn't \
+                have a curve25519 key",
             );
             return Err(EventError::MissingSenderKey.into());
         };
 
-        let session = if let Some(s) = store.get_sessions(sender_key).await? {
-            let sessions = s.lock().await;
+        let session = if let Some(s) = store.get_sessions(&sender_key.to_base64()).await? {
+            let mut sessions = s.lock().await;
+            sessions.sort_by_key(|s| s.last_use_time);
             sessions.get(0).cloned()
         } else {
             None
@@ -529,39 +554,23 @@ impl ReadOnlyDevice {
     /// Update a device with a new device keys struct.
     pub(crate) fn update_device(&mut self, device_keys: &DeviceKeys) -> Result<(), SignatureError> {
         self.verify_device_keys(device_keys)?;
-
-        let display_name = Arc::new(device_keys.unsigned.device_display_name.clone());
-
-        self.algorithms = device_keys.algorithms.as_slice().into();
-        self.keys = Arc::new(device_keys.keys.clone());
-        self.signatures = Arc::new(device_keys.signatures.clone());
-        self.display_name = display_name;
+        self.inner = device_keys.clone().into();
 
         Ok(())
     }
 
-    fn is_signed_by_device(&self, json: &mut Value) -> Result<(), SignatureError> {
-        let signing_key =
-            self.get_key(DeviceKeyAlgorithm::Ed25519).ok_or(SignatureError::MissingSigningKey)?;
+    pub(crate) fn is_signed_by_device(&self, json: &mut Value) -> Result<(), SignatureError> {
+        let key = self.ed25519_key().ok_or(SignatureError::MissingSigningKey)?;
 
-        let utility = Utility::new();
-
-        utility.verify_json(
-            &self.user_id,
+        key.verify_json(
+            self.user_id(),
             &DeviceKeyId::from_parts(DeviceKeyAlgorithm::Ed25519, self.device_id()),
-            signing_key,
             json,
         )
     }
 
-    pub(crate) fn as_device_keys(&self) -> DeviceKeys {
-        DeviceKeys::new(
-            self.user_id().clone(),
-            self.device_id().into(),
-            self.algorithms().to_vec(),
-            self.keys().clone(),
-            self.signatures().to_owned(),
-        )
+    pub(crate) fn as_device_keys(&self) -> &DeviceKeys {
+        &self.inner
     }
 
     pub(crate) fn verify_device_keys(
@@ -584,15 +593,31 @@ impl ReadOnlyDevice {
         self.deleted.store(true, Ordering::Relaxed);
     }
 
-    #[cfg(test)]
+    #[cfg(any(test, feature = "testing"))]
+    #[allow(dead_code)]
+    /// Generate the Device from the reference of an OlmMachine.
+    ///
+    /// TESTING FACILITY ONLY, DO NOT USE OUTSIDE OF TESTS
     pub async fn from_machine(machine: &OlmMachine) -> ReadOnlyDevice {
         ReadOnlyDevice::from_account(machine.account()).await
     }
 
-    #[cfg(test)]
+    /// Create a `ReadOnlyDevice` from an `Account`
+    ///
+    /// We will have our own device in the store once we receive a keys/query
+    /// response, but this is useful to create it before we receive such a
+    /// response.
+    ///
+    /// It also makes it easier to check that the server doesn't lie about our
+    /// own device.
+    ///
+    /// *Don't* use this after we received a keys/query response, other
+    /// users/devices might add signatures to our own device, which can't be
+    /// replicated locally.
     pub async fn from_account(account: &ReadOnlyAccount) -> ReadOnlyDevice {
         let device_keys = account.device_keys().await;
-        ReadOnlyDevice::try_from(&device_keys).unwrap()
+        ReadOnlyDevice::try_from(&device_keys)
+            .expect("Creating a device from our own account should always succeed")
     }
 }
 
@@ -601,12 +626,7 @@ impl TryFrom<&DeviceKeys> for ReadOnlyDevice {
 
     fn try_from(device_keys: &DeviceKeys) -> Result<Self, Self::Error> {
         let device = Self {
-            user_id: Arc::new(device_keys.user_id.clone()),
-            device_id: device_keys.device_id.clone().into(),
-            algorithms: device_keys.algorithms.as_slice().into(),
-            signatures: Arc::new(device_keys.signatures.clone()),
-            keys: Arc::new(device_keys.keys.clone()),
-            display_name: Arc::new(device_keys.unsigned.device_display_name.clone()),
+            inner: device_keys.clone().into(),
             deleted: Arc::new(AtomicBool::new(false)),
             trust_state: Arc::new(Atomic::new(LocalTrust::Unset)),
         };
@@ -622,16 +642,16 @@ impl PartialEq for ReadOnlyDevice {
     }
 }
 
-#[cfg(test)]
-pub(crate) mod test {
-    use std::convert::TryFrom;
-
-    use ruma::{encryption::DeviceKeys, user_id, DeviceKeyAlgorithm};
+#[cfg(any(test, feature = "testing"))]
+pub(crate) mod testing {
+    //! Testing Facilities for Device Management
+    #![allow(dead_code)]
     use serde_json::json;
 
-    use crate::identities::{LocalTrust, ReadOnlyDevice};
+    use crate::{identities::ReadOnlyDevice, types::DeviceKeys};
 
-    fn device_keys() -> DeviceKeys {
+    /// Generate default DeviceKeys for tests
+    pub fn device_keys() -> DeviceKeys {
         let device_keys = json!({
           "algorithms": vec![
               "m.olm.v1.curve25519-aes-sha2",
@@ -656,10 +676,20 @@ pub(crate) mod test {
         serde_json::from_value(device_keys).unwrap()
     }
 
-    pub(crate) fn get_device() -> ReadOnlyDevice {
+    /// Generate default ReadOnlyDevice for tests
+    pub fn get_device() -> ReadOnlyDevice {
         let device_keys = device_keys();
         ReadOnlyDevice::try_from(&device_keys).unwrap()
     }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use ruma::user_id;
+    use vodozemac::{Curve25519PublicKey, Ed25519PublicKey};
+
+    use super::testing::{device_keys, get_device};
+    use crate::identities::LocalTrust;
 
     #[test]
     fn create_a_device() {
@@ -668,18 +698,19 @@ pub(crate) mod test {
 
         let device = get_device();
 
-        assert_eq!(&user_id, device.user_id());
+        assert_eq!(user_id, device.user_id());
         assert_eq!(device_id, device.device_id());
-        assert_eq!(device.algorithms.len(), 2);
+        assert_eq!(device.algorithms().len(), 2);
         assert_eq!(LocalTrust::Unset, device.local_trust_state());
-        assert_eq!("Alice's mobile phone", device.display_name().as_ref().unwrap());
+        assert_eq!("Alice's mobile phone", device.display_name().unwrap());
         assert_eq!(
-            device.get_key(DeviceKeyAlgorithm::Curve25519).unwrap(),
-            "xfgbLIC5WAl1OIkpOzoxpCe8FsRDT6nch7NQsOb15nc"
+            device.curve25519_key().unwrap(),
+            Curve25519PublicKey::from_base64("xfgbLIC5WAl1OIkpOzoxpCe8FsRDT6nch7NQsOb15nc")
+                .unwrap(),
         );
         assert_eq!(
-            device.get_key(DeviceKeyAlgorithm::Ed25519).unwrap(),
-            "2/5LWJMow5zhJqakV88SIc7q/1pa8fmkfgAzx72w9G4"
+            device.ed25519_key().unwrap(),
+            Ed25519PublicKey::from_base64("2/5LWJMow5zhJqakV88SIc7q/1pa8fmkfgAzx72w9G4").unwrap(),
         );
     }
 
@@ -687,7 +718,7 @@ pub(crate) mod test {
     fn update_a_device() {
         let mut device = get_device();
 
-        assert_eq!("Alice's mobile phone", device.display_name().as_ref().unwrap());
+        assert_eq!("Alice's mobile phone", device.display_name().unwrap());
 
         let display_name = "Alice's work computer".to_owned();
 

crates/matrix-sdk-crypto/src/identities/manager.rs

@@ -13,18 +13,19 @@
 // limitations under the License.
 
 use std::{
-    collections::{BTreeMap, HashSet},
+    collections::{BTreeMap, BTreeSet, HashSet},
     convert::TryFrom,
+    ops::Deref,
     sync::Arc,
 };
 
-use futures::future::join_all;
+use futures_util::future::join_all;
 use matrix_sdk_common::executor::spawn;
 use ruma::{
-    api::client::r0::keys::get_keys::Response as KeysQueryResponse, encryption::DeviceKeys,
-    DeviceId, DeviceIdBox, UserId,
+    api::client::keys::get_keys::v3::Response as KeysQueryResponse, serde::Raw, DeviceId,
+    OwnedDeviceId, OwnedUserId, UserId,
 };
-use tracing::{info, trace, warn};
+use tracing::{debug, info, trace, warn};
 
 use crate::{
     error::OlmResult,
@@ -35,6 +36,8 @@ use crate::{
     olm::PrivateCrossSigningIdentity,
     requests::KeysQueryRequest,
     store::{Changes, DeviceChanges, IdentityChanges, Result as StoreResult, Store},
+    types::{CrossSigningKey, DeviceKeys},
+    LocalTrust,
 };
 
 enum DeviceChange {
@@ -74,27 +77,59 @@ impl IdentityManager {
         &self,
         response: &KeysQueryResponse,
     ) -> OlmResult<(DeviceChanges, IdentityChanges)> {
-        let changed_devices =
-            self.handle_devices_from_key_query(response.device_keys.clone()).await?;
-        let (changed_identities, cross_signing_identity) =
-            self.handle_cross_singing_keys(response).await?;
+        debug!(
+            users = ?response.device_keys.keys().collect::<BTreeSet<_>>(),
+            failures = ?response.failures,
+            "Handling a keys query response"
+        );
+
+        let devices = self.handle_devices_from_key_query(response.device_keys.clone()).await?;
+        let (identities, cross_signing_identity) = self.handle_cross_singing_keys(response).await?;
 
         let changes = Changes {
-            identities: changed_identities.clone(),
-            devices: changed_devices.clone(),
+            identities: identities.clone(),
+            devices: devices.clone(),
             private_identity: cross_signing_identity,
             ..Default::default()
         };
 
         // TODO turn this into a single transaction.
         self.store.save_changes(changes).await?;
-        let updated_users: Vec<&UserId> = response.device_keys.keys().collect();
+        let updated_users: Vec<&UserId> = response.device_keys.keys().map(Deref::deref).collect();
 
         for user_id in updated_users {
             self.store.update_tracked_user(user_id, false).await?;
         }
 
-        Ok((changed_devices, changed_identities))
+        let changed_devices = devices.changed.iter().fold(BTreeMap::new(), |mut acc, d| {
+            acc.entry(d.user_id()).or_insert_with(BTreeSet::new).insert(d.device_id());
+            acc
+        });
+
+        let new_devices = devices.new.iter().fold(BTreeMap::new(), |mut acc, d| {
+            acc.entry(d.user_id()).or_insert_with(BTreeSet::new).insert(d.device_id());
+            acc
+        });
+
+        let deleted_devices = devices.deleted.iter().fold(BTreeMap::new(), |mut acc, d| {
+            acc.entry(d.user_id()).or_insert_with(BTreeSet::new).insert(d.device_id());
+            acc
+        });
+
+        let new_identities = identities.new.iter().map(|i| i.user_id()).collect::<BTreeSet<_>>();
+        let changed_identities =
+            identities.changed.iter().map(|i| i.user_id()).collect::<BTreeSet<_>>();
+
+        debug!(
+            ?new_devices,
+            ?changed_devices,
+            ?deleted_devices,
+            ?new_identities,
+            ?changed_identities,
+            "Finished handling of the keys/query response"
+        );
+
+        Ok((devices, identities))
     }
 
     async fn update_or_create_device(
@@ -107,11 +142,12 @@ impl IdentityManager {
         if let Some(mut device) = old_device {
             if let Err(e) = device.update_device(&device_keys) {
                 warn!(
-                    "Failed to update the device keys for {} {}: {:?}",
-                    device.user_id(),
-                    device.device_id(),
-                    e
+                    user_id = device.user_id().as_str(),
+                    device_id = device.device_id().as_str(),
+                    error = ?e,
+                    "Failed to update device keys",
                 );
+
                 Ok(DeviceChange::None)
             } else {
                 Ok(DeviceChange::Updated(device))
@@ -119,14 +155,46 @@ impl IdentityManager {
         } else {
             match ReadOnlyDevice::try_from(&device_keys) {
                 Ok(d) => {
-                    trace!("Adding a new device to the device store {:?}", d);
-                    Ok(DeviceChange::New(d))
+                    // If this is our own device, check that the server isn't
+                    // lying about our keys, also mark the device as locally
+                    // trusted.
+                    if d.user_id() == store.user_id() && d.device_id() == store.device_id() {
+                        let local_device_keys = store.account().unsigned_device_keys();
+
+                        if d.keys() == &local_device_keys.keys {
+                            d.set_trust_state(LocalTrust::Verified);
+
+                            trace!(
+                                user_id = d.user_id().as_str(),
+                                device_id = d.device_id().as_str(),
+                                keys = ?d.keys(),
+                                "Adding our own device to the device store, \
+                                marking it as locally verified",
+                            );
+
+                            Ok(DeviceChange::New(d))
+                        } else {
+                            Ok(DeviceChange::None)
+                        }
+                    } else {
+                        trace!(
+                            user_id = d.user_id().as_str(),
+                            device_id = d.device_id().as_str(),
+                            keys = ?d.keys(),
+                            "Adding a new device to the device store",
+                        );
+
+                        Ok(DeviceChange::New(d))
+                    }
                 }
                 Err(e) => {
                     warn!(
-                        "Failed to create a new device for {} {}: {:?}",
-                        device_keys.user_id, device_keys.device_id, e
+                        user_id = device_keys.user_id.as_str(),
+                        device_id = device_keys.device_id.as_str(),
+                        error = ?e,
+                        "Failed to create a new device",
                     );
+
                     Ok(DeviceChange::None)
                 }
             }
@@ -137,27 +205,40 @@ impl IdentityManager {
         store: Store,
         own_user_id: Arc<UserId>,
         own_device_id: Arc<DeviceId>,
-        user_id: UserId,
-        device_map: BTreeMap<DeviceIdBox, DeviceKeys>,
+        user_id: OwnedUserId,
+        device_map: BTreeMap<OwnedDeviceId, Raw<ruma::encryption::DeviceKeys>>,
     ) -> StoreResult<DeviceChanges> {
-        let own_device_id = (&*own_device_id).to_owned();
+        let own_device_id = (*own_device_id).to_owned();
 
         let mut changes = DeviceChanges::default();
 
-        let current_devices: HashSet<DeviceIdBox> = device_map.keys().cloned().collect();
+        let current_devices: HashSet<OwnedDeviceId> = device_map.keys().cloned().collect();
 
-        let tasks = device_map.into_iter().filter_map(|(device_id, device_keys)| {
-            if user_id != device_keys.user_id || device_id != device_keys.device_id {
+        let tasks = device_map.into_iter().filter_map(|(device_id, device_keys)| match device_keys
+            .deserialize_as::<DeviceKeys>(
+        ) {
+            Ok(device_keys) => {
+                if user_id != device_keys.user_id || device_id != device_keys.device_id {
+                    warn!(
+                        user_id = user_id.as_str(),
+                        device_id = device_id.as_str(),
+                        device_key_user = device_keys.user_id.as_str(),
+                        device_key_device_id = device_keys.device_id.as_str(),
+                        "Mismatch in the device keys payload",
+                    );
+                    None
+                } else {
+                    Some(spawn(Self::update_or_create_device(store.clone(), device_keys)))
+                }
+            }
+            Err(e) => {
                 warn!(
                     user_id = user_id.as_str(),
                     device_id = device_id.as_str(),
-                    device_key_user = device_keys.user_id.as_str(),
-                    device_key_device_id = device_keys.device_id.as_str(),
-                    "Mismatch in the device keys payload",
+                    error = ?e,
+                    "Device keys failed to deserialize",
                 );
                 None
-            } else {
-                Some(spawn(Self::update_or_create_device(store.clone(), device_keys)))
             }
         });
 
@@ -173,14 +254,18 @@ impl IdentityManager {
             }
         }
 
-        let current_devices: HashSet<&DeviceIdBox> = current_devices.iter().collect();
+        let current_devices: HashSet<&OwnedDeviceId> = current_devices.iter().collect();
         let stored_devices = store.get_readonly_devices_unfiltered(&user_id).await?;
-        let stored_devices_set: HashSet<&DeviceIdBox> = stored_devices.keys().collect();
+        let stored_devices_set: HashSet<&OwnedDeviceId> = stored_devices.keys().collect();
         let deleted_devices_set = stored_devices_set.difference(&current_devices);
 
         for device_id in deleted_devices_set {
             if user_id == *own_user_id && *device_id == &own_device_id {
-                warn!("Our own device has been deleted");
+                warn!(
+                    user_id = own_user_id.as_str(),
+                    device_id = own_device_id.as_str(),
+                    "Our own device has been deleted"
+                );
             } else if let Some(device) = stored_devices.get(*device_id) {
                 device.mark_as_deleted();
                 changes.deleted.push(device.clone());
@@ -201,7 +286,10 @@ impl IdentityManager {
     /// they are new, one of their properties has changed or they got deleted.
     async fn handle_devices_from_key_query(
         &self,
-        device_keys_map: BTreeMap<UserId, BTreeMap<DeviceIdBox, DeviceKeys>>,
+        device_keys_map: BTreeMap<
+            OwnedUserId,
+            BTreeMap<OwnedDeviceId, Raw<ruma::encryption::DeviceKeys>>,
+        >,
     ) -> StoreResult<DeviceChanges> {
         let mut changes = DeviceChanges::default();
 
@@ -241,100 +329,131 @@ impl IdentityManager {
         let mut changes = IdentityChanges::default();
         let mut changed_identity = None;
 
+        // TODO this is a bit chunky, refactor this into smaller methods.
+
         for (user_id, master_key) in &response.master_keys {
-            let master_key = MasterPubkey::from(master_key);
+            match master_key.deserialize_as::<CrossSigningKey>() {
+                Ok(master_key) => {
+                    let master_key = MasterPubkey::from(master_key);
 
-            let self_signing = if let Some(s) = response.self_signing_keys.get(user_id) {
-                SelfSigningPubkey::from(s)
-            } else {
-                warn!(
-                    user_id = user_id.as_str(),
-                    "A user identity didn't contain a self signing pubkey"
-                );
-                continue;
-            };
+                    let self_signing = if let Some(s) = response
+                        .self_signing_keys
+                        .get(user_id)
+                        .and_then(|k| k.deserialize_as::<CrossSigningKey>().ok())
+                    {
+                        SelfSigningPubkey::from(s)
+                    } else {
+                        warn!(
+                            user_id = user_id.as_str(),
+                            "A user identity didn't contain a self signing pubkey \
+                            or the key was invalid"
+                        );
+                        continue;
+                    };
 
-            let result = if let Some(mut i) = self.store.get_user_identity(user_id).await? {
-                match &mut i {
-                    ReadOnlyUserIdentities::Own(ref mut identity) => {
-                        let user_signing = if let Some(s) = response.user_signing_keys.get(user_id)
+                    let result = if let Some(mut i) = self.store.get_user_identity(user_id).await? {
+                        match &mut i {
+                            ReadOnlyUserIdentities::Own(ref mut identity) => {
+                                let user_signing = if let Some(s) = response
+                                    .user_signing_keys
+                                    .get(user_id)
+                                    .and_then(|k| k.deserialize_as::<CrossSigningKey>().ok())
+                                {
+                                    UserSigningPubkey::from(s)
+                                } else {
+                                    warn!(
+                                        user_id = user_id.as_str(),
+                                        "User identity for our own user didn't \
+                                        contain a user signing pubkey",
+                                    );
+                                    continue;
+                                };
+
+                                identity
+                                    .update(master_key, self_signing, user_signing)
+                                    .map(|_| (i, false))
+                            }
+                            ReadOnlyUserIdentities::Other(ref mut identity) => {
+                                identity.update(master_key, self_signing).map(|_| (i, false))
+                            }
+                        }
+                    } else if user_id == self.user_id() {
+                        if let Some(s) = response
+                            .user_signing_keys
+                            .get(user_id)
+                            .and_then(|k| k.deserialize_as::<CrossSigningKey>().ok())
                         {
-                            UserSigningPubkey::from(s)
+                            let user_signing = UserSigningPubkey::from(s);
+
+                            if master_key.user_id() != user_id
+                                || self_signing.user_id() != user_id
+                                || user_signing.user_id() != user_id
+                            {
+                                warn!(
+                                    user_id = user_id.as_str(),
+                                    "User ID mismatch in one of the cross signing keys",
+                                );
+                                continue;
+                            }
+
+                            ReadOnlyOwnUserIdentity::new(master_key, self_signing, user_signing)
+                                .map(|i| (ReadOnlyUserIdentities::Own(i), true))
                         } else {
                             warn!(
                                 user_id = user_id.as_str(),
-                                "User identity for our own user didn't \
-                                contain a user signing pubkey",
+                                "User identity for our own user didn't contain a \
+                                user signing pubkey or the key isn't valid",
                             );
                             continue;
-                        };
-
-                        identity.update(master_key, self_signing, user_signing).map(|_| (i, false))
-                    }
-                    ReadOnlyUserIdentities::Other(ref mut identity) => {
-                        identity.update(master_key, self_signing).map(|_| (i, false))
-                    }
-                }
-            } else if user_id == self.user_id() {
-                if let Some(s) = response.user_signing_keys.get(user_id) {
-                    let user_signing = UserSigningPubkey::from(s);
-
-                    if master_key.user_id() != user_id
-                        || self_signing.user_id() != user_id
-                        || user_signing.user_id() != user_id
-                    {
+                        }
+                    } else if master_key.user_id() != user_id || self_signing.user_id() != user_id {
                         warn!(
-                            user_id = user_id.as_str(),
+                            user = user_id.as_str(),
                             "User ID mismatch in one of the cross signing keys",
                         );
                         continue;
-                    }
-
-                    ReadOnlyOwnUserIdentity::new(master_key, self_signing, user_signing)
-                        .map(|i| (ReadOnlyUserIdentities::Own(i), true))
-                } else {
-                    warn!(
-                        user_id = user_id.as_str(),
-                        "User identity for our own user didn't contain a \
-                        user signing pubkey",
-                    );
-                    continue;
-                }
-            } else if master_key.user_id() != user_id || self_signing.user_id() != user_id {
-                warn!(user = user_id.as_str(), "User ID mismatch in one of the cross signing keys",);
-                continue;
-            } else {
-                ReadOnlyUserIdentity::new(master_key, self_signing)
-                    .map(|i| (ReadOnlyUserIdentities::Other(i), true))
-            };
-
-            match result {
-                Ok((i, new)) => {
-                    if let Some(identity) = i.own() {
-                        let private_identity = self.store.private_identity();
-                        let private_identity = private_identity.lock().await;
-
-                        let result = private_identity.clear_if_differs(identity).await;
-
-                        if result.any_cleared() {
-                            changed_identity = Some((&*private_identity).clone());
-                            info!(cleared =? result, "Removed some or all of our private cross signing keys");
-                        }
-                    }
-
-                    if new {
-                        trace!(user_id = user_id.as_str(), identity =? i, "Created new user identity");
-                        changes.new.push(i);
                     } else {
-                        trace!(user_id = user_id.as_str(), identity =? i, "Updated a user identity");
-                        changes.changed.push(i);
+                        ReadOnlyUserIdentity::new(master_key, self_signing)
+                            .map(|i| (ReadOnlyUserIdentities::Other(i), true))
+                    };
+
+                    match result {
+                        Ok((i, new)) => {
+                            if let Some(identity) = i.own() {
+                                let private_identity = self.store.private_identity();
+                                let private_identity = private_identity.lock().await;
+
+                                let result = private_identity.clear_if_differs(identity).await;
+
+                                if result.any_cleared() {
+                                    changed_identity = Some((*private_identity).clone());
+                                    info!(cleared = ?result, "Removed some or all of our private cross signing keys");
+                                }
+                            }
+
+                            if new {
+                                trace!(user_id = user_id.as_str(), identity = ?i, "Created new user identity");
+                                changes.new.push(i);
+                            } else {
+                                trace!(user_id = user_id.as_str(), identity = ?i, "Updated a user identity");
+                                changes.changed.push(i);
+                            }
+                        }
+                        Err(e) => {
+                            warn!(
+                                user_id = user_id.as_str(),
+                                error = ?e,
+                                "Couldn't update or create new user identity"
+                            );
+                            continue;
+                        }
                     }
                 }
                 Err(e) => {
                     warn!(
                         user_id = user_id.as_str(),
-                        error =? e,
-                        "Couldn't update or create new user identity for"
+                        error = ?e,
+                        "Couldn't update or create new user identity"
                     );
                     continue;
                 }
@@ -357,10 +476,22 @@ impl IdentityManager {
     pub async fn users_for_key_query(&self) -> Vec<KeysQueryRequest> {
         let users = self.store.users_for_key_query();
 
+        // We always want to track our own user, but in case we aren't in an encrypted
+        // room yet, we won't be tracking ourselves yet. This ensures we are always
+        // tracking ourselves.
+        //
+        // The check for emptiness is done first for performance.
+        let users = if users.is_empty() && !self.store.tracked_users().contains(self.user_id()) {
+            self.update_tracked_users([self.user_id()]).await;
+            self.store.users_for_key_query()
+        } else {
+            users
+        };
+
         if users.is_empty() {
             Vec::new()
         } else {
-            let users: Vec<UserId> = users.into_iter().collect();
+            let users: Vec<OwnedUserId> = users.into_iter().collect();
 
             users
                 .chunks(Self::MAX_KEY_QUERY_USERS)
@@ -412,42 +543,42 @@ impl IdentityManager {
     }
 }
 
-#[cfg(test)]
-pub(crate) mod test {
+#[cfg(any(test, feature = "testing"))]
+pub(crate) mod testing {
+    #![allow(dead_code)]
     use std::sync::Arc;
 
     use matrix_sdk_common::locks::Mutex;
-    use matrix_sdk_test::async_test;
     use ruma::{
-        api::{client::r0::keys::get_keys::Response as KeyQueryResponse, IncomingResponse},
-        user_id, DeviceIdBox, UserId,
+        api::{client::keys::get_keys::v3::Response as KeyQueryResponse, IncomingResponse},
+        device_id, user_id, DeviceId, UserId,
     };
     use serde_json::json;
 
     use crate::{
         identities::IdentityManager,
-        machine::test::response_from_file,
+        machine::testing::response_from_file,
         olm::{PrivateCrossSigningIdentity, ReadOnlyAccount},
         store::{CryptoStore, MemoryStore, Store},
         verification::VerificationMachine,
     };
 
-    fn user_id() -> UserId {
+    pub fn user_id() -> &'static UserId {
         user_id!("@example:localhost")
     }
 
-    fn other_user_id() -> UserId {
+    pub fn other_user_id() -> &'static UserId {
         user_id!("@example2:localhost")
     }
 
-    fn device_id() -> DeviceIdBox {
-        "WSKKLTJZCL".into()
+    pub fn device_id() -> &'static DeviceId {
+        device_id!("WSKKLTJZCL")
     }
 
-    fn manager() -> IdentityManager {
+    pub(crate) fn manager() -> IdentityManager {
         let identity = Arc::new(Mutex::new(PrivateCrossSigningIdentity::empty(user_id())));
-        let user_id = Arc::new(user_id());
-        let account = ReadOnlyAccount::new(&user_id, &device_id());
+        let user_id = Arc::from(user_id());
+        let account = ReadOnlyAccount::new(&user_id, device_id());
         let store: Arc<dyn CryptoStore> = Arc::new(MemoryStore::new());
         let verification = VerificationMachine::new(account, identity.clone(), store);
         let store =
@@ -455,7 +586,7 @@ pub(crate) mod test {
         IdentityManager::new(user_id, device_id().into(), store)
     }
 
-    pub(crate) fn other_key_query() -> KeyQueryResponse {
+    pub fn other_key_query() -> KeyQueryResponse {
         let data = response_from_file(&json!({
             "device_keys": {
                 "@example2:localhost": {
@@ -514,7 +645,7 @@ pub(crate) mod test {
             .expect("Can't parse the keys upload response")
     }
 
-    pub(crate) fn own_key_query() -> KeyQueryResponse {
+    pub fn own_key_query() -> KeyQueryResponse {
         let data = response_from_file(&json!({
           "device_keys": {
             "@example:localhost": {
@@ -614,32 +745,40 @@ pub(crate) mod test {
         KeyQueryResponse::try_from_http_response(data)
             .expect("Can't parse the keys upload response")
     }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use matrix_sdk_test::async_test;
+    use ruma::device_id;
+
+    use super::testing::{manager, other_key_query, other_user_id};
 
     #[async_test]
     async fn test_manager_creation() {
         let manager = manager();
-        assert!(manager.users_for_key_query().await.is_empty())
+        assert!(manager.store.tracked_users().is_empty())
     }
 
     #[async_test]
     async fn test_manager_key_query_response() {
         let manager = manager();
         let other_user = other_user_id();
-        let devices = manager.store.get_user_devices(&other_user).await.unwrap();
+        let devices = manager.store.get_user_devices(other_user).await.unwrap();
         assert_eq!(devices.devices().count(), 0);
 
         manager.receive_keys_query_response(&other_key_query()).await.unwrap();
 
-        let devices = manager.store.get_user_devices(&other_user).await.unwrap();
+        let devices = manager.store.get_user_devices(other_user).await.unwrap();
         assert_eq!(devices.devices().count(), 1);
 
         let device = manager
             .store
-            .get_readonly_device(&other_user, "SKISMLNIMH".into())
+            .get_readonly_device(other_user, device_id!("SKISMLNIMH"))
             .await
             .unwrap()
             .unwrap();
-        let identity = manager.store.get_user_identity(&other_user).await.unwrap().unwrap();
+        let identity = manager.store.get_user_identity(other_user).await.unwrap().unwrap();
         let identity = identity.other().unwrap();
 
         assert!(identity.is_device_signed(&device).is_ok())
@@ -649,23 +788,38 @@ pub(crate) mod test {
     async fn test_manager_own_key_query_response() {
         let manager = manager();
         let other_user = other_user_id();
-        let devices = manager.store.get_user_devices(&other_user).await.unwrap();
+        let devices = manager.store.get_user_devices(other_user).await.unwrap();
         assert_eq!(devices.devices().count(), 0);
 
         manager.receive_keys_query_response(&other_key_query()).await.unwrap();
 
-        let devices = manager.store.get_user_devices(&other_user).await.unwrap();
+        let devices = manager.store.get_user_devices(other_user).await.unwrap();
         assert_eq!(devices.devices().count(), 1);
 
         let device = manager
             .store
-            .get_readonly_device(&other_user, "SKISMLNIMH".into())
+            .get_readonly_device(other_user, device_id!("SKISMLNIMH"))
             .await
             .unwrap()
             .unwrap();
-        let identity = manager.store.get_user_identity(&other_user).await.unwrap().unwrap();
+        let identity = manager.store.get_user_identity(other_user).await.unwrap().unwrap();
         let identity = identity.other().unwrap();
 
         assert!(identity.is_device_signed(&device).is_ok())
     }
+
+    #[async_test]
+    async fn no_tracked_users_key_query_request() {
+        let manager = manager();
+
+        assert!(manager.store.tracked_users().is_empty(), "No users are initially tracked");
+
+        let requests = manager.users_for_key_query().await;
+
+        assert!(!requests.is_empty(), "We query the keys for our own user");
+        assert!(
+            manager.store.tracked_users().contains(manager.user_id()),
+            "Our own user is now tracked"
+        );
+    }
 }

crates/matrix-sdk-crypto/src/identities/user.rs

@@ -23,22 +23,24 @@ use std::{
 };
 
 use ruma::{
-    api::client::r0::keys::upload_signatures::Request as SignatureUploadRequest,
-    encryption::{CrossSigningKey, DeviceKeys, KeyUsage},
+    api::client::keys::upload_signatures::v3::Request as SignatureUploadRequest,
+    encryption::KeyUsage,
     events::{
         key::verification::VerificationMethod, room::message::KeyVerificationRequestEventContent,
     },
-    DeviceIdBox, DeviceKeyId, EventId, RoomId, UserId,
+    DeviceKeyId, EventId, OwnedDeviceId, OwnedDeviceKeyId, OwnedUserId, RoomId, UserId,
 };
 use serde::{Deserialize, Serialize};
 use serde_json::to_value;
 use tracing::error;
+use vodozemac::Ed25519PublicKey;
 
 use super::{atomic_bool_deserializer, atomic_bool_serializer};
 use crate::{
     error::SignatureError,
-    olm::Utility,
+    olm::VerifyJson,
     store::{Changes, IdentityChanges},
+    types::{CrossSigningKey, DeviceKeys, SigningKey},
     verification::VerificationMachine,
     CryptoStoreError, OutgoingVerificationRequest, ReadOnlyDevice, VerificationRequest,
 };
@@ -122,7 +124,7 @@ impl OwnUserIdentity {
         };
 
         if let Err(e) = self.verification_machine.store.save_changes(changes).await {
-            error!(error =? e, "Couldn't store our own user identity after marking it as verified");
+            error!(error = ?e, "Couldn't store our own user identity after marking it as verified");
         }
 
         self.verification_machine.store.account.sign_master_key(self.master_key.clone()).await
@@ -165,7 +167,7 @@ impl OwnUserIdentity {
         &self,
         methods: Option<Vec<VerificationMethod>>,
     ) -> Result<(VerificationRequest, OutgoingVerificationRequest), CryptoStoreError> {
-        let devices: Vec<DeviceIdBox> = self
+        let devices: Vec<OwnedDeviceId> = self
             .verification_machine
             .store
             .get_user_devices(self.user_id())
@@ -225,16 +227,17 @@ impl UserIdentity {
     /// Returns a request that needs to be sent out for the user to be marked
     /// as verified.
     pub async fn verify(&self) -> Result<SignatureUploadRequest, SignatureError> {
-        if self.user_id() == self.verification_machine.own_user_id() {
+        if self.user_id() != self.verification_machine.own_user_id() {
             Ok(self
                 .verification_machine
+                .store
                 .private_identity
                 .lock()
                 .await
                 .sign_user(&self.inner)
                 .await?)
         } else {
-            Err(SignatureError::UserIdMissmatch)
+            Err(SignatureError::UserIdMismatch)
         }
     }
 
@@ -312,40 +315,19 @@ impl PartialEq for UserSigningPubkey {
 
 impl From<CrossSigningKey> for MasterPubkey {
     fn from(key: CrossSigningKey) -> Self {
-        Self(Arc::new(key))
+        Self(key.into())
     }
 }
 
 impl From<CrossSigningKey> for SelfSigningPubkey {
     fn from(key: CrossSigningKey) -> Self {
-        Self(Arc::new(key))
+        Self(key.into())
     }
 }
 
 impl From<CrossSigningKey> for UserSigningPubkey {
     fn from(key: CrossSigningKey) -> Self {
-        Self(Arc::new(key))
-    }
-}
-
-#[allow(clippy::from_over_into)]
-impl Into<CrossSigningKey> for MasterPubkey {
-    fn into(self) -> CrossSigningKey {
-        self.0.as_ref().clone()
-    }
-}
-
-#[allow(clippy::from_over_into)]
-impl Into<CrossSigningKey> for UserSigningPubkey {
-    fn into(self) -> CrossSigningKey {
-        self.0.as_ref().clone()
-    }
-}
-
-#[allow(clippy::from_over_into)]
-impl Into<CrossSigningKey> for SelfSigningPubkey {
-    fn into(self) -> CrossSigningKey {
-        self.0.as_ref().clone()
+        Self(key.into())
     }
 }
 
@@ -367,24 +349,6 @@ impl AsRef<CrossSigningKey> for UserSigningPubkey {
     }
 }
 
-impl From<&CrossSigningKey> for MasterPubkey {
-    fn from(key: &CrossSigningKey) -> Self {
-        Self(Arc::new(key.clone()))
-    }
-}
-
-impl From<&CrossSigningKey> for SelfSigningPubkey {
-    fn from(key: &CrossSigningKey) -> Self {
-        Self(Arc::new(key.clone()))
-    }
-}
-
-impl From<&CrossSigningKey> for UserSigningPubkey {
-    fn from(key: &CrossSigningKey) -> Self {
-        Self(Arc::new(key.clone()))
-    }
-}
-
 impl<'a> From<&'a SelfSigningPubkey> for CrossSigningSubKeys<'a> {
     fn from(key: &'a SelfSigningPubkey) -> Self {
         CrossSigningSubKeys::SelfSigning(key)
@@ -430,7 +394,7 @@ impl MasterPubkey {
     }
 
     /// Get the keys map of containing the master keys.
-    pub fn keys(&self) -> &BTreeMap<String, String> {
+    pub fn keys(&self) -> &BTreeMap<OwnedDeviceKeyId, SigningKey> {
         &self.0.keys
     }
 
@@ -440,7 +404,7 @@ impl MasterPubkey {
     }
 
     /// Get the signatures map of this cross signing key.
-    pub fn signatures(&self) -> &BTreeMap<UserId, BTreeMap<String, String>> {
+    pub fn signatures(&self) -> &BTreeMap<OwnedUserId, BTreeMap<OwnedDeviceKeyId, String>> {
         &self.0.signatures
     }
 
@@ -449,16 +413,20 @@ impl MasterPubkey {
     /// # Arguments
     ///
     /// * `key_id` - The id of the key that should be fetched.
-    pub fn get_key(&self, key_id: &DeviceKeyId) -> Option<&str> {
-        self.0.keys.get(key_id.as_str()).map(|k| k.as_str())
+    pub fn get_key(&self, key_id: &DeviceKeyId) -> Option<&SigningKey> {
+        self.0.keys.get(key_id)
     }
 
     /// Get the first available master key.
     ///
     /// There's usually only a single master key so this will usually fetch the
     /// only key.
-    pub fn get_first_key(&self) -> Option<&str> {
-        self.0.keys.values().map(|k| k.as_str()).next()
+    pub fn get_first_key(&self) -> Option<Ed25519PublicKey> {
+        if let Some(SigningKey::Ed25519(k)) = self.0.keys.values().next() {
+            Some(*k)
+        } else {
+            None
+        }
     }
 
     /// Check if the given cross signing sub-key is signed by the master key.
@@ -475,31 +443,34 @@ impl MasterPubkey {
     ) -> Result<(), SignatureError> {
         let (key_id, key) = self.0.keys.iter().next().ok_or(SignatureError::MissingSigningKey)?;
 
-        let key_id = DeviceKeyId::try_from(key_id.as_str())?;
+        let key_id = <&DeviceKeyId>::try_from(key_id.as_str())?;
 
         // FIXME `KeyUsage is missing PartialEq.
         // if self.0.usage.contains(&KeyUsage::Master) {
         //     return Err(SignatureError::MissingSigningKey);
         // }
-        let subkey: CrossSigningSubKeys = subkey.into();
+        let subkey: CrossSigningSubKeys<'_> = subkey.into();
 
-        if &self.0.user_id != subkey.user_id() {
-            return Err(SignatureError::UserIdMissmatch);
+        if self.0.user_id != subkey.user_id() {
+            return Err(SignatureError::UserIdMismatch);
         }
 
-        let utility = Utility::new();
-        utility.verify_json(
-            &self.0.user_id,
-            &key_id,
-            key,
-            &mut to_value(subkey.cross_signing_key()).map_err(|_| SignatureError::NotAnObject)?,
-        )
+        if let SigningKey::Ed25519(key) = key {
+            key.verify_json(
+                &self.0.user_id,
+                key_id,
+                &mut to_value(subkey.cross_signing_key())
+                    .map_err(|_| SignatureError::NotAnObject)?,
+            )
+        } else {
+            Err(SignatureError::UnsupportedAlgorithm)
+        }
     }
 }
 
 impl<'a> IntoIterator for &'a MasterPubkey {
-    type Item = (&'a String, &'a String);
-    type IntoIter = Iter<'a, String, String>;
+    type Item = (&'a OwnedDeviceKeyId, &'a SigningKey);
+    type IntoIter = Iter<'a, OwnedDeviceKeyId, SigningKey>;
 
     fn into_iter(self) -> Self::IntoIter {
         self.keys().iter()
@@ -513,7 +484,7 @@ impl UserSigningPubkey {
     }
 
     /// Get the keys map of containing the user signing keys.
-    pub fn keys(&self) -> &BTreeMap<String, String> {
+    pub fn keys(&self) -> &BTreeMap<OwnedDeviceKeyId, SigningKey> {
         &self.0.keys
     }
 
@@ -534,19 +505,21 @@ impl UserSigningPubkey {
 
         // TODO check that the usage is OK.
 
-        let utility = Utility::new();
-        utility.verify_json(
-            &self.0.user_id,
-            &DeviceKeyId::try_from(key_id.as_str())?,
-            key,
-            &mut to_value(&*master_key.0).map_err(|_| SignatureError::NotAnObject)?,
-        )
+        if let SigningKey::Ed25519(key) = key {
+            key.verify_json(
+                &self.0.user_id,
+                key_id.as_str().try_into()?,
+                &mut to_value(&master_key.0).map_err(|_| SignatureError::NotAnObject)?,
+            )
+        } else {
+            Err(SignatureError::UnsupportedAlgorithm)
+        }
     }
 }
 
 impl<'a> IntoIterator for &'a UserSigningPubkey {
-    type Item = (&'a String, &'a String);
-    type IntoIter = Iter<'a, String, String>;
+    type Item = (&'a OwnedDeviceKeyId, &'a SigningKey);
+    type IntoIter = Iter<'a, OwnedDeviceKeyId, SigningKey>;
 
     fn into_iter(self) -> Self::IntoIter {
         self.keys().iter()
@@ -560,7 +533,7 @@ impl SelfSigningPubkey {
     }
 
     /// Get the keys map of containing the self signing keys.
-    pub fn keys(&self) -> &BTreeMap<String, String> {
+    pub fn keys(&self) -> &BTreeMap<OwnedDeviceKeyId, SigningKey> {
         &self.0.keys
     }
 
@@ -570,14 +543,11 @@ impl SelfSigningPubkey {
 
         let mut device = to_value(device_keys)?;
 
-        let utility = Utility::new();
-
-        utility.verify_json(
-            &self.0.user_id,
-            &DeviceKeyId::try_from(key_id.as_str())?,
-            key,
-            &mut device,
-        )
+        if let SigningKey::Ed25519(key) = key {
+            key.verify_json(&self.0.user_id, key_id.as_str().try_into()?, &mut device)
+        } else {
+            Err(SignatureError::UnsupportedAlgorithm)
+        }
     }
 
     /// Check if the given device is signed by this self signing key.
@@ -589,13 +559,13 @@ impl SelfSigningPubkey {
     /// Returns an empty result if the signature check succeeded, otherwise a
     /// SignatureError indicating why the check failed.
     pub(crate) fn verify_device(&self, device: &ReadOnlyDevice) -> Result<(), SignatureError> {
-        self.verify_device_keys(device.as_device_keys())
+        self.verify_device_keys(device.as_device_keys().to_owned())
     }
 }
 
 impl<'a> IntoIterator for &'a SelfSigningPubkey {
-    type Item = (&'a String, &'a String);
-    type IntoIter = Iter<'a, String, String>;
+    type Item = (&'a OwnedDeviceKeyId, &'a SigningKey);
+    type IntoIter = Iter<'a, OwnedDeviceKeyId, SigningKey>;
 
     fn into_iter(self) -> Self::IntoIter {
         self.keys().iter()
@@ -718,7 +688,7 @@ impl ReadOnlyUserIdentity {
     ) -> Result<Self, SignatureError> {
         master_key.verify_subkey(&self_signing_key)?;
 
-        Ok(Self { user_id: Arc::new(master_key.0.user_id.clone()), master_key, self_signing_key })
+        Ok(Self { user_id: (&*master_key.0.user_id).into(), master_key, self_signing_key })
     }
 
     #[cfg(test)]
@@ -727,7 +697,7 @@ impl ReadOnlyUserIdentity {
         let self_signing_key =
             identity.self_signing_key.lock().await.as_ref().unwrap().public_key.clone();
 
-        Self { user_id: Arc::new(identity.user_id().clone()), master_key, self_signing_key }
+        Self { user_id: identity.user_id().into(), master_key, self_signing_key }
     }
 
     /// Get the user id of this identity.
@@ -781,7 +751,7 @@ impl ReadOnlyUserIdentity {
     /// SignatureError indicating why the check failed.
     pub(crate) fn is_device_signed(&self, device: &ReadOnlyDevice) -> Result<(), SignatureError> {
         if self.user_id() != device.user_id() {
-            return Err(SignatureError::UserIdMissmatch);
+            return Err(SignatureError::UserIdMismatch);
         }
 
         self.self_signing_key.verify_device(device)
@@ -831,7 +801,7 @@ impl ReadOnlyOwnUserIdentity {
         master_key.verify_subkey(&user_signing_key)?;
 
         Ok(Self {
-            user_id: Arc::new(master_key.0.user_id.clone()),
+            user_id: (&*master_key.0.user_id).into(),
             master_key,
             self_signing_key,
             user_signing_key,
@@ -889,7 +859,7 @@ impl ReadOnlyOwnUserIdentity {
     /// SignatureError indicating why the check failed.
     pub(crate) fn is_device_signed(&self, device: &ReadOnlyDevice) -> Result<(), SignatureError> {
         if self.user_id() != device.user_id() {
-            return Err(SignatureError::UserIdMissmatch);
+            return Err(SignatureError::UserIdMismatch);
         }
 
         self.self_signing_key.verify_device(device)
@@ -931,7 +901,7 @@ impl ReadOnlyOwnUserIdentity {
         self.user_signing_key = user_signing_key;
 
         if self.master_key != master_key {
-            self.verified.store(false, Ordering::SeqCst)
+            self.verified.store(false, Ordering::SeqCst);
         }
 
         self.master_key = master_key;
@@ -940,65 +910,96 @@ impl ReadOnlyOwnUserIdentity {
     }
 }
 
-#[cfg(test)]
-pub(crate) mod test {
-    use std::{convert::TryFrom, sync::Arc};
+#[cfg(any(test, feature = "testing"))]
+pub(crate) mod testing {
+    //! Testing Facilities
+    #![allow(dead_code)]
+    use ruma::{api::client::keys::get_keys::v3::Response as KeyQueryResponse, user_id};
 
-    use matrix_sdk_common::locks::Mutex;
-    use matrix_sdk_test::async_test;
-    use ruma::{api::client::r0::keys::get_keys::Response as KeyQueryResponse, user_id};
-
-    use super::{ReadOnlyOwnUserIdentity, ReadOnlyUserIdentities, ReadOnlyUserIdentity};
+    use super::{ReadOnlyOwnUserIdentity, ReadOnlyUserIdentity};
     use crate::{
         identities::{
-            manager::test::{other_key_query, own_key_query},
-            Device, ReadOnlyDevice,
+            manager::testing::{other_key_query, own_key_query},
+            ReadOnlyDevice,
         },
-        olm::{PrivateCrossSigningIdentity, ReadOnlyAccount},
-        store::MemoryStore,
-        verification::VerificationMachine,
+        types::CrossSigningKey,
     };
 
-    fn device(response: &KeyQueryResponse) -> (ReadOnlyDevice, ReadOnlyDevice) {
+    /// Generate test devices from KeyQueryResponse
+    pub fn device(response: &KeyQueryResponse) -> (ReadOnlyDevice, ReadOnlyDevice) {
         let mut devices = response.device_keys.values().next().unwrap().values();
-        let first = ReadOnlyDevice::try_from(devices.next().unwrap()).unwrap();
-        let second = ReadOnlyDevice::try_from(devices.next().unwrap()).unwrap();
+        let first =
+            ReadOnlyDevice::try_from(&devices.next().unwrap().deserialize_as().unwrap()).unwrap();
+        let second =
+            ReadOnlyDevice::try_from(&devices.next().unwrap().deserialize_as().unwrap()).unwrap();
         (first, second)
     }
 
-    fn own_identity(response: &KeyQueryResponse) -> ReadOnlyOwnUserIdentity {
+    /// Generate ReadOnlyOwnUserIdentity from KeyQueryResponse for testing
+    pub fn own_identity(response: &KeyQueryResponse) -> ReadOnlyOwnUserIdentity {
         let user_id = user_id!("@example:localhost");
 
-        let master_key = response.master_keys.get(&user_id).unwrap();
-        let user_signing = response.user_signing_keys.get(&user_id).unwrap();
-        let self_signing = response.self_signing_keys.get(&user_id).unwrap();
+        let master_key: CrossSigningKey =
+            response.master_keys.get(user_id).unwrap().deserialize_as().unwrap();
+        let user_signing: CrossSigningKey =
+            response.user_signing_keys.get(user_id).unwrap().deserialize_as().unwrap();
+        let self_signing: CrossSigningKey =
+            response.self_signing_keys.get(user_id).unwrap().deserialize_as().unwrap();
 
         ReadOnlyOwnUserIdentity::new(master_key.into(), self_signing.into(), user_signing.into())
             .unwrap()
     }
 
-    pub(crate) fn get_own_identity() -> ReadOnlyOwnUserIdentity {
+    /// Generate default own identity for tests
+    pub fn get_own_identity() -> ReadOnlyOwnUserIdentity {
         own_identity(&own_key_query())
     }
 
-    pub(crate) fn get_other_identity() -> ReadOnlyUserIdentity {
+    /// Generate default other identify for tests
+    pub fn get_other_identity() -> ReadOnlyUserIdentity {
         let user_id = user_id!("@example2:localhost");
         let response = other_key_query();
 
-        let master_key = response.master_keys.get(&user_id).unwrap();
-        let self_signing = response.self_signing_keys.get(&user_id).unwrap();
+        let master_key: CrossSigningKey =
+            response.master_keys.get(user_id).unwrap().deserialize_as().unwrap();
+        let self_signing: CrossSigningKey =
+            response.self_signing_keys.get(user_id).unwrap().deserialize_as().unwrap();
 
         ReadOnlyUserIdentity::new(master_key.into(), self_signing.into()).unwrap()
     }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use std::sync::Arc;
+
+    use matrix_sdk_common::locks::Mutex;
+    use matrix_sdk_test::async_test;
+    use ruma::user_id;
+
+    use super::{
+        testing::{device, get_other_identity, get_own_identity},
+        ReadOnlyOwnUserIdentity, ReadOnlyUserIdentities,
+    };
+    use crate::{
+        identities::{manager::testing::own_key_query, Device},
+        olm::{PrivateCrossSigningIdentity, ReadOnlyAccount},
+        store::MemoryStore,
+        types::CrossSigningKey,
+        verification::VerificationMachine,
+    };
 
     #[test]
     fn own_identity_create() {
         let user_id = user_id!("@example:localhost");
         let response = own_key_query();
 
-        let master_key = response.master_keys.get(&user_id).unwrap();
-        let user_signing = response.user_signing_keys.get(&user_id).unwrap();
-        let self_signing = response.self_signing_keys.get(&user_id).unwrap();
+        let master_key: CrossSigningKey =
+            response.master_keys.get(user_id).unwrap().deserialize_as().unwrap();
+        let user_signing: CrossSigningKey =
+            response.user_signing_keys.get(user_id).unwrap().deserialize_as().unwrap();
+        let self_signing: CrossSigningKey =
+            response.self_signing_keys.get(user_id).unwrap().deserialize_as().unwrap();
 
         ReadOnlyOwnUserIdentity::new(master_key.into(), self_signing.into(), user_signing.into())
             .unwrap();
@@ -1019,7 +1020,7 @@ pub(crate) mod test {
         assert!(identity.is_device_signed(&second).is_ok());
 
         let private_identity =
-            Arc::new(Mutex::new(PrivateCrossSigningIdentity::empty(second.user_id().clone())));
+            Arc::new(Mutex::new(PrivateCrossSigningIdentity::empty(second.user_id())));
         let verification_machine = VerificationMachine::new(
             ReadOnlyAccount::new(second.user_id(), second.device_id()),
             private_identity,
@@ -1057,7 +1058,7 @@ pub(crate) mod test {
         let (_, device) = device(&response);
 
         let account = ReadOnlyAccount::new(device.user_id(), device.device_id());
-        let (identity, _, _) = PrivateCrossSigningIdentity::new_with_account(&account).await;
+        let (identity, _, _) = PrivateCrossSigningIdentity::with_account(&account).await;
 
         let id = Arc::new(Mutex::new(identity.clone()));
 
@@ -1078,10 +1079,10 @@ pub(crate) mod test {
 
         assert!(!device.verified());
 
-        let mut device_keys = device.as_device_keys();
+        let mut device_keys = device.as_device_keys().to_owned();
 
         identity.sign_device_keys(&mut device_keys).await.unwrap();
-        device.inner.signatures = Arc::new(device_keys.signatures);
+        device.inner.update_device(&device_keys).expect("Couldn't update newly signed device keys");
         assert!(device.verified());
     }
 }

crates/matrix-sdk-crypto/src/lib.rs

@@ -0,0 +1,92 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![doc = include_str!("../README.md")]
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+#![warn(missing_docs, missing_debug_implementations)]
+
+#[cfg(feature = "backups_v1")]
+pub mod backups;
+mod error;
+mod file_encryption;
+mod gossiping;
+mod identities;
+mod machine;
+pub mod olm;
+mod requests;
+mod session_manager;
+pub mod store;
+pub mod types;
+mod utilities;
+mod verification;
+
+#[cfg(feature = "testing")]
+/// Testing facilities and helpers for crypto tests
+pub mod testing {
+    pub use crate::identities::{
+        device::testing::get_device,
+        user::testing::{get_other_identity, get_own_identity},
+    };
+}
+
+use std::collections::{BTreeMap, BTreeSet};
+
+use ruma::OwnedRoomId;
+
+/// Return type for the room key importing.
+#[derive(Debug, Clone, PartialEq)]
+pub struct RoomKeyImportResult {
+    /// The number of room keys that were imported.
+    pub imported_count: usize,
+    /// The total number of room keys that were found in the export.
+    pub total_count: usize,
+    /// The map of keys that were imported.
+    ///
+    /// It's a map from room id to a map of the sender key to a set of session
+    /// ids.
+    pub keys: BTreeMap<OwnedRoomId, BTreeMap<String, BTreeSet<String>>>,
+}
+
+impl RoomKeyImportResult {
+    pub(crate) fn new(
+        imported_count: usize,
+        total_count: usize,
+        keys: BTreeMap<OwnedRoomId, BTreeMap<String, BTreeSet<String>>>,
+    ) -> Self {
+        Self { imported_count, total_count, keys }
+    }
+}
+
+pub use error::{MegolmError, OlmError, SignatureError};
+pub use file_encryption::{
+    decrypt_key_export, encrypt_key_export, AttachmentDecryptor, AttachmentEncryptor,
+    DecryptorError, KeyExportError, MediaEncryptionInfo,
+};
+pub use gossiping::GossipRequest;
+pub use identities::{
+    Device, LocalTrust, MasterPubkey, OwnUserIdentity, ReadOnlyDevice, ReadOnlyOwnUserIdentity,
+    ReadOnlyUserIdentities, ReadOnlyUserIdentity, UserDevices, UserIdentities, UserIdentity,
+};
+pub use machine::OlmMachine;
+#[cfg(feature = "qrcode")]
+pub use matrix_sdk_qrcode;
+pub use olm::{CrossSigningStatus, EncryptionSettings, ReadOnlyAccount};
+pub use requests::{
+    IncomingResponse, KeysBackupRequest, KeysQueryRequest, OutgoingRequest, OutgoingRequests,
+    OutgoingVerificationRequest, RoomMessageRequest, ToDeviceRequest, UploadSigningKeysRequest,
+};
+pub use store::{CrossSigningKeyExport, CryptoStoreError, SecretImportError, SecretInfo};
+pub use verification::{AcceptSettings, CancelInfo, Emoji, Sas, Verification, VerificationRequest};
+#[cfg(feature = "qrcode")]
+pub use verification::{QrVerification, ScanError};

crates/matrix-sdk-crypto/src/olm/group_sessions/inbound.rs

@@ -12,34 +12,42 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::{collections::BTreeMap, convert::TryFrom, fmt, mem, sync::Arc};
+#![warn(missing_docs)]
+
+use std::{
+    collections::BTreeMap,
+    fmt,
+    sync::{
+        atomic::{AtomicBool, Ordering::SeqCst},
+        Arc,
+    },
+};
 
 use matrix_sdk_common::locks::Mutex;
-pub use olm_rs::{
-    account::IdentityKeys,
-    session::{OlmMessage, PreKeyMessage},
-    utility::OlmUtility,
-};
-use olm_rs::{
-    errors::OlmGroupSessionError, inbound_group_session::OlmInboundGroupSession, PicklingMode,
-};
 use ruma::{
     events::{
-        forwarded_room_key::ForwardedRoomKeyToDeviceEventContent,
+        forwarded_room_key::ToDeviceForwardedRoomKeyEventContent,
         room::{
-            encrypted::{EncryptedEventContent, EncryptedEventScheme},
+            encrypted::{EncryptedEventScheme, OriginalSyncRoomEncryptedEvent},
             history_visibility::HistoryVisibility,
         },
-        AnySyncRoomEvent, SyncMessageEvent,
+        AnyRoomEvent,
     },
     serde::Raw,
-    DeviceKeyAlgorithm, EventEncryptionAlgorithm, RoomId,
+    DeviceKeyAlgorithm, EventEncryptionAlgorithm, OwnedRoomId, RoomId,
 };
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use zeroize::Zeroizing;
+use vodozemac::{
+    megolm::{
+        DecryptedMessage, DecryptionError, ExportedSessionKey, InboundGroupSession as InnerSession,
+        InboundGroupSessionPickle, MegolmMessage, SessionKeyDecodeError,
+    },
+    PickleError,
+};
+use zeroize::Zeroize;
 
-use super::{ExportedGroupSessionKey, ExportedRoomKey, GroupSessionKey};
+use super::{BackedUpRoomKey, ExportedRoomKey, SessionKey};
 use crate::error::{EventError, MegolmResult};
 
 // TODO add creation times to the inbound group sessions so we can export
@@ -52,16 +60,20 @@ use crate::error::{EventError, MegolmResult};
 /// participants. Inbound group sessions are used to decrypt the room messages.
 #[derive(Clone)]
 pub struct InboundGroupSession {
-    inner: Arc<Mutex<OlmInboundGroupSession>>,
+    inner: Arc<Mutex<InnerSession>>,
     history_visibility: Arc<Option<HistoryVisibility>>,
-    session_id: Arc<str>,
+    /// The SessionId associated to this GroupSession
+    pub session_id: Arc<str>,
     first_known_index: u32,
-    pub(crate) sender_key: Arc<str>,
-    pub(crate) signing_keys: Arc<BTreeMap<DeviceKeyAlgorithm, String>>,
-    pub(crate) room_id: Arc<RoomId>,
+    /// The sender_key associated to this GroupSession
+    pub sender_key: Arc<str>,
+    /// Map of DeviceKeyAlgorithm to the public ed25519 key of the account
+    pub signing_keys: Arc<BTreeMap<DeviceKeyAlgorithm, String>>,
+    /// The Room this GroupSession belongs to
+    pub room_id: Arc<RoomId>,
     forwarding_chains: Arc<Vec<String>>,
     imported: bool,
-    backed_up: bool,
+    backed_up: Arc<AtomicBool>,
 }
 
 impl InboundGroupSession {
@@ -81,32 +93,32 @@ impl InboundGroupSession {
     ///
     /// * `session_key` - The private session key that is used to decrypt
     /// messages.
-    pub(crate) fn new(
+    pub fn new(
         sender_key: &str,
         signing_key: &str,
         room_id: &RoomId,
-        session_key: GroupSessionKey,
+        session_key: SessionKey,
         history_visibility: Option<HistoryVisibility>,
-    ) -> Result<Self, OlmGroupSessionError> {
-        let session = OlmInboundGroupSession::new(&session_key.0)?;
+    ) -> Self {
+        let session = InnerSession::new(&session_key);
         let session_id = session.session_id();
         let first_known_index = session.first_known_index();
 
         let mut keys: BTreeMap<DeviceKeyAlgorithm, String> = BTreeMap::new();
         keys.insert(DeviceKeyAlgorithm::Ed25519, signing_key.to_owned());
 
-        Ok(InboundGroupSession {
+        InboundGroupSession {
             inner: Arc::new(Mutex::new(session)),
             session_id: session_id.into(),
             history_visibility: history_visibility.into(),
             sender_key: sender_key.to_owned().into(),
             first_known_index,
             signing_keys: keys.into(),
-            room_id: room_id.clone().into(),
+            room_id: room_id.into(),
             forwarding_chains: Vec::new().into(),
             imported: false,
-            backed_up: false,
-        })
+            backed_up: AtomicBool::new(false).into(),
+        }
     }
 
     /// Create a InboundGroupSession from an exported version of the group
@@ -115,12 +127,25 @@ impl InboundGroupSession {
     /// Most notably this can be called with an `ExportedRoomKey` from a
     /// previous [`export()`] call.
     ///
-    ///
     /// [`export()`]: #method.export
-    pub fn from_export(
-        exported_session: impl Into<ExportedRoomKey>,
-    ) -> Result<Self, OlmGroupSessionError> {
-        Self::try_from(exported_session.into())
+    pub fn from_export(exported_session: ExportedRoomKey) -> Self {
+        Self::from(exported_session)
+    }
+
+    #[allow(dead_code)]
+    fn from_backup(room_id: &RoomId, backup: BackedUpRoomKey) -> Self {
+        let session = InnerSession::import(&backup.session_key);
+        let session_id = session.session_id();
+
+        Self::from_export(ExportedRoomKey {
+            algorithm: backup.algorithm,
+            room_id: room_id.to_owned(),
+            sender_key: backup.sender_key,
+            session_id,
+            session_key: backup.session_key,
+            sender_claimed_keys: backup.sender_claimed_keys,
+            forwarding_curve25519_key_chain: backup.forwarding_curve25519_key_chain,
+        })
     }
 
     /// Create a new inbound group session from a forwarded room key content.
@@ -132,13 +157,14 @@ impl InboundGroupSession {
     ///
     /// * `content` - A forwarded room key content that contains the session key
     /// to create the `InboundGroupSession`.
-    pub(crate) fn from_forwarded_key(
+    pub fn from_forwarded_key(
         sender_key: &str,
-        content: &mut ForwardedRoomKeyToDeviceEventContent,
-    ) -> Result<Self, OlmGroupSessionError> {
-        let key = Zeroizing::from(mem::take(&mut content.session_key));
+        content: &mut ToDeviceForwardedRoomKeyEventContent,
+    ) -> Result<Self, SessionKeyDecodeError> {
+        let key = ExportedSessionKey::from_base64(&content.session_key)?;
+        content.session_key.zeroize();
 
-        let session = OlmInboundGroupSession::import(&key)?;
+        let session = InnerSession::import(&key);
         let first_known_index = session.first_known_index();
         let mut forwarding_chains = content.forwarding_curve25519_key_chain.clone();
         forwarding_chains.push(sender_key.to_owned());
@@ -154,10 +180,10 @@ impl InboundGroupSession {
             first_known_index,
             history_visibility: None.into(),
             signing_keys: sender_claimed_key.into(),
-            room_id: content.room_id.clone().into(),
+            room_id: (&*content.room_id).into(),
             forwarding_chains: forwarding_chains.into(),
             imported: true,
-            backed_up: false,
+            backed_up: AtomicBool::new(false).into(),
         })
     }
 
@@ -167,17 +193,17 @@ impl InboundGroupSession {
     ///
     /// * `pickle_mode` - The mode that was used to pickle the group session,
     /// either an unencrypted mode or an encrypted using passphrase.
-    pub async fn pickle(&self, pickle_mode: PicklingMode) -> PickledInboundGroupSession {
-        let pickle = self.inner.lock().await.pickle(pickle_mode);
+    pub async fn pickle(&self) -> PickledInboundGroupSession {
+        let pickle = self.inner.lock().await.pickle();
 
         PickledInboundGroupSession {
-            pickle: InboundGroupSessionPickle::from(pickle),
+            pickle,
             sender_key: self.sender_key.to_string(),
-            signing_key: (&*self.signing_keys).clone(),
-            room_id: (&*self.room_id).clone(),
+            signing_key: (*self.signing_keys).clone(),
+            room_id: (*self.room_id).to_owned(),
             forwarding_chains: self.forwarding_key_chain().to_vec(),
             imported: self.imported,
-            backed_up: self.backed_up,
+            backed_up: self.backed_up(),
             history_visibility: self.history_visibility.as_ref().clone(),
         }
     }
@@ -195,6 +221,22 @@ impl InboundGroupSession {
         &self.sender_key
     }
 
+    /// Has the session been backed up to the server.
+    pub fn backed_up(&self) -> bool {
+        self.backed_up.load(SeqCst)
+    }
+
+    /// Reset the backup state of the inbound group session.
+    pub fn reset_backup_state(&self) {
+        self.backed_up.store(false, SeqCst)
+    }
+
+    /// For testing, allow to manually mark this GroupSession to have been
+    /// backed up
+    pub fn mark_as_backed_up(&self) {
+        self.backed_up.store(true, SeqCst)
+    }
+
     /// Get the map of signing keys this session was received from.
     pub fn signing_keys(&self) -> &BTreeMap<DeviceKeyAlgorithm, String> {
         &self.signing_keys
@@ -213,24 +255,23 @@ impl InboundGroupSession {
     pub async fn export_at_index(&self, message_index: u32) -> ExportedRoomKey {
         let message_index = std::cmp::max(self.first_known_index(), message_index);
 
-        let session_key = ExportedGroupSessionKey(
-            self.inner.lock().await.export(message_index).expect("Can't export session"),
-        );
+        let session_key =
+            self.inner.lock().await.export_at(message_index).expect("Can't export session");
 
         ExportedRoomKey {
             algorithm: EventEncryptionAlgorithm::MegolmV1AesSha2,
-            room_id: (&*self.room_id).clone(),
-            sender_key: (&*self.sender_key).to_owned(),
+            room_id: (*self.room_id).to_owned(),
+            sender_key: (*self.sender_key).to_owned(),
             session_id: self.session_id().to_owned(),
             forwarding_curve25519_key_chain: self.forwarding_key_chain().to_vec(),
-            sender_claimed_keys: (&*self.signing_keys).clone(),
+            sender_claimed_keys: (*self.signing_keys).clone(),
             session_key,
         }
     }
 
     /// Restore a Session from a previously pickled string.
     ///
-    /// Returns the restored group session or a `OlmGroupSessionError` if there
+    /// Returns the restored group session or a `UnpicklingError` if there
     /// was an error.
     ///
     /// # Arguments
@@ -239,11 +280,8 @@ impl InboundGroupSession {
     ///
     /// * `pickle_mode` - The mode that was used to pickle the session, either
     /// an unencrypted mode or an encrypted using passphrase.
-    pub fn from_pickle(
-        pickle: PickledInboundGroupSession,
-        pickle_mode: PicklingMode,
-    ) -> Result<Self, OlmGroupSessionError> {
-        let session = OlmInboundGroupSession::unpickle(pickle.pickle.0, pickle_mode)?;
+    pub fn from_pickle(pickle: PickledInboundGroupSession) -> Result<Self, PickleError> {
+        let session: InnerSession = pickle.pickle.into();
         let first_known_index = session.first_known_index();
         let session_id = session.session_id();
 
@@ -254,9 +292,9 @@ impl InboundGroupSession {
             history_visibility: pickle.history_visibility.into(),
             first_known_index,
             signing_keys: pickle.signing_key.into(),
-            room_id: pickle.room_id.into(),
+            room_id: (&*pickle.room_id).into(),
             forwarding_chains: pickle.forwarding_chains.into(),
-            backed_up: pickle.backed_up,
+            backed_up: AtomicBool::from(pickle.backed_up).into(),
             imported: pickle.imported,
         })
     }
@@ -278,7 +316,7 @@ impl InboundGroupSession {
 
     /// Decrypt the given ciphertext.
     ///
-    /// Returns the decrypted plaintext or an `OlmGroupSessionError` if
+    /// Returns the decrypted plaintext or an `DecryptionError` if
     /// decryption failed.
     ///
     /// # Arguments
@@ -286,28 +324,37 @@ impl InboundGroupSession {
     /// * `message` - The message that should be decrypted.
     pub(crate) async fn decrypt_helper(
         &self,
-        message: String,
-    ) -> Result<(String, u32), OlmGroupSessionError> {
+        message: &MegolmMessage,
+    ) -> Result<DecryptedMessage, DecryptionError> {
         self.inner.lock().await.decrypt(message)
     }
 
+    /// Export the inbound group session into a format that can be uploaded to
+    /// the server as a backup.
+    #[cfg(feature = "backups_v1")]
+    pub async fn to_backup(&self) -> BackedUpRoomKey {
+        self.export().await.into()
+    }
+
     /// Decrypt an event from a room timeline.
     ///
     /// # Arguments
     ///
     /// * `event` - The event that should be decrypted.
-    pub(crate) async fn decrypt(
+    pub async fn decrypt(
         &self,
-        event: &SyncMessageEvent<EncryptedEventContent>,
-    ) -> MegolmResult<(Raw<AnySyncRoomEvent>, u32)> {
+        event: &OriginalSyncRoomEncryptedEvent,
+    ) -> MegolmResult<(Raw<AnyRoomEvent>, u32)> {
         let content = match &event.content.scheme {
             EncryptedEventScheme::MegolmV1AesSha2(c) => c,
             _ => return Err(EventError::UnsupportedAlgorithm.into()),
         };
 
-        let (plaintext, message_index) = self.decrypt_helper(content.ciphertext.clone()).await?;
+        let message = MegolmMessage::from_base64(&content.ciphertext)?;
 
-        let mut decrypted_value = serde_json::from_str::<Value>(&plaintext)?;
+        let decrypted = self.decrypt_helper(&message).await?;
+
+        let mut decrypted_value = serde_json::from_str::<Value>(&decrypted.plaintext)?;
         let decrypted_object = decrypted_value.as_object_mut().ok_or(EventError::NotAnObject)?;
 
         let server_ts: i64 = event.origin_server_ts.0.into();
@@ -316,13 +363,23 @@ impl InboundGroupSession {
         decrypted_object.insert("event_id".to_owned(), event.event_id.to_string().into());
         decrypted_object.insert("origin_server_ts".to_owned(), server_ts.into());
 
+        let room_id = decrypted_object
+            .get("room_id")
+            .and_then(|r| r.as_str().and_then(|r| RoomId::parse(r).ok()));
+
+        // Check that we have a room id and that the event wasn't forwarded from
+        // another room.
+        if room_id.as_deref() != Some(self.room_id()) {
+            return Err(EventError::MismatchedRoom(self.room_id().to_owned(), room_id).into());
+        }
+
         decrypted_object.insert(
             "unsigned".to_owned(),
             serde_json::to_value(&event.unsigned).unwrap_or_default(),
         );
 
         if let Some(decrypted_content) =
-            decrypted_object.get_mut("content").map(|c| c.as_object_mut()).flatten()
+            decrypted_object.get_mut("content").and_then(|c| c.as_object_mut())
         {
             if !decrypted_content.contains_key("m.relates_to") {
                 let content = serde_json::to_value(&event.content)?;
@@ -332,7 +389,7 @@ impl InboundGroupSession {
             }
         }
 
-        Ok((serde_json::from_value::<Raw<AnySyncRoomEvent>>(decrypted_value)?, message_index))
+        Ok((serde_json::from_value::<Raw<AnyRoomEvent>>(decrypted_value)?, decrypted.message_index))
     }
 }
 
@@ -353,7 +410,8 @@ impl PartialEq for InboundGroupSession {
 ///
 /// Holds all the information that needs to be stored in a database to restore
 /// an InboundGroupSession.
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Serialize, Deserialize)]
+#[allow(missing_debug_implementations)]
 pub struct PickledInboundGroupSession {
     /// The pickle string holding the InboundGroupSession.
     pub pickle: InboundGroupSessionPickle,
@@ -362,7 +420,7 @@ pub struct PickledInboundGroupSession {
     /// The public ed25519 key of the account that sent us the session.
     pub signing_key: BTreeMap<DeviceKeyAlgorithm, String>,
     /// The id of the room that the session is used in.
-    pub room_id: RoomId,
+    pub room_id: OwnedRoomId,
     /// The list of claimed ed25519 that forwarded us this key. Will be None if
     /// we directly received this session.
     #[serde(default)]
@@ -377,42 +435,22 @@ pub struct PickledInboundGroupSession {
     pub history_visibility: Option<HistoryVisibility>,
 }
 
-/// The typed representation of a base64 encoded string of the GroupSession
-/// pickle.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct InboundGroupSessionPickle(String);
-
-impl From<String> for InboundGroupSessionPickle {
-    fn from(pickle_string: String) -> Self {
-        InboundGroupSessionPickle(pickle_string)
-    }
-}
-
-impl InboundGroupSessionPickle {
-    /// Get the string representation of the pickle.
-    pub fn as_str(&self) -> &str {
-        &self.0
-    }
-}
-
-impl TryFrom<ExportedRoomKey> for InboundGroupSession {
-    type Error = OlmGroupSessionError;
-
-    fn try_from(key: ExportedRoomKey) -> Result<Self, Self::Error> {
-        let session = OlmInboundGroupSession::import(&key.session_key.0)?;
+impl From<ExportedRoomKey> for InboundGroupSession {
+    fn from(key: ExportedRoomKey) -> Self {
+        let session = InnerSession::import(&key.session_key);
         let first_known_index = session.first_known_index();
 
-        Ok(InboundGroupSession {
-            inner: Arc::new(Mutex::new(session)),
+        InboundGroupSession {
+            inner: Mutex::new(session).into(),
             session_id: key.session_id.into(),
             sender_key: key.sender_key.into(),
             history_visibility: None.into(),
             first_known_index,
-            signing_keys: Arc::new(key.sender_claimed_keys),
-            room_id: Arc::new(key.room_id),
-            forwarding_chains: Arc::new(key.forwarding_curve25519_key_chain),
+            signing_keys: key.sender_claimed_keys.into(),
+            room_id: (&*key.room_id).into(),
+            forwarding_chains: key.forwarding_curve25519_key_chain.into(),
             imported: true,
-            backed_up: false,
-        })
+            backed_up: AtomicBool::from(false).into(),
+        }
     }
 }

crates/matrix-sdk-crypto/src/olm/group_sessions/mod.rs

@@ -16,43 +16,35 @@ use std::{collections::BTreeMap, convert::TryInto};
 
 use ruma::{
     events::forwarded_room_key::{
-        ForwardedRoomKeyToDeviceEventContent, ForwardedRoomKeyToDeviceEventContentInit,
+        ToDeviceForwardedRoomKeyEventContent, ToDeviceForwardedRoomKeyEventContentInit,
     },
-    DeviceKeyAlgorithm, EventEncryptionAlgorithm, RoomId,
+    DeviceKeyAlgorithm, EventEncryptionAlgorithm, OwnedRoomId,
 };
 use serde::{Deserialize, Serialize};
-use zeroize::Zeroize;
 
 mod inbound;
 mod outbound;
 
-pub use inbound::{InboundGroupSession, InboundGroupSessionPickle, PickledInboundGroupSession};
+pub use inbound::{InboundGroupSession, PickledInboundGroupSession};
+pub(crate) use outbound::ShareState;
 pub use outbound::{
-    EncryptionSettings, OutboundGroupSession, PickledOutboundGroupSession, ShareInfo, ShareState,
+    EncryptionSettings, GroupSession, OutboundGroupSession, PickledOutboundGroupSession, ShareInfo,
 };
+use vodozemac::megolm::SessionKeyDecodeError;
+pub use vodozemac::megolm::{ExportedSessionKey, SessionKey};
+use zeroize::Zeroize;
 
-/// The private session key of a group session.
-/// Can be used to create a new inbound group session.
-#[derive(Clone, Debug, Serialize, Deserialize, Zeroize)]
-#[zeroize(drop)]
-pub struct GroupSessionKey(pub String);
-
-/// The exported version of an private session key of a group session.
-/// Can be used to create a new inbound group session.
-#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Zeroize)]
-#[zeroize(drop)]
-pub struct ExportedGroupSessionKey(pub String);
-
-/// An exported version of a `InboundGroupSession`
+/// An exported version of an `InboundGroupSession`
 ///
 /// This can be used to share the `InboundGroupSession` in an exported file.
-#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+#[derive(Deserialize, Serialize)]
+#[allow(missing_debug_implementations)]
 pub struct ExportedRoomKey {
     /// The encryption algorithm that the session uses.
     pub algorithm: EventEncryptionAlgorithm,
 
     /// The room where the session is used.
-    pub room_id: RoomId,
+    pub room_id: OwnedRoomId,
 
     /// The Curve25519 key of the device which initiated the session originally.
     pub sender_key: String,
@@ -61,7 +53,32 @@ pub struct ExportedRoomKey {
     pub session_id: String,
 
     /// The key for the session.
-    pub session_key: ExportedGroupSessionKey,
+    pub session_key: ExportedSessionKey,
+
+    /// The Ed25519 key of the device which initiated the session originally.
+    #[serde(default)]
+    pub sender_claimed_keys: BTreeMap<DeviceKeyAlgorithm, String>,
+
+    /// Chain of Curve25519 keys through which this session was forwarded, via
+    /// m.forwarded_room_key events.
+    #[serde(default)]
+    pub forwarding_curve25519_key_chain: Vec<String>,
+}
+
+/// A backed up version of an `InboundGroupSession`
+///
+/// This can be used to backup the `InboundGroupSession` to the server.
+#[derive(Deserialize, Serialize)]
+#[allow(missing_debug_implementations)]
+pub struct BackedUpRoomKey {
+    /// The encryption algorithm that the session uses.
+    pub algorithm: EventEncryptionAlgorithm,
+
+    /// The Curve25519 key of the device which initiated the session originally.
+    pub sender_key: String,
+
+    /// The key for the session.
+    pub session_key: ExportedSessionKey,
 
     /// The Ed25519 key of the device which initiated the session originally.
     pub sender_claimed_keys: BTreeMap<DeviceKeyAlgorithm, String>,
@@ -71,7 +88,7 @@ pub struct ExportedRoomKey {
     pub forwarding_curve25519_key_chain: Vec<String>,
 }
 
-impl TryInto<ForwardedRoomKeyToDeviceEventContent> for ExportedRoomKey {
+impl TryInto<ToDeviceForwardedRoomKeyEventContent> for ExportedRoomKey {
     type Error = ();
 
     /// Convert an exported room key into a content for a forwarded room key
@@ -80,7 +97,7 @@ impl TryInto<ForwardedRoomKeyToDeviceEventContent> for ExportedRoomKey {
     /// This will fail if the exported room key has multiple sender claimed keys
     /// or if the algorithm of the claimed sender key isn't
     /// `DeviceKeyAlgorithm::Ed25519`.
-    fn try_into(self) -> Result<ForwardedRoomKeyToDeviceEventContent, Self::Error> {
+    fn try_into(self) -> Result<ToDeviceForwardedRoomKeyEventContent, Self::Error> {
         if self.sender_claimed_keys.len() != 1 {
             Err(())
         } else {
@@ -90,12 +107,12 @@ impl TryInto<ForwardedRoomKeyToDeviceEventContent> for ExportedRoomKey {
                 return Err(());
             }
 
-            Ok(ForwardedRoomKeyToDeviceEventContentInit {
+            Ok(ToDeviceForwardedRoomKeyEventContentInit {
                 algorithm: self.algorithm,
                 room_id: self.room_id,
                 sender_key: self.sender_key,
                 session_id: self.session_id,
-                session_key: self.session_key.0.clone(),
+                session_key: self.session_key.to_base64(),
                 sender_claimed_ed25519_key: claimed_key.to_owned(),
                 forwarding_curve25519_key_chain: self.forwarding_curve25519_key_chain,
             }
@@ -104,71 +121,40 @@ impl TryInto<ForwardedRoomKeyToDeviceEventContent> for ExportedRoomKey {
     }
 }
 
-impl From<ForwardedRoomKeyToDeviceEventContent> for ExportedRoomKey {
+impl From<ExportedRoomKey> for BackedUpRoomKey {
+    fn from(k: ExportedRoomKey) -> Self {
+        Self {
+            algorithm: k.algorithm,
+            sender_key: k.sender_key,
+            session_key: k.session_key,
+            sender_claimed_keys: k.sender_claimed_keys,
+            forwarding_curve25519_key_chain: k.forwarding_curve25519_key_chain,
+        }
+    }
+}
+
+impl TryFrom<ToDeviceForwardedRoomKeyEventContent> for ExportedRoomKey {
+    type Error = SessionKeyDecodeError;
+
     /// Convert the content of a forwarded room key into a exported room key.
-    fn from(forwarded_key: ForwardedRoomKeyToDeviceEventContent) -> Self {
+    fn try_from(
+        mut forwarded_key: ToDeviceForwardedRoomKeyEventContent,
+    ) -> Result<Self, Self::Error> {
         let mut sender_claimed_keys: BTreeMap<DeviceKeyAlgorithm, String> = BTreeMap::new();
         sender_claimed_keys
             .insert(DeviceKeyAlgorithm::Ed25519, forwarded_key.sender_claimed_ed25519_key);
 
-        Self {
+        let session_key = ExportedSessionKey::from_base64(&forwarded_key.session_key)?;
+        forwarded_key.session_key.zeroize();
+
+        Ok(Self {
             algorithm: forwarded_key.algorithm,
             room_id: forwarded_key.room_id,
             session_id: forwarded_key.session_id,
             forwarding_curve25519_key_chain: forwarded_key.forwarding_curve25519_key_chain,
             sender_claimed_keys,
             sender_key: forwarded_key.sender_key,
-            session_key: ExportedGroupSessionKey(forwarded_key.session_key),
-        }
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use std::{
-        sync::Arc,
-        time::{Duration, Instant},
-    };
-
-    use ruma::{
-        events::{room::message::MessageEventContent, AnyMessageEventContent},
-        room_id, user_id,
-    };
-
-    use super::EncryptionSettings;
-    use crate::ReadOnlyAccount;
-
-    #[tokio::test]
-    #[cfg(target_os = "linux")]
-    async fn expiration() {
-        let settings = EncryptionSettings { rotation_period_msgs: 1, ..Default::default() };
-
-        let account = ReadOnlyAccount::new(&user_id!("@alice:example.org"), "DEVICEID".into());
-        let (session, _) = account
-            .create_group_session_pair(&room_id!("!test_room:example.org"), settings)
-            .await
-            .unwrap();
-
-        assert!(!session.expired());
-        let _ = session
-            .encrypt(AnyMessageEventContent::RoomMessage(MessageEventContent::text_plain(
-                "Test message",
-            )))
-            .await;
-        assert!(session.expired());
-
-        let settings = EncryptionSettings {
-            rotation_period: Duration::from_millis(100),
-            ..Default::default()
-        };
-
-        let (mut session, _) = account
-            .create_group_session_pair(&room_id!("!test_room:example.org"), settings)
-            .await
-            .unwrap();
-
-        assert!(!session.expired());
-        session.creation_time = Arc::new(Instant::now() - Duration::from_secs(60 * 60));
-        assert!(session.expired());
+            session_key,
+        })
     }
 }

crates/matrix-sdk-crypto/src/olm/group_sessions/outbound.rs

@@ -14,7 +14,7 @@
 
 use std::{
     cmp::max,
-    collections::BTreeMap,
+    collections::{BTreeMap, BTreeSet},
     fmt,
     sync::{
         atomic::{AtomicBool, AtomicU64, Ordering},
@@ -24,42 +24,39 @@ use std::{
 };
 
 use dashmap::DashMap;
-use matrix_sdk_common::{instant::Instant, locks::Mutex, uuid::Uuid};
-pub use olm_rs::{
-    account::IdentityKeys,
-    session::{OlmMessage, PreKeyMessage},
-    utility::OlmUtility,
-};
-use olm_rs::{
-    errors::OlmGroupSessionError, outbound_group_session::OlmOutboundGroupSession, PicklingMode,
-};
+use matrix_sdk_common::locks::Mutex;
 use ruma::{
     events::{
         room::{
-            encrypted::{EncryptedEventContent, EncryptedEventScheme, MegolmV1AesSha2ContentInit},
-            encryption::EncryptionEventContent,
+            encrypted::{
+                EncryptedEventScheme, MegolmV1AesSha2ContentInit, RoomEncryptedEventContent,
+            },
+            encryption::RoomEncryptionEventContent,
             history_visibility::HistoryVisibility,
         },
-        room_key::RoomKeyToDeviceEventContent,
-        AnyMessageEventContent, AnyToDeviceEventContent, EventContent,
+        room_key::ToDeviceRoomKeyEventContent,
+        AnyToDeviceEventContent,
     },
-    DeviceId, DeviceIdBox, DeviceKeyAlgorithm, EventEncryptionAlgorithm, RoomId, UserId,
+    DeviceId, EventEncryptionAlgorithm, OwnedDeviceId, OwnedTransactionId, OwnedUserId, RoomId,
+    SecondsSinceUnixEpoch, TransactionId, UserId,
 };
 use serde::{Deserialize, Serialize};
-use serde_json::json;
-use tracing::{debug, error, trace};
-
-use super::{
-    super::{deserialize_instant, serialize_instant},
-    GroupSessionKey,
+use serde_json::{json, Value};
+use tracing::{debug, error, info};
+use vodozemac::Curve25519PublicKey;
+pub use vodozemac::{
+    megolm::{GroupSession, GroupSessionPickle, MegolmMessage, SessionKey},
+    olm::IdentityKeys,
+    PickleError,
 };
+
 use crate::{Device, ToDeviceRequest};
 
 const ROTATION_PERIOD: Duration = Duration::from_millis(604800000);
 const ROTATION_MESSAGES: u64 = 100;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum ShareState {
+pub(crate) enum ShareState {
     NotShared,
     SharedButChangedSenderKey,
     Shared(u32),
@@ -92,9 +89,9 @@ impl Default for EncryptionSettings {
 }
 
 impl EncryptionSettings {
-    /// Create new encryption settings using an `EncryptionEventContent` and a
-    /// history visibility.
-    pub fn new(content: EncryptionEventContent, history_visibility: HistoryVisibility) -> Self {
+    /// Create new encryption settings using an `RoomEncryptionEventContent` and
+    /// a history visibility.
+    pub fn new(content: RoomEncryptionEventContent, history_visibility: HistoryVisibility) -> Self {
         let rotation_period: Duration =
             content.rotation_period_ms.map_or(ROTATION_PERIOD, |r| Duration::from_millis(r.into()));
         let rotation_period_msgs: u64 =
@@ -116,31 +113,31 @@ impl EncryptionSettings {
 /// messages.
 #[derive(Clone)]
 pub struct OutboundGroupSession {
-    inner: Arc<Mutex<OlmOutboundGroupSession>>,
+    inner: Arc<Mutex<GroupSession>>,
     device_id: Arc<DeviceId>,
     account_identity_keys: Arc<IdentityKeys>,
     session_id: Arc<str>,
     room_id: Arc<RoomId>,
-    pub(crate) creation_time: Arc<Instant>,
+    pub(crate) creation_time: SecondsSinceUnixEpoch,
     message_count: Arc<AtomicU64>,
     shared: Arc<AtomicBool>,
     invalidated: Arc<AtomicBool>,
     settings: Arc<EncryptionSettings>,
-    pub(crate) shared_with_set: Arc<DashMap<UserId, DashMap<DeviceIdBox, ShareInfo>>>,
-    to_share_with_set: Arc<DashMap<Uuid, (Arc<ToDeviceRequest>, ShareInfoSet)>>,
+    pub(crate) shared_with_set: Arc<DashMap<OwnedUserId, DashMap<OwnedDeviceId, ShareInfo>>>,
+    to_share_with_set: Arc<DashMap<OwnedTransactionId, (Arc<ToDeviceRequest>, ShareInfoSet)>>,
 }
 
 /// A a map of userid/device it to a `ShareInfo`.
 ///
 /// Holds the `ShareInfo` for all the user/device pairs that will receive the
 /// room key.
-pub type ShareInfoSet = BTreeMap<UserId, BTreeMap<DeviceIdBox, ShareInfo>>;
+pub type ShareInfoSet = BTreeMap<OwnedUserId, BTreeMap<OwnedDeviceId, ShareInfo>>;
 
 /// Struct holding info about the share state of a outbound group session.
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct ShareInfo {
     /// The sender key of the device that was used to encrypt the room key.
-    pub sender_key: String,
+    pub sender_key: Curve25519PublicKey,
     /// The message index that the device received.
     pub message_index: u32,
 }
@@ -167,16 +164,16 @@ impl OutboundGroupSession {
         room_id: &RoomId,
         settings: EncryptionSettings,
     ) -> Self {
-        let session = OlmOutboundGroupSession::new();
+        let session = GroupSession::new();
         let session_id = session.session_id();
 
         OutboundGroupSession {
             inner: Arc::new(Mutex::new(session)),
-            room_id: Arc::new(room_id.to_owned()),
+            room_id: room_id.into(),
             device_id,
             account_identity_keys: identity_keys,
             session_id: session_id.into(),
-            creation_time: Arc::new(Instant::now()),
+            creation_time: SecondsSinceUnixEpoch::now(),
             message_count: Arc::new(AtomicU64::new(0)),
             shared: Arc::new(AtomicBool::new(false)),
             invalidated: Arc::new(AtomicBool::new(false)),
@@ -188,7 +185,7 @@ impl OutboundGroupSession {
 
     pub(crate) fn add_request(
         &self,
-        request_id: Uuid,
+        request_id: OwnedTransactionId,
         request: Arc<ToDeviceRequest>,
         share_infos: ShareInfoSet,
     ) {
@@ -209,15 +206,19 @@ impl OutboundGroupSession {
     ///
     /// This removes the request from the queue and marks the set of
     /// users/devices that received the session.
-    pub fn mark_request_as_sent(&self, request_id: &Uuid) {
+    pub fn mark_request_as_sent(&self, request_id: &TransactionId) {
         if let Some((_, (_, r))) = self.to_share_with_set.remove(request_id) {
-            trace!(
-                request_id = request_id.to_string().as_str(),
+            let recipients: BTreeMap<&UserId, BTreeSet<&DeviceId>> =
+                r.iter().map(|(u, d)| (&**u, d.keys().map(|d| d.as_ref()).collect())).collect();
+
+            info!(
+                ?request_id,
+                ?recipients,
                 "Marking to-device request carrying a room key as sent"
             );
 
-            for (user_id, info) in r.into_iter() {
-                self.shared_with_set.entry(user_id).or_insert_with(DashMap::new).extend(info)
+            for (user_id, info) in r {
+                self.shared_with_set.entry(user_id).or_default().extend(info)
             }
 
             if self.to_share_with_set.is_empty() {
@@ -249,8 +250,8 @@ impl OutboundGroupSession {
     /// # Arguments
     ///
     /// * `plaintext` - The plaintext that should be encrypted.
-    pub(crate) async fn encrypt_helper(&self, plaintext: String) -> String {
-        let session = self.inner.lock().await;
+    pub(crate) async fn encrypt_helper(&self, plaintext: String) -> MegolmMessage {
+        let mut session = self.inner.lock().await;
         self.message_count.fetch_add(1, Ordering::SeqCst);
         session.encrypt(&plaintext)
     }
@@ -268,50 +269,66 @@ impl OutboundGroupSession {
     /// # Arguments
     ///
     /// * `content` - The plaintext content of the message that should be
-    /// encrypted.
+    /// encrypted in raw json [`Value`] form.
+    ///
+    /// * `event_type` - The plaintext type of the event, the outer type of the
+    /// event will become `m.room.encrypted`.
     ///
     /// # Panics
     ///
     /// Panics if the content can't be serialized.
-    pub async fn encrypt(&self, content: AnyMessageEventContent) -> EncryptedEventContent {
+    pub async fn encrypt(&self, content: Value, event_type: &str) -> RoomEncryptedEventContent {
         let json_content = json!({
             "content": content,
             "room_id": &*self.room_id,
-            "type": content.event_type(),
+            "type": event_type,
         });
 
         let plaintext = json_content.to_string();
-        let relation = content.relation();
+        let relation = serde_json::from_value(content).ok();
 
         let ciphertext = self.encrypt_helper(plaintext).await;
 
         let encrypted_content = MegolmV1AesSha2ContentInit {
-            ciphertext,
-            sender_key: self.account_identity_keys.curve25519().to_owned(),
+            ciphertext: ciphertext.to_base64(),
+            sender_key: self.account_identity_keys.curve25519.to_base64(),
             session_id: self.session_id().to_owned(),
-            device_id: (&*self.device_id).to_owned(),
+            device_id: (*self.device_id).to_owned(),
         }
         .into();
 
-        EncryptedEventContent::new(
+        RoomEncryptedEventContent::new(
             EncryptedEventScheme::MegolmV1AesSha2(encrypted_content),
             relation,
         )
     }
 
+    fn elapsed(&self) -> bool {
+        let creation_time = Duration::from_secs(self.creation_time.get().into());
+        let now = Duration::from_secs(SecondsSinceUnixEpoch::now().get().into());
+
+        // Since the encryption settings are provided by users and not
+        // checked someone could set a really low rotation period so
+        // clamp it to an hour.
+        now.checked_sub(creation_time)
+            .map(|elapsed| elapsed >= max(self.settings.rotation_period, Duration::from_secs(3600)))
+            .unwrap_or(true)
+    }
+
     /// Check if the session has expired and if it should be rotated.
     ///
     /// A session will expire after some time or if enough messages have been
     /// encrypted using it.
     pub fn expired(&self) -> bool {
         let count = self.message_count.load(Ordering::SeqCst);
+        // We clamp the rotation period for message counts to be between 1 and
+        // 10000. The Megolm session should be usable for at least 1 message,
+        // and at most 10000 messages. Realistically Megolm uses u32 for it's
+        // internal counter and one could use the Megolm session for up to
+        // u32::MAX messages, but we're staying on the safe side of things.
+        let rotation_period_msgs = self.settings.rotation_period_msgs.clamp(1, 10_000);
 
-        count >= self.settings.rotation_period_msgs
-            || self.creation_time.elapsed()
-                // Since the encryption settings are provided by users and not
-                // checked someone could set a really low rotation period so
-                // clamp it to an hour.
-                >= max(self.settings.rotation_period, Duration::from_secs(3600))
+        count >= rotation_period_msgs || self.elapsed()
     }
 
     /// Has the session been invalidated.
@@ -335,9 +352,9 @@ impl OutboundGroupSession {
     /// Get the session key of this session.
     ///
     /// A session key can be used to to create an `InboundGroupSession`.
-    pub async fn session_key(&self) -> GroupSessionKey {
+    pub async fn session_key(&self) -> SessionKey {
         let session = self.inner.lock().await;
-        GroupSessionKey(session.session_key())
+        session.session_key()
     }
 
     /// Get the room id of the room this session belongs to.
@@ -356,17 +373,17 @@ impl OutboundGroupSession {
     /// message index that will be used for the next encrypted message.
     pub async fn message_index(&self) -> u32 {
         let session = self.inner.lock().await;
-        session.session_message_index()
+        session.message_index()
     }
 
     pub(crate) async fn as_content(&self) -> AnyToDeviceEventContent {
         let session_key = self.session_key().await;
 
-        AnyToDeviceEventContent::RoomKey(RoomKeyToDeviceEventContent::new(
+        AnyToDeviceEventContent::RoomKey(ToDeviceRoomKeyEventContent::new(
             EventEncryptionAlgorithm::MegolmV1AesSha2,
             self.room_id().to_owned(),
             self.session_id().to_owned(),
-            session_key.0.clone(),
+            session_key.to_base64(),
         ))
     }
 
@@ -375,7 +392,7 @@ impl OutboundGroupSession {
         // Check if we shared the session.
         let shared_state = self.shared_with_set.get(device.user_id()).and_then(|d| {
             d.get(device.device_id()).map(|s| {
-                if Some(&s.sender_key) == device.get_key(DeviceKeyAlgorithm::Curve25519) {
+                if Some(s.sender_key) == device.curve25519_key() {
                     ShareState::Shared(s.message_index)
                 } else {
                     ShareState::SharedButChangedSenderKey
@@ -396,8 +413,7 @@ impl OutboundGroupSession {
 
                 share_info.get(device.user_id()).and_then(|d| {
                     d.get(device.device_id()).map(|info| {
-                        if Some(&info.sender_key) == device.get_key(DeviceKeyAlgorithm::Curve25519)
-                        {
+                        if Some(info.sender_key) == device.curve25519_key() {
                             ShareState::Shared(info.message_index)
                         } else {
                             ShareState::SharedButChangedSenderKey
@@ -410,12 +426,34 @@ impl OutboundGroupSession {
         }
     }
 
-    /// Mark that the session was shared with the given user/device pair.
+    /// Mark the session as shared with the given user/device pair, starting
+    /// from some message index.
     #[cfg(test)]
-    pub fn mark_shared_with(&self, user_id: &UserId, device_id: &DeviceId, sender_key: &str) {
-        self.shared_with_set.entry(user_id.to_owned()).or_insert_with(DashMap::new).insert(
+    pub fn mark_shared_with_from_index(
+        &self,
+        user_id: &UserId,
+        device_id: &DeviceId,
+        sender_key: Curve25519PublicKey,
+        index: u32,
+    ) {
+        self.shared_with_set
+            .entry(user_id.to_owned())
+            .or_default()
+            .insert(device_id.to_owned(), ShareInfo { sender_key, message_index: index });
+    }
+
+    /// Mark the session as shared with the given user/device pair, starting
+    /// from the current index.
+    #[cfg(test)]
+    pub async fn mark_shared_with(
+        &self,
+        user_id: &UserId,
+        device_id: &DeviceId,
+        sender_key: Curve25519PublicKey,
+    ) {
+        self.shared_with_set.entry(user_id.to_owned()).or_default().insert(
             device_id.to_owned(),
-            ShareInfo { sender_key: sender_key.to_owned(), message_index: 0 },
+            ShareInfo { sender_key, message_index: self.message_index().await },
         );
     }
 
@@ -426,8 +464,8 @@ impl OutboundGroupSession {
     }
 
     /// Get the list of request ids this session is waiting for to be sent out.
-    pub(crate) fn pending_request_ids(&self) -> Vec<Uuid> {
-        self.to_share_with_set.iter().map(|e| *e.key()).collect()
+    pub(crate) fn pending_request_ids(&self) -> Vec<OwnedTransactionId> {
+        self.to_share_with_set.iter().map(|e| e.key().clone()).collect()
     }
 
     /// Restore a Session from a previously pickled string.
@@ -451,9 +489,8 @@ impl OutboundGroupSession {
         device_id: Arc<DeviceId>,
         identity_keys: Arc<IdentityKeys>,
         pickle: PickledOutboundGroupSession,
-        pickling_mode: PicklingMode,
-    ) -> Result<Self, OlmGroupSessionError> {
-        let inner = OlmOutboundGroupSession::unpickle(pickle.pickle.0, pickling_mode)?;
+    ) -> Result<Self, PickleError> {
+        let inner: GroupSession = pickle.pickle.into();
         let session_id = inner.session_id();
 
         Ok(Self {
@@ -462,7 +499,7 @@ impl OutboundGroupSession {
             account_identity_keys: identity_keys,
             session_id: session_id.into(),
             room_id: pickle.room_id,
-            creation_time: pickle.creation_time.into(),
+            creation_time: pickle.creation_time,
             message_count: AtomicU64::from(pickle.message_count).into(),
             shared: AtomicBool::from(pickle.shared).into(),
             invalidated: AtomicBool::from(pickle.invalidated).into(),
@@ -486,15 +523,14 @@ impl OutboundGroupSession {
     /// * `pickle_mode` - The mode that should be used to pickle the group
     ///   session,
     /// either an unencrypted mode or an encrypted using passphrase.
-    pub async fn pickle(&self, pickling_mode: PicklingMode) -> PickledOutboundGroupSession {
-        let pickle: OutboundGroupSessionPickle =
-            self.inner.lock().await.pickle(pickling_mode).into();
+    pub async fn pickle(&self) -> PickledOutboundGroupSession {
+        let pickle = self.inner.lock().await.pickle();
 
         PickledOutboundGroupSession {
             pickle,
             room_id: self.room_id.clone(),
             settings: self.settings.clone(),
-            creation_time: *self.creation_time,
+            creation_time: self.creation_time,
             message_count: self.message_count.load(Ordering::SeqCst),
             shared: self.shared(),
             invalidated: self.invalidated(),
@@ -511,7 +547,7 @@ impl OutboundGroupSession {
             requests: self
                 .to_share_with_set
                 .iter()
-                .map(|r| (*r.key(), r.value().clone()))
+                .map(|r| (r.key().clone(), r.value().clone()))
                 .collect(),
         }
     }
@@ -542,17 +578,17 @@ impl std::fmt::Debug for OutboundGroupSession {
 ///
 /// Holds all the information that needs to be stored in a database to restore
 /// an InboundGroupSession.
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Deserialize, Serialize)]
+#[allow(missing_debug_implementations)]
 pub struct PickledOutboundGroupSession {
     /// The pickle string holding the OutboundGroupSession.
-    pub pickle: OutboundGroupSessionPickle,
+    pub pickle: GroupSessionPickle,
     /// The settings this session adheres to.
     pub settings: Arc<EncryptionSettings>,
     /// The room id this session is used for.
     pub room_id: Arc<RoomId>,
     /// The timestamp when this session was created.
-    #[serde(deserialize_with = "deserialize_instant", serialize_with = "serialize_instant")]
-    pub creation_time: Instant,
+    pub creation_time: SecondsSinceUnixEpoch,
     /// The number of messages this session has already encrypted.
     pub message_count: u64,
     /// Is the session shared.
@@ -560,25 +596,33 @@ pub struct PickledOutboundGroupSession {
     /// Has the session been invalidated.
     pub invalidated: bool,
     /// The set of users the session has been already shared with.
-    pub shared_with_set: BTreeMap<UserId, BTreeMap<DeviceIdBox, ShareInfo>>,
+    pub shared_with_set: BTreeMap<OwnedUserId, BTreeMap<OwnedDeviceId, ShareInfo>>,
     /// Requests that need to be sent out to share the session.
-    pub requests: BTreeMap<Uuid, (Arc<ToDeviceRequest>, ShareInfoSet)>,
+    pub requests: BTreeMap<OwnedTransactionId, (Arc<ToDeviceRequest>, ShareInfoSet)>,
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use std::time::Duration;
 
+    use atomic::Ordering;
+    use matrix_sdk_test::async_test;
     use ruma::{
-        events::room::{encryption::EncryptionEventContent, history_visibility::HistoryVisibility},
-        uint, EventEncryptionAlgorithm,
+        device_id,
+        events::room::{
+            encryption::RoomEncryptionEventContent, history_visibility::HistoryVisibility,
+            message::RoomMessageEventContent,
+        },
+        room_id, uint, user_id, EventEncryptionAlgorithm,
     };
 
     use super::{EncryptionSettings, ROTATION_MESSAGES, ROTATION_PERIOD};
+    use crate::{MegolmError, ReadOnlyAccount};
 
     #[test]
     fn encryption_settings_conversion() {
-        let mut content = EncryptionEventContent::new(EventEncryptionAlgorithm::MegolmV1AesSha2);
+        let mut content =
+            RoomEncryptionEventContent::new(EventEncryptionAlgorithm::MegolmV1AesSha2);
         let settings = EncryptionSettings::new(content.clone(), HistoryVisibility::Joined);
 
         assert_eq!(settings.rotation_period, ROTATION_PERIOD);
@@ -592,4 +636,77 @@ mod test {
         assert_eq!(settings.rotation_period, Duration::from_millis(3600));
         assert_eq!(settings.rotation_period_msgs, 500);
     }
+
+    #[async_test]
+    #[cfg(any(target_os = "linux", target_arch = "wasm32"))]
+    async fn expiration() -> Result<(), MegolmError> {
+        use ruma::SecondsSinceUnixEpoch;
+
+        let settings = EncryptionSettings { rotation_period_msgs: 1, ..Default::default() };
+
+        let account = ReadOnlyAccount::new(user_id!("@alice:example.org"), device_id!("DEVICEID"));
+        let (session, _) = account
+            .create_group_session_pair(room_id!("!test_room:example.org"), settings)
+            .await
+            .unwrap();
+
+        assert!(!session.expired());
+        let _ = session
+            .encrypt(
+                serde_json::to_value(RoomMessageEventContent::text_plain("Test message"))?,
+                "m.room.message",
+            )
+            .await;
+        assert!(session.expired());
+
+        let settings = EncryptionSettings {
+            rotation_period: Duration::from_millis(100),
+            ..Default::default()
+        };
+
+        let (mut session, _) = account
+            .create_group_session_pair(room_id!("!test_room:example.org"), settings)
+            .await
+            .unwrap();
+
+        assert!(!session.expired());
+
+        let now = SecondsSinceUnixEpoch::now();
+        session.creation_time = SecondsSinceUnixEpoch(now.get() - uint!(3600));
+        assert!(session.expired());
+
+        let settings = EncryptionSettings { rotation_period_msgs: 0, ..Default::default() };
+
+        let (session, _) = account
+            .create_group_session_pair(room_id!("!test_room:example.org"), settings)
+            .await
+            .unwrap();
+
+        assert!(!session.expired());
+
+        let _ = session
+            .encrypt(
+                serde_json::to_value(RoomMessageEventContent::text_plain("Test message"))?,
+                "m.room.message",
+            )
+            .await;
+        assert!(session.expired());
+
+        let settings = EncryptionSettings { rotation_period_msgs: 100_000, ..Default::default() };
+
+        let (session, _) = account
+            .create_group_session_pair(room_id!("!test_room:example.org"), settings)
+            .await
+            .unwrap();
+
+        assert!(!session.expired());
+        session.message_count.store(1000, Ordering::SeqCst);
+        assert!(!session.expired());
+        session.message_count.store(9999, Ordering::SeqCst);
+        assert!(!session.expired());
+        session.message_count.store(10_000, Ordering::SeqCst);
+        assert!(session.expired());
+
+        Ok(())
+    }
 }

crates/matrix-sdk-crypto/src/olm/mod.rs

@@ -0,0 +1,261 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The crypto specific Olm objects.
+//!
+//! Note: You'll only be interested in these if you are implementing a custom
+//! `CryptoStore`.
+
+mod account;
+mod group_sessions;
+mod session;
+mod signing;
+mod utility;
+
+pub(crate) use account::{Account, OlmDecryptionInfo, SessionType};
+pub use account::{OlmMessageHash, PickledAccount, ReadOnlyAccount};
+pub(crate) use group_sessions::ShareState;
+pub use group_sessions::{
+    EncryptionSettings, ExportedRoomKey, InboundGroupSession, OutboundGroupSession,
+    PickledInboundGroupSession, PickledOutboundGroupSession, SessionKey, ShareInfo,
+};
+pub use session::{PickledSession, Session};
+pub use signing::{CrossSigningStatus, PickledCrossSigningIdentity, PrivateCrossSigningIdentity};
+pub(crate) use utility::VerifyJson;
+pub use vodozemac::olm::IdentityKeys;
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use matches::assert_matches;
+    use matrix_sdk_test::async_test;
+    use ruma::{
+        device_id, event_id,
+        events::{
+            forwarded_room_key::ToDeviceForwardedRoomKeyEventContent,
+            room::message::{Relation, Replacement, RoomMessageEventContent},
+            AnyMessageLikeEvent, AnyRoomEvent, AnySyncMessageLikeEvent, AnySyncRoomEvent,
+            MessageLikeEvent, SyncMessageLikeEvent,
+        },
+        room_id, user_id, DeviceId, UserId,
+    };
+    use serde_json::json;
+    use vodozemac::olm::OlmMessage;
+
+    use crate::olm::{ExportedRoomKey, InboundGroupSession, ReadOnlyAccount, Session};
+
+    fn alice_id() -> &'static UserId {
+        user_id!("@alice:example.org")
+    }
+
+    fn alice_device_id() -> &'static DeviceId {
+        device_id!("ALICEDEVICE")
+    }
+
+    fn bob_id() -> &'static UserId {
+        user_id!("@bob:example.org")
+    }
+
+    fn bob_device_id() -> &'static DeviceId {
+        device_id!("BOBDEVICE")
+    }
+
+    pub(crate) async fn get_account_and_session() -> (ReadOnlyAccount, Session) {
+        let alice = ReadOnlyAccount::new(alice_id(), alice_device_id());
+        let bob = ReadOnlyAccount::new(bob_id(), bob_device_id());
+
+        bob.generate_one_time_keys_helper(1).await;
+        let one_time_key = *bob.one_time_keys().await.values().next().unwrap();
+        let sender_key = bob.identity_keys().curve25519;
+        let session = alice.create_outbound_session_helper(sender_key, one_time_key, false).await;
+
+        (alice, session)
+    }
+
+    #[test]
+    fn account_creation() {
+        let account = ReadOnlyAccount::new(alice_id(), alice_device_id());
+
+        assert!(!account.shared());
+
+        account.mark_as_shared();
+        assert!(account.shared());
+    }
+
+    #[async_test]
+    async fn one_time_keys_creation() {
+        let account = ReadOnlyAccount::new(alice_id(), alice_device_id());
+        let one_time_keys = account.one_time_keys().await;
+
+        assert!(one_time_keys.is_empty());
+        assert_ne!(account.max_one_time_keys().await, 0);
+
+        account.generate_one_time_keys_helper(10).await;
+        let one_time_keys = account.one_time_keys().await;
+
+        assert_ne!(one_time_keys.values().len(), 0);
+        assert_ne!(one_time_keys.keys().len(), 0);
+        assert_ne!(one_time_keys.iter().len(), 0);
+
+        account.mark_keys_as_published().await;
+        let one_time_keys = account.one_time_keys().await;
+        assert!(one_time_keys.is_empty());
+    }
+
+    #[async_test]
+    async fn session_creation() {
+        let alice = ReadOnlyAccount::new(alice_id(), alice_device_id());
+        let bob = ReadOnlyAccount::new(bob_id(), bob_device_id());
+        let alice_keys = alice.identity_keys();
+        alice.generate_one_time_keys_helper(1).await;
+        let one_time_keys = alice.one_time_keys().await;
+        alice.mark_keys_as_published().await;
+
+        let one_time_key = *one_time_keys.values().next().unwrap();
+
+        let mut bob_session =
+            bob.create_outbound_session_helper(alice_keys.curve25519, one_time_key, false).await;
+
+        let plaintext = "Hello world";
+
+        let message = bob_session.encrypt_helper(plaintext).await;
+
+        let prekey_message = match message.clone() {
+            OlmMessage::PreKey(m) => m,
+            OlmMessage::Normal(_) => panic!("Incorrect message type"),
+        };
+
+        let bob_keys = bob.identity_keys();
+        let result = alice
+            .create_inbound_session(&bob_keys.curve25519.to_base64(), &prekey_message)
+            .await
+            .unwrap();
+
+        assert_eq!(bob_session.session_id(), result.session.session_id());
+
+        assert_eq!(plaintext, result.plaintext);
+    }
+
+    #[async_test]
+    async fn group_session_creation() {
+        let alice = ReadOnlyAccount::new(alice_id(), alice_device_id());
+        let room_id = room_id!("!test:localhost");
+
+        let (outbound, _) = alice.create_group_session_pair_with_defaults(room_id).await;
+
+        assert_eq!(0, outbound.message_index().await);
+        assert!(!outbound.shared());
+        outbound.mark_as_shared();
+        assert!(outbound.shared());
+
+        let inbound = InboundGroupSession::new(
+            "test_key",
+            "test_key",
+            room_id,
+            outbound.session_key().await,
+            None,
+        );
+
+        assert_eq!(0, inbound.first_known_index());
+
+        assert_eq!(outbound.session_id(), inbound.session_id());
+
+        let plaintext = "This is a secret to everybody".to_owned();
+        let ciphertext = outbound.encrypt_helper(plaintext.clone()).await;
+
+        assert_eq!(plaintext, inbound.decrypt_helper(&ciphertext).await.unwrap().plaintext);
+    }
+
+    #[async_test]
+    async fn edit_decryption() {
+        let alice = ReadOnlyAccount::new(alice_id(), alice_device_id());
+        let room_id = room_id!("!test:localhost");
+        let event_id = event_id!("$1234adfad:asdf");
+
+        let (outbound, _) = alice.create_group_session_pair_with_defaults(room_id).await;
+
+        assert_eq!(0, outbound.message_index().await);
+        assert!(!outbound.shared());
+        outbound.mark_as_shared();
+        assert!(outbound.shared());
+
+        let mut content = RoomMessageEventContent::text_plain("Hello");
+        content.relates_to = Some(Relation::Replacement(Replacement::new(
+            event_id.to_owned(),
+            RoomMessageEventContent::text_plain("Hello edit").into(),
+        )));
+
+        let inbound = InboundGroupSession::new(
+            "test_key",
+            "test_key",
+            room_id,
+            outbound.session_key().await,
+            None,
+        );
+
+        assert_eq!(0, inbound.first_known_index());
+
+        assert_eq!(outbound.session_id(), inbound.session_id());
+
+        let encrypted_content =
+            outbound.encrypt(serde_json::to_value(content).unwrap(), "m.room.message").await;
+
+        let event = json!({
+            "sender": alice.user_id(),
+            "event_id": event_id,
+            "origin_server_ts": 0u64,
+            "room_id": room_id,
+            "type": "m.room.encrypted",
+            "content": encrypted_content,
+        })
+        .to_string();
+
+        let event: AnySyncRoomEvent = serde_json::from_str(&event).unwrap();
+
+        let event = if let AnySyncRoomEvent::MessageLike(AnySyncMessageLikeEvent::RoomEncrypted(
+            SyncMessageLikeEvent::Original(event),
+        )) = event
+        {
+            event
+        } else {
+            panic!("Invalid event type")
+        };
+
+        let decrypted = inbound.decrypt(&event).await.unwrap().0;
+
+        if let AnyRoomEvent::MessageLike(AnyMessageLikeEvent::RoomMessage(
+            MessageLikeEvent::Original(e),
+        )) = decrypted.deserialize().unwrap()
+        {
+            assert_matches!(e.content.relates_to, Some(Relation::Replacement(_)));
+        } else {
+            panic!("Invalid event type")
+        }
+    }
+
+    #[async_test]
+    async fn group_session_export() {
+        let alice = ReadOnlyAccount::new(alice_id(), alice_device_id());
+        let room_id = room_id!("!test:localhost");
+
+        let (_, inbound) = alice.create_group_session_pair_with_defaults(room_id).await;
+
+        let export = inbound.export().await;
+        let export: ToDeviceForwardedRoomKeyEventContent = export.try_into().unwrap();
+        let export = ExportedRoomKey::try_from(export).unwrap();
+
+        let imported = InboundGroupSession::from_export(export);
+
+        assert_eq!(inbound.session_id(), imported.session_id());
+    }
+}

crates/matrix-sdk-crypto/src/olm/session.rs

@@ -14,28 +14,27 @@
 
 use std::{collections::BTreeMap, fmt, sync::Arc};
 
-use matrix_sdk_common::{instant::Instant, locks::Mutex};
-use olm_rs::{errors::OlmSessionError, session::OlmSession, PicklingMode};
-pub use olm_rs::{
-    session::{OlmMessage, PreKeyMessage},
-    utility::OlmUtility,
-};
+use matrix_sdk_common::locks::Mutex;
 use ruma::{
     events::{
         room::encrypted::{
-            CiphertextInfo, EncryptedEventScheme, EncryptedToDeviceEventContent,
-            OlmV1Curve25519AesSha2Content,
+            CiphertextInfo, EncryptedEventScheme, OlmV1Curve25519AesSha2Content,
+            ToDeviceRoomEncryptedEventContent,
         },
         AnyToDeviceEventContent, EventContent,
     },
-    DeviceId, DeviceKeyAlgorithm, UserId,
+    DeviceId, SecondsSinceUnixEpoch, UserId,
 };
 use serde::{Deserialize, Serialize};
 use serde_json::json;
+use vodozemac::{
+    olm::{DecryptionError, OlmMessage, Session as InnerSession, SessionPickle},
+    Curve25519PublicKey,
+};
 
-use super::{deserialize_instant, serialize_instant, IdentityKeys};
+use super::IdentityKeys;
 use crate::{
-    error::{EventError, OlmResult, SessionUnpicklingError},
+    error::{EventError, OlmResult},
     ReadOnlyDevice,
 };
 
@@ -43,14 +42,24 @@ use crate::{
 /// `Account`s
 #[derive(Clone)]
 pub struct Session {
-    pub(crate) user_id: Arc<UserId>,
-    pub(crate) device_id: Arc<DeviceId>,
-    pub(crate) our_identity_keys: Arc<IdentityKeys>,
-    pub(crate) inner: Arc<Mutex<OlmSession>>,
-    pub(crate) session_id: Arc<str>,
-    pub(crate) sender_key: Arc<str>,
-    pub(crate) creation_time: Arc<Instant>,
-    pub(crate) last_use_time: Arc<Instant>,
+    /// The `UserId` associated with this session
+    pub user_id: Arc<UserId>,
+    /// The specific `DeviceId` associated with this session
+    pub device_id: Arc<DeviceId>,
+    /// The `IdentityKeys` associated with this session
+    pub our_identity_keys: Arc<IdentityKeys>,
+    /// The OlmSession
+    pub inner: Arc<Mutex<InnerSession>>,
+    /// Our sessionId
+    pub session_id: Arc<str>,
+    /// The Key of the sender
+    pub sender_key: Curve25519PublicKey,
+    /// Has this been created using the fallback key
+    pub created_using_fallback_key: bool,
+    /// When the session was created
+    pub creation_time: SecondsSinceUnixEpoch,
+    /// When the session was last used
+    pub last_use_time: SecondsSinceUnixEpoch,
 }
 
 #[cfg(not(tarpaulin_include))]
@@ -66,21 +75,21 @@ impl fmt::Debug for Session {
 impl Session {
     /// Decrypt the given Olm message.
     ///
-    /// Returns the decrypted plaintext or an `OlmSessionError` if decryption
+    /// Returns the decrypted plaintext or an `DecrypitonError` if decryption
     /// failed.
     ///
     /// # Arguments
     ///
     /// * `message` - The Olm message that should be decrypted.
-    pub async fn decrypt(&mut self, message: OlmMessage) -> Result<String, OlmSessionError> {
+    pub async fn decrypt(&mut self, message: &OlmMessage) -> Result<String, DecryptionError> {
         let plaintext = self.inner.lock().await.decrypt(message)?;
-        self.last_use_time = Arc::new(Instant::now());
+        self.last_use_time = SecondsSinceUnixEpoch::now();
         Ok(plaintext)
     }
 
     /// Get the sender key that was used to establish this Session.
-    pub fn sender_key(&self) -> &str {
-        &self.sender_key
+    pub fn sender_key(&self) -> Curve25519PublicKey {
+        self.sender_key
     }
 
     /// Encrypt the given plaintext as a OlmMessage.
@@ -92,7 +101,7 @@ impl Session {
     /// * `plaintext` - The plaintext that should be encrypted.
     pub(crate) async fn encrypt_helper(&mut self, plaintext: &str) -> OlmMessage {
         let message = self.inner.lock().await.encrypt(plaintext);
-        self.last_use_time = Arc::new(Instant::now());
+        self.last_use_time = SecondsSinceUnixEpoch::now();
         message
     }
 
@@ -110,10 +119,9 @@ impl Session {
         &mut self,
         recipient_device: &ReadOnlyDevice,
         content: AnyToDeviceEventContent,
-    ) -> OlmResult<EncryptedToDeviceEventContent> {
-        let recipient_signing_key = recipient_device
-            .get_key(DeviceKeyAlgorithm::Ed25519)
-            .ok_or(EventError::MissingSigningKey)?;
+    ) -> OlmResult<ToDeviceRoomEncryptedEventContent> {
+        let recipient_signing_key =
+            recipient_device.ed25519_key().ok_or(EventError::MissingSigningKey)?;
 
         let event_type = content.event_type();
 
@@ -121,51 +129,32 @@ impl Session {
             "sender": self.user_id.as_str(),
             "sender_device": self.device_id.as_ref(),
             "keys": {
-                "ed25519": self.our_identity_keys.ed25519(),
+                "ed25519": self.our_identity_keys.ed25519.to_base64(),
             },
             "recipient": recipient_device.user_id(),
             "recipient_keys": {
-                "ed25519": recipient_signing_key,
+                "ed25519": recipient_signing_key.to_base64(),
             },
             "type": event_type,
             "content": content,
         });
 
         let plaintext = serde_json::to_string(&payload)?;
-        let ciphertext = self.encrypt_helper(&plaintext).await.to_tuple();
+        let ciphertext = self.encrypt_helper(&plaintext).await.to_parts();
 
         let message_type = ciphertext.0;
         let ciphertext = CiphertextInfo::new(ciphertext.1, (message_type as u32).into());
 
         let mut content = BTreeMap::new();
-        content.insert((&*self.sender_key).to_owned(), ciphertext);
+        content.insert(self.sender_key.to_base64(), ciphertext);
 
         Ok(EncryptedEventScheme::OlmV1Curve25519AesSha2(OlmV1Curve25519AesSha2Content::new(
             content,
-            self.our_identity_keys.curve25519().to_string(),
+            self.our_identity_keys.curve25519.to_base64(),
         ))
         .into())
     }
 
-    /// Check if a pre-key Olm message was encrypted for this session.
-    ///
-    /// Returns true if it matches, false if not and a OlmSessionError if there
-    /// was an error checking if it matches.
-    ///
-    /// # Arguments
-    ///
-    /// * `their_identity_key` - The identity/curve25519 key of the account
-    /// that encrypted this Olm message.
-    ///
-    /// * `message` - The pre-key Olm message that should be checked.
-    pub async fn matches(
-        &self,
-        their_identity_key: &str,
-        message: PreKeyMessage,
-    ) -> Result<bool, OlmSessionError> {
-        self.inner.lock().await.matches_inbound_session_from(their_identity_key, message)
-    }
-
     /// Returns the unique identifier for this session.
     pub fn session_id(&self) -> &str {
         &self.session_id
@@ -177,14 +166,15 @@ impl Session {
     ///
     /// * `pickle_mode` - The mode that was used to pickle the session, either
     /// an unencrypted mode or an encrypted using passphrase.
-    pub async fn pickle(&self, pickle_mode: PicklingMode) -> PickledSession {
-        let pickle = self.inner.lock().await.pickle(pickle_mode);
+    pub async fn pickle(&self) -> PickledSession {
+        let pickle = self.inner.lock().await.pickle();
 
         PickledSession {
-            pickle: SessionPickle::from(pickle),
-            sender_key: self.sender_key.to_string(),
-            creation_time: *self.creation_time,
-            last_use_time: *self.last_use_time,
+            pickle,
+            sender_key: self.sender_key,
+            created_using_fallback_key: self.created_using_fallback_key,
+            creation_time: self.creation_time,
+            last_use_time: self.last_use_time,
         }
     }
 
@@ -210,21 +200,21 @@ impl Session {
         device_id: Arc<DeviceId>,
         our_identity_keys: Arc<IdentityKeys>,
         pickle: PickledSession,
-        pickle_mode: PicklingMode,
-    ) -> Result<Self, SessionUnpicklingError> {
-        let session = OlmSession::unpickle(pickle.pickle.0, pickle_mode)?;
+    ) -> Self {
+        let session: vodozemac::olm::Session = pickle.pickle.into();
         let session_id = session.session_id();
 
-        Ok(Session {
+        Session {
             user_id,
             device_id,
             our_identity_keys,
             inner: Arc::new(Mutex::new(session)),
             session_id: session_id.into(),
-            sender_key: pickle.sender_key.into(),
-            creation_time: Arc::new(pickle.creation_time),
-            last_use_time: Arc::new(pickle.last_use_time),
-        })
+            created_using_fallback_key: pickle.created_using_fallback_key,
+            sender_key: pickle.sender_key,
+            creation_time: pickle.creation_time,
+            last_use_time: pickle.last_use_time,
+        }
     }
 }
 
@@ -238,34 +228,18 @@ impl PartialEq for Session {
 ///
 /// Holds all the information that needs to be stored in a database to restore
 /// a Session.
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Serialize, Deserialize)]
+#[allow(missing_debug_implementations)]
 pub struct PickledSession {
     /// The pickle string holding the Olm Session.
     pub pickle: SessionPickle,
     /// The curve25519 key of the other user that we share this session with.
-    pub sender_key: String,
-    /// The relative time elapsed since the session was created.
-    #[serde(deserialize_with = "deserialize_instant", serialize_with = "serialize_instant")]
-    pub creation_time: Instant,
-    /// The relative time elapsed since the session was last used.
-    #[serde(deserialize_with = "deserialize_instant", serialize_with = "serialize_instant")]
-    pub last_use_time: Instant,
-}
-
-/// The typed representation of a base64 encoded string of the Olm Session
-/// pickle.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct SessionPickle(String);
-
-impl From<String> for SessionPickle {
-    fn from(pickle_string: String) -> Self {
-        SessionPickle(pickle_string)
-    }
-}
-
-impl SessionPickle {
-    /// Get the string representation of the pickle.
-    pub fn as_str(&self) -> &str {
-        &self.0
-    }
+    pub sender_key: Curve25519PublicKey,
+    /// Was the session created using a fallback key.
+    #[serde(default)]
+    pub created_using_fallback_key: bool,
+    /// The Unix timestamp when the session was created.
+    pub creation_time: SecondsSinceUnixEpoch,
+    /// The Unix timestamp when the session was last used.
+    pub last_use_time: SecondsSinceUnixEpoch,
 }

crates/matrix-sdk-crypto/src/olm/signing/mod.rs

@@ -14,31 +14,29 @@
 
 mod pk_signing;
 
-use std::{
-    collections::BTreeMap,
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc,
-    },
+use std::sync::{
+    atomic::{AtomicBool, Ordering},
+    Arc,
 };
 
 use matrix_sdk_common::locks::Mutex;
 use pk_signing::{MasterSigning, PickledSignings, SelfSigning, Signing, SigningError, UserSigning};
 use ruma::{
-    api::client::r0::keys::upload_signatures::Request as SignatureUploadRequest,
-    encryption::{DeviceKeys, KeyUsage},
+    api::client::keys::upload_signatures::v3::{Request as SignatureUploadRequest, SignedKeys},
+    encryption::KeyUsage,
     events::secret::request::SecretName,
-    UserId,
+    OwnedUserId, UserId,
 };
 use serde::{Deserialize, Serialize};
 use serde_json::Error as JsonError;
+use vodozemac::Ed25519Signature;
 
 use crate::{
     error::SignatureError,
     identities::{MasterPubkey, SelfSigningPubkey, UserSigningPubkey},
     requests::UploadSigningKeysRequest,
     store::SecretImportError,
-    utilities::decode,
+    types::DeviceKeys,
     OwnUserIdentity, ReadOnlyAccount, ReadOnlyDevice, ReadOnlyOwnUserIdentity,
     ReadOnlyUserIdentity,
 };
@@ -84,14 +82,15 @@ impl ClearResult {
 /// The pickled version of a `PrivateCrossSigningIdentity`.
 ///
 /// Can be used to store the identity.
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Serialize, Deserialize)]
+#[allow(missing_debug_implementations)]
 pub struct PickledCrossSigningIdentity {
     /// The user id of the identity owner.
-    pub user_id: UserId,
+    pub user_id: OwnedUserId,
     /// Have the public keys of the identity been shared.
     pub shared: bool,
-    /// The encrypted pickle of the identity.
-    pub pickle: String,
+    /// The pickled signing keys
+    pub keys: PickledSignings,
 }
 
 /// Struct representing the state of our private cross signing keys, it shows
@@ -218,40 +217,36 @@ impl PrivateCrossSigningIdentity {
         user_signing_key: Option<&str>,
     ) -> Result<(), SecretImportError> {
         let master = if let Some(master_key) = master_key {
-            let seed = decode(master_key)?;
-
-            let master = MasterSigning::from_seed(self.user_id().clone(), seed);
+            let master = MasterSigning::from_base64(self.user_id().to_owned(), master_key)?;
 
             if public_identity.master_key() == &master.public_key {
                 Ok(Some(master))
             } else {
-                Err(SecretImportError::MissmatchedPublicKeys)
+                Err(SecretImportError::MismatchedPublicKeys)
             }
         } else {
             Ok(None)
         }?;
 
         let user_signing = if let Some(user_signing_key) = user_signing_key {
-            let seed = decode(user_signing_key)?;
-            let subkey = UserSigning::from_seed(self.user_id().clone(), seed);
+            let subkey = UserSigning::from_base64(self.user_id().to_owned(), user_signing_key)?;
 
             if public_identity.user_signing_key() == &subkey.public_key {
                 Ok(Some(subkey))
             } else {
-                Err(SecretImportError::MissmatchedPublicKeys)
+                Err(SecretImportError::MismatchedPublicKeys)
             }
         } else {
             Ok(None)
         }?;
 
         let self_signing = if let Some(self_signing_key) = self_signing_key {
-            let seed = decode(self_signing_key)?;
-            let subkey = SelfSigning::from_seed(self.user_id().clone(), seed);
+            let subkey = SelfSigning::from_base64(self.user_id().to_owned(), self_signing_key)?;
 
             if public_identity.self_signing_key() == &subkey.public_key {
                 Ok(Some(subkey))
             } else {
-                Err(SecretImportError::MissmatchedPublicKeys)
+                Err(SecretImportError::MismatchedPublicKeys)
             }
         } else {
             Ok(None)
@@ -272,6 +267,37 @@ impl PrivateCrossSigningIdentity {
         Ok(())
     }
 
+    /// Import the private parts of the cross signing keys into this identity.
+    ///
+    /// The private parts should be unexpanded Ed25519 keys encoded as a base64
+    /// string.
+    ///
+    /// *Note*: This method won't check if the public keys match the public
+    /// keys present on the server.
+    pub async fn import_secrets_unchecked(
+        &self,
+        master_key: Option<&str>,
+        self_signing_key: Option<&str>,
+        user_signing_key: Option<&str>,
+    ) -> Result<(), SecretImportError> {
+        if let Some(master_key) = master_key {
+            let master = MasterSigning::from_base64(self.user_id().to_owned(), master_key)?;
+            *self.master_key.lock().await = Some(master);
+        }
+
+        if let Some(user_signing_key) = user_signing_key {
+            let subkey = UserSigning::from_base64(self.user_id().to_owned(), user_signing_key)?;
+            *self.user_signing_key.lock().await = Some(subkey);
+        };
+
+        if let Some(self_signing_key) = self_signing_key {
+            let subkey = SelfSigning::from_base64(self.user_id().to_owned(), self_signing_key)?;
+            *self.self_signing_key.lock().await = Some(subkey);
+        }
+
+        Ok(())
+    }
+
     /// Remove our private cross signing key if the public keys differ from
     /// what's found in the `ReadOnlyOwnUserIdentity`.
     pub(crate) async fn clear_if_differs(
@@ -344,9 +370,9 @@ impl PrivateCrossSigningIdentity {
     }
 
     /// Create a new empty identity.
-    pub(crate) fn empty(user_id: UserId) -> Self {
+    pub fn empty(user_id: &UserId) -> Self {
         Self {
-            user_id: Arc::new(user_id),
+            user_id: user_id.into(),
             shared: Arc::new(AtomicBool::new(false)),
             master_key: Arc::new(Mutex::new(None)),
             self_signing_key: Arc::new(Mutex::new(None)),
@@ -401,20 +427,20 @@ impl PrivateCrossSigningIdentity {
             .await
             .as_ref()
             .ok_or(SignatureError::MissingSigningKey)?
-            .sign_user(user_identity)
-            .await?;
+            .sign_user(user_identity)?;
 
-        let mut signed_keys = BTreeMap::new();
-
-        signed_keys.entry(user_identity.user_id().to_owned()).or_insert_with(BTreeMap::new).insert(
+        let mut user_signed_keys = SignedKeys::new();
+        user_signed_keys.add_cross_signing_keys(
             user_identity
                 .master_key()
                 .get_first_key()
                 .ok_or(SignatureError::MissingSigningKey)?
-                .to_owned(),
-            serde_json::to_value(master_key)?,
+                .to_base64()
+                .into(),
+            master_key.to_raw(),
         );
 
+        let signed_keys = [(user_identity.user_id().to_owned(), user_signed_keys)].into();
         Ok(SignatureUploadRequest::new(signed_keys))
     }
 
@@ -423,7 +449,7 @@ impl PrivateCrossSigningIdentity {
         &self,
         device: &ReadOnlyDevice,
     ) -> Result<SignatureUploadRequest, SignatureError> {
-        let mut device_keys = device.as_device_keys();
+        let mut device_keys = device.as_device_keys().to_owned();
         device_keys.signatures.clear();
         self.sign_device_keys(&mut device_keys).await
     }
@@ -439,25 +465,32 @@ impl PrivateCrossSigningIdentity {
 
     pub(crate) async fn sign_device_keys(
         &self,
-        mut device_keys: &mut DeviceKeys,
+        device_keys: &mut DeviceKeys,
     ) -> Result<SignatureUploadRequest, SignatureError> {
         self.self_signing_key
             .lock()
             .await
             .as_ref()
             .ok_or(SignatureError::MissingSigningKey)?
-            .sign_device(&mut device_keys)
-            .await?;
+            .sign_device(device_keys)?;
 
-        let mut signed_keys = BTreeMap::new();
-        signed_keys
-            .entry((&*self.user_id).to_owned())
-            .or_insert_with(BTreeMap::new)
-            .insert(device_keys.device_id.to_string(), serde_json::to_value(device_keys)?);
+        let mut user_signed_keys = SignedKeys::new();
+        user_signed_keys.add_device_keys(device_keys.device_id.clone(), device_keys.to_raw());
 
+        let signed_keys = [((*self.user_id).to_owned(), user_signed_keys)].into();
         Ok(SignatureUploadRequest::new(signed_keys))
     }
 
+    pub(crate) async fn sign(&self, message: &str) -> Result<Ed25519Signature, SignatureError> {
+        Ok(self
+            .master_key
+            .lock()
+            .await
+            .as_ref()
+            .ok_or(SignatureError::MissingSigningKey)?
+            .sign(message))
+    }
+
     /// Create a new identity for the given Olm Account.
     ///
     /// Returns the new identity, the upload signing keys request and a
@@ -469,7 +502,7 @@ impl PrivateCrossSigningIdentity {
     /// * `account` - The Olm account that is creating the new identity. The
     /// account will sign the master key and the self signing key will sign the
     /// account.
-    pub(crate) async fn new_with_account(
+    pub(crate) async fn with_account(
         account: &ReadOnlyAccount,
     ) -> (Self, UploadSigningKeysRequest, SignatureUploadRequest) {
         let master = Signing::new();
@@ -488,7 +521,7 @@ impl PrivateCrossSigningIdentity {
         let signature_request = identity
             .sign_account(account)
             .await
-            .expect("Can't sign own device with new cross signign keys");
+            .expect("Can't sign own device with new cross signing keys");
 
         let request = identity.as_upload_request().await;
 
@@ -498,19 +531,19 @@ impl PrivateCrossSigningIdentity {
     async fn new_helper(user_id: &UserId, master: MasterSigning) -> Self {
         let user = Signing::new();
         let mut public_key = user.cross_signing_key(user_id.to_owned(), KeyUsage::UserSigning);
-        master.sign_subkey(&mut public_key).await;
+        master.sign_subkey(&mut public_key);
 
         let user = UserSigning { inner: user, public_key: public_key.into() };
 
         let self_signing = Signing::new();
         let mut public_key =
             self_signing.cross_signing_key(user_id.to_owned(), KeyUsage::SelfSigning);
-        master.sign_subkey(&mut public_key).await;
+        master.sign_subkey(&mut public_key);
 
         let self_signing = SelfSigning { inner: self_signing, public_key: public_key.into() };
 
         Self {
-            user_id: Arc::new(user_id.to_owned()),
+            user_id: user_id.into(),
             shared: Arc::new(AtomicBool::new(false)),
             master_key: Arc::new(Mutex::new(Some(master))),
             self_signing_key: Arc::new(Mutex::new(Some(self_signing))),
@@ -520,8 +553,9 @@ impl PrivateCrossSigningIdentity {
 
     /// Create a new cross signing identity without signing the device that
     /// created it.
-    #[cfg(test)]
-    pub(crate) async fn new(user_id: UserId) -> Self {
+    #[cfg(any(test, feature = "testing"))]
+    #[allow(dead_code)]
+    pub async fn new(user_id: OwnedUserId) -> Self {
         let master = Signing::new();
 
         let public_key = master.cross_signing_key(user_id.clone(), KeyUsage::Master);
@@ -530,8 +564,11 @@ impl PrivateCrossSigningIdentity {
         Self::new_helper(&user_id, master).await
     }
 
-    #[cfg(test)]
-    pub(crate) async fn reset(&mut self) {
+    #[cfg(any(test, feature = "testing"))]
+    #[allow(dead_code)]
+    /// Testing helper to reset this CrossSigning with a fresh one using the
+    /// local ideniy
+    pub async fn reset(&mut self) {
         let new = Self::new(self.user_id().to_owned()).await;
         *self = new
     }
@@ -559,36 +596,19 @@ impl PrivateCrossSigningIdentity {
     /// # Panics
     ///
     /// This will panic if the provided pickle key isn't 32 bytes long.
-    pub async fn pickle(
-        &self,
-        pickle_key: &[u8],
-    ) -> Result<PickledCrossSigningIdentity, JsonError> {
-        let master_key = if let Some(m) = self.master_key.lock().await.as_ref() {
-            Some(m.pickle(pickle_key).await)
-        } else {
-            None
-        };
+    pub async fn pickle(&self) -> Result<PickledCrossSigningIdentity, JsonError> {
+        let master_key = self.master_key.lock().await.as_ref().map(|m| m.pickle());
 
-        let self_signing_key = if let Some(m) = self.self_signing_key.lock().await.as_ref() {
-            Some(m.pickle(pickle_key).await)
-        } else {
-            None
-        };
+        let self_signing_key = self.self_signing_key.lock().await.as_ref().map(|m| m.pickle());
 
-        let user_signing_key = if let Some(m) = self.user_signing_key.lock().await.as_ref() {
-            Some(m.pickle(pickle_key).await)
-        } else {
-            None
-        };
+        let user_signing_key = self.user_signing_key.lock().await.as_ref().map(|m| m.pickle());
 
-        let pickle = PickledSignings { master_key, user_signing_key, self_signing_key };
-
-        let pickle = serde_json::to_string(&pickle)?;
+        let keys = PickledSignings { master_key, user_signing_key, self_signing_key };
 
         Ok(PickledCrossSigningIdentity {
             user_id: self.user_id.as_ref().to_owned(),
             shared: self.shared(),
-            pickle,
+            keys,
         })
     }
 
@@ -597,27 +617,15 @@ impl PrivateCrossSigningIdentity {
     /// # Panic
     ///
     /// Panics if the pickle_key isn't 32 bytes long.
-    pub async fn from_pickle(
-        pickle: PickledCrossSigningIdentity,
-        pickle_key: &[u8],
-    ) -> Result<Self, SigningError> {
-        let signings: PickledSignings = serde_json::from_str(&pickle.pickle)?;
+    pub async fn from_pickle(pickle: PickledCrossSigningIdentity) -> Result<Self, SigningError> {
+        let keys = pickle.keys;
 
-        let master =
-            signings.master_key.map(|m| MasterSigning::from_pickle(m, pickle_key)).transpose()?;
-
-        let self_signing = signings
-            .self_signing_key
-            .map(|s| SelfSigning::from_pickle(s, pickle_key))
-            .transpose()?;
-
-        let user_signing = signings
-            .user_signing_key
-            .map(|s| UserSigning::from_pickle(s, pickle_key))
-            .transpose()?;
+        let master = keys.master_key.map(MasterSigning::from_pickle).transpose()?;
+        let self_signing = keys.self_signing_key.map(SelfSigning::from_pickle).transpose()?;
+        let user_signing = keys.user_signing_key.map(UserSigning::from_pickle).transpose()?;
 
         Ok(Self {
-            user_id: Arc::new(pickle.user_id),
+            user_id: (&*pickle.user_id).into(),
             shared: Arc::new(AtomicBool::from(pickle.shared)),
             master_key: Arc::new(Mutex::new(master)),
             self_signing_key: Arc::new(Mutex::new(self_signing)),
@@ -629,24 +637,22 @@ impl PrivateCrossSigningIdentity {
     /// identity.
     pub(crate) async fn as_upload_request(&self) -> UploadSigningKeysRequest {
         let master_key =
-            self.master_key.lock().await.as_ref().map(|k| k.public_key.to_owned().into());
+            self.master_key.lock().await.as_ref().map(|k| k.public_key.as_ref().clone());
 
         let user_signing_key =
-            self.user_signing_key.lock().await.as_ref().map(|k| k.public_key.to_owned().into());
+            self.user_signing_key.lock().await.as_ref().map(|k| k.public_key.as_ref().clone());
 
         let self_signing_key =
-            self.self_signing_key.lock().await.as_ref().map(|k| k.public_key.to_owned().into());
+            self.self_signing_key.lock().await.as_ref().map(|k| k.public_key.as_ref().clone());
 
         UploadSigningKeysRequest { master_key, self_signing_key, user_signing_key }
     }
 }
 
 #[cfg(test)]
-mod test {
-    use std::sync::Arc;
-
+mod tests {
     use matrix_sdk_test::async_test;
-    use ruma::{user_id, UserId};
+    use ruma::{device_id, user_id, UserId};
 
     use super::{PrivateCrossSigningIdentity, Signing};
     use crate::{
@@ -654,43 +660,33 @@ mod test {
         olm::ReadOnlyAccount,
     };
 
-    fn user_id() -> UserId {
+    fn user_id() -> &'static UserId {
         user_id!("@example:localhost")
     }
 
-    fn pickle_key() -> &'static [u8] {
-        &[0u8; 32]
-    }
-
     #[test]
-    fn signing_creation() {
-        let signing = Signing::new();
-        assert!(!signing.public_key().as_str().is_empty());
-    }
-
-    #[async_test]
-    async fn signature_verification() {
+    fn signature_verification() {
         let signing = Signing::new();
 
         let message = "Hello world";
 
-        let signature = signing.sign(message).await;
-        assert!(signing.verify(message, &signature).await.is_ok());
+        let signature = signing.sign(message);
+        assert!(signing.verify(message, &signature).is_ok());
     }
 
-    #[async_test]
-    async fn pickling_signing() {
+    #[test]
+    fn pickling_signing() {
         let signing = Signing::new();
-        let pickled = signing.pickle(pickle_key()).await;
+        let pickled = signing.pickle();
 
-        let unpickled = Signing::from_pickle(pickled, pickle_key()).unwrap();
+        let unpickled = Signing::from_pickle(pickled).unwrap();
 
         assert_eq!(signing.public_key(), unpickled.public_key());
     }
 
     #[async_test]
     async fn private_identity_creation() {
-        let identity = PrivateCrossSigningIdentity::new(user_id()).await;
+        let identity = PrivateCrossSigningIdentity::new(user_id().to_owned()).await;
 
         let master_key = identity.master_key.lock().await;
         let master_key = master_key.as_ref().unwrap();
@@ -708,12 +704,11 @@ mod test {
 
     #[async_test]
     async fn identity_pickling() {
-        let identity = PrivateCrossSigningIdentity::new(user_id()).await;
+        let identity = PrivateCrossSigningIdentity::new(user_id().to_owned()).await;
 
-        let pickled = identity.pickle(pickle_key()).await.unwrap();
+        let pickled = identity.pickle().await.unwrap();
 
-        let unpickled =
-            PrivateCrossSigningIdentity::from_pickle(pickled, pickle_key()).await.unwrap();
+        let unpickled = PrivateCrossSigningIdentity::from_pickle(pickled).await.unwrap();
 
         assert_eq!(identity.user_id, unpickled.user_id);
         assert_eq!(&*identity.master_key.lock().await, &*unpickled.master_key.lock().await);
@@ -728,9 +723,9 @@ mod test {
     }
 
     #[async_test]
-    async fn private_identity_signed_by_accound() {
-        let account = ReadOnlyAccount::new(&user_id(), "DEVICEID".into());
-        let (identity, _, _) = PrivateCrossSigningIdentity::new_with_account(&account).await;
+    async fn private_identity_signed_by_account() {
+        let account = ReadOnlyAccount::new(user_id(), device_id!("DEVICEID"));
+        let (identity, _, _) = PrivateCrossSigningIdentity::with_account(&account).await;
         let master = identity.master_key.lock().await;
         let master = master.as_ref().unwrap();
 
@@ -739,16 +734,16 @@ mod test {
 
     #[async_test]
     async fn sign_device() {
-        let account = ReadOnlyAccount::new(&user_id(), "DEVICEID".into());
-        let (identity, _, _) = PrivateCrossSigningIdentity::new_with_account(&account).await;
+        let account = ReadOnlyAccount::new(user_id(), device_id!("DEVICEID"));
+        let (identity, _, _) = PrivateCrossSigningIdentity::with_account(&account).await;
 
         let mut device = ReadOnlyDevice::from_account(&account).await;
         let self_signing = identity.self_signing_key.lock().await;
         let self_signing = self_signing.as_ref().unwrap();
 
-        let mut device_keys = device.as_device_keys();
-        self_signing.sign_device(&mut device_keys).await.unwrap();
-        device.signatures = Arc::new(device_keys.signatures);
+        let mut device_keys = device.as_device_keys().to_owned();
+        self_signing.sign_device(&mut device_keys).unwrap();
+        device.update_device(&device_keys).unwrap();
 
         let public_key = &self_signing.public_key;
         public_key.verify_device(&device).unwrap()
@@ -756,17 +751,20 @@ mod test {
 
     #[async_test]
     async fn sign_user_identity() {
-        let account = ReadOnlyAccount::new(&user_id(), "DEVICEID".into());
-        let (identity, _, _) = PrivateCrossSigningIdentity::new_with_account(&account).await;
+        let account = ReadOnlyAccount::new(user_id(), device_id!("DEVICEID"));
+        let (identity, _, _) = PrivateCrossSigningIdentity::with_account(&account).await;
 
-        let bob_account = ReadOnlyAccount::new(&user_id!("@bob:localhost"), "DEVICEID".into());
-        let (bob_private, _, _) = PrivateCrossSigningIdentity::new_with_account(&bob_account).await;
+        let bob_account = ReadOnlyAccount::new(user_id!("@bob:localhost"), device_id!("DEVICEID"));
+        let (bob_private, _, _) = PrivateCrossSigningIdentity::with_account(&bob_account).await;
         let mut bob_public = ReadOnlyUserIdentity::from_private(&bob_private).await;
 
         let user_signing = identity.user_signing_key.lock().await;
         let user_signing = user_signing.as_ref().unwrap();
 
-        let master = user_signing.sign_user(&bob_public).await.unwrap();
+        let master = user_signing.sign_user(&bob_public).unwrap();
+
+        let num_signatures: usize = master.signatures.iter().map(|(_, u)| u.len()).sum();
+        assert_eq!(num_signatures, 1, "We're only uploading our own signature");
 
         bob_public.master_key = master.into();
 

crates/matrix-sdk-crypto/src/olm/signing/pk_signing.rs

@@ -0,0 +1,337 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{collections::BTreeMap, convert::TryInto};
+
+use ruma::{
+    encryption::KeyUsage, serde::CanonicalJsonValue, DeviceKeyAlgorithm, DeviceKeyId, OwnedUserId,
+};
+use serde::{Deserialize, Serialize};
+use serde_json::{Error as JsonError, Value};
+use thiserror::Error;
+use vodozemac::{Ed25519PublicKey, Ed25519SecretKey, Ed25519Signature, KeyError};
+
+use crate::{
+    error::SignatureError,
+    identities::{MasterPubkey, SelfSigningPubkey, UserSigningPubkey},
+    types::{CrossSigningKey, CrossSigningKeySignatures, DeviceKeys},
+    utilities::{encode, DecodeError},
+    ReadOnlyUserIdentity,
+};
+
+/// Error type reporting failures in the signing operations.
+#[derive(Debug, Error)]
+pub enum SigningError {
+    /// Error decoding the base64 encoded pickle data.
+    #[error(transparent)]
+    Decode(#[from] DecodeError),
+
+    /// Error decrypting the pickled signing seed
+    #[error("Error decrypting the pickled signing seed")]
+    Decryption(String),
+
+    /// Error deserializing the pickle data.
+    #[error(transparent)]
+    Json(#[from] JsonError),
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Signing {
+    inner: Ed25519SecretKey,
+    public_key: Ed25519PublicKey,
+}
+
+impl std::fmt::Debug for Signing {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Signing").field("public_key", &self.public_key.to_base64()).finish()
+    }
+}
+
+impl PartialEq for Signing {
+    fn eq(&self, other: &Signing) -> bool {
+        self.public_key == other.public_key
+    }
+}
+
+#[derive(PartialEq, Debug)]
+pub struct MasterSigning {
+    pub inner: Signing,
+    pub public_key: MasterPubkey,
+}
+
+#[derive(Deserialize, Serialize)]
+#[allow(missing_debug_implementations)]
+pub struct PickledMasterSigning {
+    pickle: PickledSigning,
+    public_key: CrossSigningKey,
+}
+
+#[derive(Deserialize, Serialize)]
+#[allow(missing_debug_implementations)]
+pub struct PickledUserSigning {
+    pickle: PickledSigning,
+    public_key: CrossSigningKey,
+}
+
+#[derive(Deserialize, Serialize)]
+#[allow(missing_debug_implementations)]
+pub struct PickledSelfSigning {
+    pickle: PickledSigning,
+    public_key: CrossSigningKey,
+}
+
+impl MasterSigning {
+    pub fn pickle(&self) -> PickledMasterSigning {
+        let pickle = self.inner.pickle();
+        let public_key = self.public_key.as_ref().clone();
+        PickledMasterSigning { pickle, public_key }
+    }
+
+    pub fn export_seed(&self) -> String {
+        encode(self.inner.as_bytes())
+    }
+
+    pub fn from_base64(user_id: OwnedUserId, key: &str) -> Result<Self, KeyError> {
+        let inner = Signing::from_base64(key)?;
+        let public_key = inner.cross_signing_key(user_id, KeyUsage::Master).into();
+
+        Ok(Self { inner, public_key })
+    }
+
+    pub fn from_pickle(pickle: PickledMasterSigning) -> Result<Self, SigningError> {
+        let inner = Signing::from_pickle(pickle.pickle)?;
+
+        Ok(Self { inner, public_key: pickle.public_key.into() })
+    }
+
+    pub fn sign(&self, message: &str) -> Ed25519Signature {
+        self.inner.sign(message)
+    }
+
+    pub fn sign_subkey(&self, subkey: &mut CrossSigningKey) {
+        let json_subkey = serde_json::to_value(&subkey).expect("Can't serialize cross signing key");
+        let signature = self.inner.sign_json(json_subkey).expect("Can't sign cross signing keys");
+
+        subkey
+            .signatures
+            .entry(self.public_key.user_id().to_owned())
+            .or_insert_with(BTreeMap::new)
+            .insert(
+                DeviceKeyId::from_parts(
+                    DeviceKeyAlgorithm::Ed25519,
+                    self.inner.public_key.to_base64().as_str().into(),
+                ),
+                signature.to_base64(),
+            );
+    }
+}
+
+impl UserSigning {
+    pub fn pickle(&self) -> PickledUserSigning {
+        let pickle = self.inner.pickle();
+        let public_key = self.public_key.as_ref().clone();
+        PickledUserSigning { pickle, public_key }
+    }
+
+    pub fn export_seed(&self) -> String {
+        encode(self.inner.as_bytes())
+    }
+
+    pub fn from_base64(user_id: OwnedUserId, key: &str) -> Result<Self, KeyError> {
+        let inner = Signing::from_base64(key)?;
+        let public_key = inner.cross_signing_key(user_id, KeyUsage::UserSigning).into();
+
+        Ok(Self { inner, public_key })
+    }
+
+    pub fn sign_user(
+        &self,
+        user: &ReadOnlyUserIdentity,
+    ) -> Result<CrossSigningKey, SignatureError> {
+        let signatures = self.sign_user_helper(user)?;
+        let mut master_key = user.master_key().as_ref().clone();
+
+        master_key.signatures = signatures;
+
+        Ok(master_key)
+    }
+
+    pub fn sign_user_helper(
+        &self,
+        user: &ReadOnlyUserIdentity,
+    ) -> Result<CrossSigningKeySignatures, SignatureError> {
+        let user_master: &CrossSigningKey = user.master_key().as_ref();
+        let signature = self.inner.sign_json(serde_json::to_value(user_master)?)?;
+
+        let mut signatures = BTreeMap::new();
+
+        signatures
+            .entry(self.public_key.user_id().to_owned())
+            .or_insert_with(BTreeMap::new)
+            .insert(
+                DeviceKeyId::from_parts(
+                    DeviceKeyAlgorithm::Ed25519,
+                    self.inner.public_key.to_base64().as_str().into(),
+                ),
+                signature.to_base64(),
+            );
+
+        Ok(signatures)
+    }
+
+    pub fn from_pickle(pickle: PickledUserSigning) -> Result<Self, SigningError> {
+        let inner = Signing::from_pickle(pickle.pickle)?;
+
+        Ok(Self { inner, public_key: pickle.public_key.into() })
+    }
+}
+
+impl SelfSigning {
+    pub fn pickle(&self) -> PickledSelfSigning {
+        let pickle = self.inner.pickle();
+        let public_key = self.public_key.as_ref().clone();
+        PickledSelfSigning { pickle, public_key }
+    }
+
+    pub fn export_seed(&self) -> String {
+        encode(self.inner.as_bytes())
+    }
+
+    pub fn from_base64(user_id: OwnedUserId, key: &str) -> Result<Self, KeyError> {
+        let inner = Signing::from_base64(key)?;
+        let public_key = inner.cross_signing_key(user_id, KeyUsage::SelfSigning).into();
+
+        Ok(Self { inner, public_key })
+    }
+
+    pub fn sign_device_helper(&self, value: Value) -> Result<Ed25519Signature, SignatureError> {
+        self.inner.sign_json(value)
+    }
+
+    pub fn sign_device(&self, device_keys: &mut DeviceKeys) -> Result<(), SignatureError> {
+        let signature = self.sign_device_helper(serde_json::to_value(&device_keys)?)?;
+
+        device_keys.signatures.entry(self.public_key.user_id().to_owned()).or_default().insert(
+            DeviceKeyId::from_parts(
+                DeviceKeyAlgorithm::Ed25519,
+                self.inner.public_key.to_base64().as_str().into(),
+            ),
+            signature.to_base64(),
+        );
+
+        Ok(())
+    }
+
+    pub fn from_pickle(pickle: PickledSelfSigning) -> Result<Self, SigningError> {
+        let inner = Signing::from_pickle(pickle.pickle)?;
+
+        Ok(Self { inner, public_key: pickle.public_key.into() })
+    }
+}
+
+#[derive(PartialEq, Debug)]
+pub struct SelfSigning {
+    pub inner: Signing,
+    pub public_key: SelfSigningPubkey,
+}
+
+#[derive(PartialEq, Debug)]
+pub struct UserSigning {
+    pub inner: Signing,
+    pub public_key: UserSigningPubkey,
+}
+
+#[derive(Serialize, Deserialize)]
+#[allow(missing_debug_implementations)]
+pub struct PickledSignings {
+    pub master_key: Option<PickledMasterSigning>,
+    pub user_signing_key: Option<PickledUserSigning>,
+    pub self_signing_key: Option<PickledSelfSigning>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct PickledSigning(Ed25519SecretKey);
+
+impl Signing {
+    pub fn new() -> Self {
+        let secret_key = Ed25519SecretKey::new();
+        Self::new_helper(secret_key)
+    }
+
+    fn new_helper(secret_key: Ed25519SecretKey) -> Self {
+        let public_key = secret_key.public_key();
+
+        Signing { inner: secret_key, public_key }
+    }
+
+    pub fn from_base64(key: &str) -> Result<Self, KeyError> {
+        let key = Ed25519SecretKey::from_base64(key)?;
+        Ok(Self::new_helper(key))
+    }
+
+    pub fn as_bytes(&self) -> &[u8] {
+        self.inner.as_bytes()
+    }
+
+    pub fn from_pickle(pickle: PickledSigning) -> Result<Self, SigningError> {
+        Ok(Self::new_helper(pickle.0))
+    }
+
+    pub fn pickle(&self) -> PickledSigning {
+        PickledSigning(
+            Ed25519SecretKey::from_slice(self.inner.as_bytes())
+                .expect("Copying the private key should work"),
+        )
+    }
+
+    pub fn public_key(&self) -> Ed25519PublicKey {
+        self.public_key
+    }
+
+    pub fn cross_signing_key(&self, user_id: OwnedUserId, usage: KeyUsage) -> CrossSigningKey {
+        let keys = BTreeMap::from([(
+            DeviceKeyId::from_parts(
+                DeviceKeyAlgorithm::Ed25519,
+                self.public_key().to_base64().as_str().into(),
+            ),
+            self.inner.public_key().into(),
+        )]);
+
+        CrossSigningKey::new(user_id, vec![usage], keys, BTreeMap::new())
+    }
+
+    #[cfg(test)]
+    pub fn verify(
+        &self,
+        message: &str,
+        signature: &Ed25519Signature,
+    ) -> Result<(), SignatureError> {
+        Ok(self.public_key.verify(message.as_bytes(), signature)?)
+    }
+
+    pub fn sign_json(&self, mut json: Value) -> Result<Ed25519Signature, SignatureError> {
+        let json_object = json.as_object_mut().ok_or(SignatureError::NotAnObject)?;
+        let _ = json_object.remove("signatures");
+        let _ = json_object.remove("unsigned");
+
+        let canonical_json: CanonicalJsonValue =
+            json.try_into().expect("Can't canonicalize the json value");
+
+        Ok(self.sign(&canonical_json.to_string()))
+    }
+
+    pub fn sign(&self, message: &str) -> Ed25519Signature {
+        self.inner.sign(message.as_bytes())
+    }
+}

crates/matrix-sdk-crypto/src/olm/utility.rs

@@ -14,21 +14,12 @@
 
 use std::convert::TryInto;
 
-use olm_rs::utility::OlmUtility;
 use ruma::{serde::CanonicalJsonValue, DeviceKeyAlgorithm, DeviceKeyId, UserId};
 use serde_json::Value;
 
 use crate::error::SignatureError;
 
-pub(crate) struct Utility {
-    inner: OlmUtility,
-}
-
-impl Utility {
-    pub fn new() -> Self {
-        Self { inner: OlmUtility::new() }
-    }
-
+pub trait VerifyJson {
     /// Verify a signed JSON object.
     ///
     /// The object must have a signatures key associated  with an object of the
@@ -43,15 +34,20 @@ impl Utility {
     ///
     /// * `key_id` - The id of the key that signed the JSON object.
     ///
-    /// * `signing_key` - The public ed25519 key which was used to sign the JSON
-    ///   object.
-    ///
     /// * `json` - The JSON object that should be verified.
-    pub(crate) fn verify_json(
+    fn verify_json(
+        &self,
+        user_id: &UserId,
+        key_id: &DeviceKeyId,
+        json: &mut Value,
+    ) -> Result<(), SignatureError>;
+}
+
+impl VerifyJson for vodozemac::Ed25519PublicKey {
+    fn verify_json(
         &self,
         user_id: &UserId,
         key_id: &DeviceKeyId,
-        signing_key: &str,
         json: &mut Value,
     ) -> Result<(), SignatureError> {
         if key_id.algorithm() != DeviceKeyAlgorithm::Ed25519 {
@@ -75,30 +71,31 @@ impl Utility {
             signature.get(key_id.to_string()).ok_or(SignatureError::NoSignatureFound)?;
         let signature = signature.as_str().ok_or(SignatureError::NoSignatureFound)?;
 
-        let ret =
-            match self.inner.ed25519_verify(signing_key, &canonical_json, signature.to_owned()) {
-                Ok(_) => Ok(()),
-                Err(_) => Err(SignatureError::VerificationError),
-            };
+        let signature = vodozemac::Ed25519Signature::from_base64(signature)?;
+
+        let ret = self
+            .verify(canonical_json.as_bytes(), &signature)
+            .map_err(SignatureError::VerificationError);
 
         let json_object = json.as_object_mut().ok_or(SignatureError::NotAnObject)?;
 
         if let Some(u) = unsigned {
-            json_object.insert("unsigned".to_string(), u);
+            json_object.insert("unsigned".to_owned(), u);
         }
 
-        json_object.insert("signatures".to_string(), signatures);
+        json_object.insert("signatures".to_owned(), signatures);
 
         ret
     }
 }
 
 #[cfg(test)]
-mod test {
-    use ruma::{user_id, DeviceKeyAlgorithm, DeviceKeyId};
+mod tests {
+    use ruma::{device_id, user_id, DeviceKeyAlgorithm, DeviceKeyId};
     use serde_json::json;
+    use vodozemac::Ed25519PublicKey;
 
-    use super::Utility;
+    use super::VerifyJson;
 
     #[test]
     fn signature_test() {
@@ -125,13 +122,13 @@ mod test {
 
         let signing_key = "n469gw7zm+KW+JsFIJKnFVvCKU14HwQyocggcCIQgZY";
 
-        let utility = Utility::new();
+        let signing_key = Ed25519PublicKey::from_base64(signing_key)
+            .expect("The signing key wasn't proper base64");
 
-        utility
+        signing_key
             .verify_json(
-                &user_id!("@example:localhost"),
-                &DeviceKeyId::from_parts(DeviceKeyAlgorithm::Ed25519, "GBEWHQOYGS".into()),
-                signing_key,
+                user_id!("@example:localhost"),
+                &DeviceKeyId::from_parts(DeviceKeyAlgorithm::Ed25519, device_id!("GBEWHQOYGS")),
                 &mut device_keys,
             )
             .expect("Can't verify device keys");

crates/matrix-sdk-crypto/src/requests.rs

@@ -12,42 +12,44 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::{collections::BTreeMap, sync::Arc, time::Duration};
+use std::{collections::BTreeMap, iter, sync::Arc, time::Duration};
 
-use matrix_sdk_common::uuid::Uuid;
 use ruma::{
-    api::client::r0::{
+    api::client::{
+        backup::{add_backup_keys::v3::Response as KeysBackupResponse, RoomKeyBackup},
         keys::{
-            claim_keys::{Request as KeysClaimRequest, Response as KeysClaimResponse},
-            get_keys::Response as KeysQueryResponse,
-            upload_keys::{Request as KeysUploadRequest, Response as KeysUploadResponse},
-            upload_signatures::{
+            claim_keys::v3::{Request as KeysClaimRequest, Response as KeysClaimResponse},
+            get_keys::v3::Response as KeysQueryResponse,
+            upload_keys::v3::{Request as KeysUploadRequest, Response as KeysUploadResponse},
+            upload_signatures::v3::{
                 Request as SignatureUploadRequest, Response as SignatureUploadResponse,
             },
-            upload_signing_keys::Response as SigningKeysUploadResponse,
+            upload_signing_keys::v3::Response as SigningKeysUploadResponse,
         },
-        message::send_message_event::Response as RoomMessageResponse,
-        to_device::send_event_to_device::Response as ToDeviceResponse,
+        message::send_message_event::v3::Response as RoomMessageResponse,
+        to_device::send_event_to_device::v3::Response as ToDeviceResponse,
+    },
+    events::{
+        AnyMessageLikeEventContent, AnyToDeviceEventContent, EventContent, ToDeviceEventType,
     },
-    encryption::CrossSigningKey,
-    events::{AnyMessageEventContent, AnyToDeviceEventContent, EventContent, EventType},
     serde::Raw,
     to_device::DeviceIdOrAllDevices,
-    DeviceIdBox, RoomId, UserId,
+    OwnedDeviceId, OwnedRoomId, OwnedTransactionId, OwnedUserId, TransactionId, UserId,
 };
 use serde::{Deserialize, Serialize};
 
+use crate::types::CrossSigningKey;
+
 /// Customized version of
-/// `ruma_client_api::r0::to_device::send_event_to_device::Request`,
-/// using a UUID for the transaction ID.
+/// `ruma_client_api::to_device::send_event_to_device::v3::Request`
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct ToDeviceRequest {
     /// Type of event being sent to each device.
-    pub event_type: EventType,
+    pub event_type: ToDeviceEventType,
 
     /// A request identifier unique to the access token used to send the
     /// request.
-    pub txn_id: Uuid,
+    pub txn_id: OwnedTransactionId,
 
     /// A map of users to devices to a content for a message event to be
     /// sent to the user's device. Individual message events can be sent
@@ -55,7 +57,8 @@ pub struct ToDeviceRequest {
     /// The content's type for this field will be updated in a future
     /// release, until then you can create a value using
     /// `serde_json::value::to_raw_value`.
-    pub messages: BTreeMap<UserId, BTreeMap<DeviceIdOrAllDevices, Raw<AnyToDeviceEventContent>>>,
+    pub messages:
+        BTreeMap<OwnedUserId, BTreeMap<DeviceIdOrAllDevices, Raw<AnyToDeviceEventContent>>>,
 }
 
 impl ToDeviceRequest {
@@ -75,53 +78,48 @@ impl ToDeviceRequest {
         recipient_device: impl Into<DeviceIdOrAllDevices>,
         content: AnyToDeviceEventContent,
     ) -> Self {
-        Self::new_with_id(recipient, recipient_device, content, Uuid::new_v4())
+        Self::with_id(recipient, recipient_device, content, TransactionId::new())
     }
 
-    pub(crate) fn new_for_recipients(
+    pub(crate) fn for_recipients(
         recipient: &UserId,
-        recipient_devices: Vec<DeviceIdBox>,
+        recipient_devices: Vec<OwnedDeviceId>,
         content: AnyToDeviceEventContent,
-        txn_id: Uuid,
+        txn_id: OwnedTransactionId,
     ) -> Self {
-        let mut messages = BTreeMap::new();
-        let event_type = EventType::from(content.event_type());
-
         if recipient_devices.is_empty() {
             Self::new(recipient, DeviceIdOrAllDevices::AllDevices, content)
         } else {
+            let event_type = content.event_type();
             let device_messages = recipient_devices
                 .into_iter()
-                .map(|d| (DeviceIdOrAllDevices::DeviceId(d), Raw::from(content.clone())))
+                .map(|d| {
+                    let raw_content =
+                        Raw::new(&content).expect("Failed to serialize to-device event");
+                    (DeviceIdOrAllDevices::DeviceId(d), raw_content)
+                })
                 .collect();
 
-            messages.insert(recipient.clone(), device_messages);
-
+            let messages = iter::once((recipient.to_owned(), device_messages)).collect();
             ToDeviceRequest { event_type, txn_id, messages }
         }
     }
 
-    pub(crate) fn new_with_id(
+    pub(crate) fn with_id(
         recipient: &UserId,
         recipient_device: impl Into<DeviceIdOrAllDevices>,
         content: AnyToDeviceEventContent,
-        txn_id: Uuid,
+        txn_id: OwnedTransactionId,
     ) -> Self {
-        let mut messages = BTreeMap::new();
-        let mut user_messages = BTreeMap::new();
-        let event_type = EventType::from(content.event_type());
+        let event_type = content.event_type();
+        let raw_content = Raw::new(&content).expect("Failed to serialize to-device event");
 
-        user_messages.insert(recipient_device.into(), Raw::from(content));
-        messages.insert(recipient.clone(), user_messages);
+        let user_messages = iter::once((recipient_device.into(), raw_content)).collect();
+        let messages = iter::once((recipient.to_owned(), user_messages)).collect();
 
         ToDeviceRequest { event_type, txn_id, messages }
     }
 
-    /// Gets the transaction ID as a string.
-    pub fn txn_id_string(&self) -> String {
-        self.txn_id.to_string()
-    }
-
     /// Get the number of unique messages this request contains.
     ///
     /// *Note*: A single message may be sent to multiple devices, so this may or
@@ -149,7 +147,7 @@ pub struct UploadSigningKeysRequest {
 }
 
 /// Customized version of
-/// `ruma_client_api::r0::keys::get_keys::Request`, without any
+/// `ruma_client_api::keys::get_keys::v3::Request`, without any
 /// references.
 #[derive(Clone, Debug)]
 pub struct KeysQueryRequest {
@@ -159,7 +157,7 @@ pub struct KeysQueryRequest {
 
     /// The keys to be downloaded. An empty list indicates all devices for
     /// the corresponding user.
-    pub device_keys: BTreeMap<UserId, Vec<DeviceIdBox>>,
+    pub device_keys: BTreeMap<OwnedUserId, Vec<OwnedDeviceId>>,
 
     /// If the client is fetching keys as a result of a device update
     /// received in a sync request, this should be the 'since' token of that
@@ -170,7 +168,7 @@ pub struct KeysQueryRequest {
 }
 
 impl KeysQueryRequest {
-    pub(crate) fn new(device_keys: BTreeMap<UserId, Vec<DeviceIdBox>>) -> Self {
+    pub(crate) fn new(device_keys: BTreeMap<OwnedUserId, Vec<OwnedDeviceId>>) -> Self {
         Self { timeout: None, device_keys, token: None }
     }
 }
@@ -197,6 +195,8 @@ pub enum OutgoingRequests {
     /// A room message request, usually for sending in-room interactive
     /// verification events.
     RoomMessage(RoomMessageRequest),
+    /// A request that will back up a batch of room keys to the server.
+    KeysBackup(KeysBackupRequest),
 }
 
 #[cfg(test)]
@@ -215,6 +215,12 @@ impl From<KeysQueryRequest> for OutgoingRequests {
     }
 }
 
+impl From<KeysBackupRequest> for OutgoingRequests {
+    fn from(r: KeysBackupRequest) -> Self {
+        OutgoingRequests::KeysBackup(r)
+    }
+}
+
 impl From<KeysClaimRequest> for OutgoingRequests {
     fn from(r: KeysClaimRequest) -> Self {
         Self::KeysClaim(r)
@@ -247,13 +253,13 @@ impl From<SignatureUploadRequest> for OutgoingRequests {
 
 impl From<OutgoingVerificationRequest> for OutgoingRequest {
     fn from(r: OutgoingVerificationRequest) -> Self {
-        Self { request_id: r.request_id(), request: Arc::new(r.into()) }
+        Self { request_id: r.request_id().to_owned(), request: Arc::new(r.into()) }
     }
 }
 
 impl From<SignatureUploadRequest> for OutgoingRequest {
     fn from(r: SignatureUploadRequest) -> Self {
-        Self { request_id: Uuid::new_v4(), request: Arc::new(r.into()) }
+        Self { request_id: TransactionId::new(), request: Arc::new(r.into()) }
     }
 }
 
@@ -278,6 +284,8 @@ pub enum IncomingResponse<'a> {
     SignatureUpload(&'a SignatureUploadResponse),
     /// A room message response, usually for interactive verifications.
     RoomMessage(&'a RoomMessageResponse),
+    /// Response for the server-side room key backup request.
+    KeysBackup(&'a KeysBackupResponse),
 }
 
 impl<'a> From<&'a KeysUploadResponse> for IncomingResponse<'a> {
@@ -286,6 +294,12 @@ impl<'a> From<&'a KeysUploadResponse> for IncomingResponse<'a> {
     }
 }
 
+impl<'a> From<&'a KeysBackupResponse> for IncomingResponse<'a> {
+    fn from(response: &'a KeysBackupResponse) -> Self {
+        IncomingResponse::KeysBackup(response)
+    }
+}
+
 impl<'a> From<&'a KeysQueryResponse> for IncomingResponse<'a> {
     fn from(response: &'a KeysQueryResponse) -> Self {
         IncomingResponse::KeysQuery(response)
@@ -322,14 +336,14 @@ impl<'a> From<&'a SignatureUploadResponse> for IncomingResponse<'a> {
 pub struct OutgoingRequest {
     /// The unique id of a request, needs to be passed when receiving a
     /// response.
-    pub(crate) request_id: Uuid,
+    pub(crate) request_id: OwnedTransactionId,
     /// The underlying outgoing request.
     pub(crate) request: Arc<OutgoingRequests>,
 }
 
 impl OutgoingRequest {
     /// Get the unique id of this request.
-    pub fn request_id(&self) -> &Uuid {
+    pub fn request_id(&self) -> &TransactionId {
         &self.request_id
     }
 
@@ -343,17 +357,27 @@ impl OutgoingRequest {
 #[derive(Clone, Debug)]
 pub struct RoomMessageRequest {
     /// The room to send the event to.
-    pub room_id: RoomId,
+    pub room_id: OwnedRoomId,
 
     /// The transaction ID for this event.
     ///
     /// Clients should generate an ID unique across requests with the
     /// same access token; it will be used by the server to ensure
     /// idempotency of requests.
-    pub txn_id: Uuid,
+    pub txn_id: OwnedTransactionId,
 
     /// The event content to send.
-    pub content: AnyMessageEventContent,
+    pub content: AnyMessageLikeEventContent,
+}
+
+/// A request that will back up a batch of room keys to the server.
+#[derive(Clone, Debug)]
+pub struct KeysBackupRequest {
+    /// The backup version that these room keys should be part of.
+    pub version: String,
+    /// The map from room id to a backed up room key that we're going to upload
+    /// to the server.
+    pub rooms: BTreeMap<OwnedRoomId, RoomKeyBackup>,
 }
 
 /// An enum over the different outgoing verification based requests.
@@ -367,10 +391,10 @@ pub enum OutgoingVerificationRequest {
 
 impl OutgoingVerificationRequest {
     /// Get the unique id of this request.
-    pub fn request_id(&self) -> Uuid {
+    pub fn request_id(&self) -> &TransactionId {
         match self {
-            OutgoingVerificationRequest::ToDevice(t) => t.txn_id,
-            OutgoingVerificationRequest::InRoom(r) => r.txn_id,
+            OutgoingVerificationRequest::ToDevice(t) => &t.txn_id,
+            OutgoingVerificationRequest::InRoom(r) => &r.txn_id,
         }
     }
 }