v31.3.0-rc.4

[Backport staging] Add utility to check for non migrated legacy db

.github/CODEOWNERS

@@ -1,8 +1,15 @@
-*                         @matrix-org/element-web
-/.github/workflows/**     @matrix-org/element-web-app-team
-/package.json             @matrix-org/element-web-app-team
-/yarn.lock                @matrix-org/element-web-app-team
+*                         @matrix-org/element-web-reviewers
+/.github/workflows/**     @matrix-org/element-web-team
+/package.json             @matrix-org/element-web-team
+/yarn.lock                @matrix-org/element-web-team
 /src/webrtc               @matrix-org/element-call-reviewers
 /src/matrixrtc               @matrix-org/element-call-reviewers
 /spec/*/webrtc            @matrix-org/element-call-reviewers
 /spec/*/matrixrtc            @matrix-org/element-call-reviewers
+
+/src/crypto               @matrix-org/element-crypto-web-reviewers
+/src/rust-crypto          @matrix-org/element-crypto-web-reviewers
+/spec/integ/crypto        @matrix-org/element-crypto-web-reviewers
+/spec/unit/crypto.spec.ts @matrix-org/element-crypto-web-reviewers
+/spec/unit/crypto         @matrix-org/element-crypto-web-reviewers
+/spec/unit/rust-crypto    @matrix-org/element-crypto-web-reviewers

.github/actions/sign-release-tarball/action.yml

@@ -0,0 +1,28 @@
+name: Sign Release Tarball
+description: Generates signature for release tarball and uploads it as a release asset
+inputs:
+    gpg-fingerprint:
+        description: Fingerprint of the GPG key to use for signing the tarball.
+        required: true
+    upload-url:
+        description: GitHub release upload URL to upload the signature file to.
+        required: true
+runs:
+    using: composite
+    steps:
+        - name: Generate tarball signature
+          shell: bash
+          run: |
+              git -c tar.tar.gz.command='gzip -cn' archive --format=tar.gz --prefix="${REPO#*/}-${VERSION#v}/" -o "/tmp/${VERSION}.tar.gz" "${VERSION}"
+              gpg -u "$GPG_FINGERPRINT" --armor --output "${VERSION}.tar.gz.asc" --detach-sig "/tmp/${VERSION}.tar.gz"
+              rm "/tmp/${VERSION}.tar.gz"
+          env:
+              GPG_FINGERPRINT: ${{ inputs.gpg-fingerprint }}
+              REPO: ${{ github.repository }}
+
+        - name: Upload tarball signature
+          if: ${{ inputs.upload-url }}
+          uses: shogo82148/actions-upload-release-asset@dccd6d23e64fd6a746dce6814c0bde0a04886085 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ env.VERSION }}.tar.gz.asc

.github/actions/upload-release-assets/action.yml

@@ -0,0 +1,41 @@
+name: Upload release assets
+description: Uploads assets to an existing release and optionally signs them
+inputs:
+    gpg-fingerprint:
+        description: Fingerprint of the GPG key to use for signing the assets, if any.
+        required: false
+    upload-url:
+        description: GitHub release upload URL to upload the assets to.
+        required: true
+    asset-path:
+        description: |
+            The path to the asset you want to upload, if any. You can use glob patterns here.
+            Will be GPG signed and an `.asc` file included in the release artifacts if `gpg-fingerprint` is set.
+        required: true
+runs:
+    using: composite
+    steps:
+        - name: Sign assets
+          if: inputs.gpg-fingerprint
+          shell: bash
+          run: |
+              for FILE in $ASSET_PATH
+              do
+                  gpg -u "$GPG_FINGERPRINT" --armor --output "$FILE".asc --detach-sig "$FILE"
+              done
+          env:
+              GPG_FINGERPRINT: ${{ inputs.gpg-fingerprint }}
+              ASSET_PATH: ${{ inputs.asset-path }}
+
+        - name: Upload asset signatures
+          if: inputs.gpg-fingerprint
+          uses: shogo82148/actions-upload-release-asset@dccd6d23e64fd6a746dce6814c0bde0a04886085 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ inputs.asset-path }}.asc
+
+        - name: Upload assets
+          uses: shogo82148/actions-upload-release-asset@dccd6d23e64fd6a746dce6814c0bde0a04886085 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ inputs.asset-path }}

.github/release-drafter.yml

@@ -0,0 +1,35 @@
+name-template: "v$RESOLVED_VERSION"
+tag-template: "v$RESOLVED_VERSION"
+change-template: "* $TITLE ([#$NUMBER]($URL)). Contributed by @$AUTHOR."
+categories:
+    - title: "🚨 BREAKING CHANGES"
+      label: "X-Breaking-Change"
+    - title: "🦖 Deprecations"
+      label: "T-Deprecation"
+    - title: "✨ Features"
+      label: "T-Enhancement"
+    - title: "🐛 Bug Fixes"
+      label: "T-Defect"
+    - title: "🧰 Maintenance"
+      label: "Dependencies"
+      collapse-after: 5
+change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
+version-resolver:
+    major:
+        labels:
+            - "X-Breaking-Change"
+    default: minor
+exclude-labels:
+    - "T-Task"
+    - "X-Reverted"
+    - "backport staging"
+exclude-contributors:
+    - "RiotRobot"
+template: |
+    $CHANGES
+#no-changes-template: ""
+prerelease: true
+prerelease-identifier: rc
+include-pre-releases: false
+stable-ref: master
+staging-ref: staging

.github/workflows/docs-pr-netlify.yaml

@@ -11,18 +11,16 @@ jobs:
         if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
         runs-on: ubuntu-latest
         steps:
-            # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
-            # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
             - name: 📥 Download artifact
-              uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 # v2
+              uses: actions/download-artifact@v4
               with:
-                  workflow: static_analysis.yml
-                  run_id: ${{ github.event.workflow_run.id }}
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  run-id: ${{ github.event.workflow_run.id }}
                   name: docs
                   path: docs
 
             - name: 📤 Deploy to Netlify
-              uses: matrix-org/netlify-pr-preview@v2
+              uses: matrix-org/netlify-pr-preview@v3
               with:
                   path: docs
                   owner: ${{ github.event.workflow_run.head_repository.owner.login }}
@@ -32,3 +30,4 @@ jobs:
                   site_id: ${{ secrets.NETLIFY_SITE_ID }}
                   desc: Documentation preview
                   deployment_env: PR Documentation Preview
+        environment: PR Documentation Preview

.github/workflows/downstream-artifacts.yml

@@ -5,13 +5,9 @@ on:
 
     pull_request: {}
 
-    # For now at least, we don't run this or the cypress-tests against pushes
+    # For now at least, we don't run this or the downstream-end-to-end-tests against pushes
     # to develop or master.
     #
-    # Note that if we later choose to do so, we'll need to find a way to stop
-    # the results in Cypress Cloud from clobbering those from the 'develop'
-    # branch of matrix-react-sdk.
-    #
     #push:
     #    branches: [develop, master]
 concurrency:
@@ -20,7 +16,7 @@ concurrency:
 jobs:
     build-element-web:
         name: Build element-web
-        uses: matrix-org/matrix-react-sdk/.github/workflows/element-web.yaml@v3.82.0
+        uses: matrix-org/matrix-react-sdk/.github/workflows/element-web.yaml@v3.90.0
         with:
             matrix-js-sdk-sha: ${{ github.sha }}
             react-sdk-repository: matrix-org/matrix-react-sdk

.github/workflows/downstream-end-to-end-tests.yml

@@ -1,7 +1,7 @@
 # Triggers after the "Downstream artifacts" build has finished, to run the
-# cypress tests (with access to repo secrets)
+# matrix-react-sdk playwright tests (with access to repo secrets)
 
-name: matrix-react-sdk Cypress End to End Tests
+name: matrix-react-sdk End to End Tests
 on:
     workflow_run:
         workflows: ["Build downstream artifacts"]
@@ -13,36 +13,31 @@ concurrency:
     cancel-in-progress: ${{ github.event.workflow_run.event == 'pull_request' }}
 
 jobs:
-    cypress:
-        name: Cypress
-
-        # We only want to run the cypress tests on merge queue to prevent regressions
-        # from creeping in. They take a long time to run and consume 4 concurrent runners.
+    playwright:
+        name: Playwright
+        # We only want to run the playwright tests on merge queue to prevent regressions
+        # from creeping in. They take a long time to run and consume multiple concurrent runners.
         if: github.event.workflow_run.event == 'merge_group'
-
-        uses: matrix-org/matrix-react-sdk/.github/workflows/cypress.yaml@v3.83.0-rc.1
+        uses: matrix-org/matrix-react-sdk/.github/workflows/end-to-end-tests.yaml@develop
         permissions:
             actions: read
             issues: read
             statuses: write
             pull-requests: read
-        secrets:
-            # secrets are not automatically shared with called workflows, so share the cypress dashboard key, and the Kiwi login details
-            CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-            TCMS_USERNAME: ${{ secrets.TCMS_USERNAME }}
-            TCMS_PASSWORD: ${{ secrets.TCMS_PASSWORD }}
+            deployments: write
         with:
             react-sdk-repository: matrix-org/matrix-react-sdk
-            rust-crypto: true
+        secrets:
+            ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
 
-    # We want to make the cypress tests a required check for the merge queue.
+    # We want to make the Playwright tests a required check for the merge queue.
     #
-    # Unfortunately, github doesn't distinguish between "checks needed for branch
+    # Unfortunately, GitHub doesn't distinguish between "checks needed for branch
     # protection" (ie, the things that must pass before the PR will even be added
     # to the merge queue) and "checks needed in the merge queue". We just have to add
     # the check to the branch protection list.
     #
-    # Ergo, if we know we're not going to run the cypress tests, we need to add a
+    # Ergo, if we know we're not going to run the Playwright tests, we need to add a
     # passing status check manually.
     mark_skipped:
         if: github.event.workflow_run.event != 'merge_group'
@@ -50,10 +45,14 @@ jobs:
             statuses: write
         runs-on: ubuntu-latest
         steps:
-            - uses: Sibz/github-status-action@650dd1a882a76dbbbc4576fb5974b8d22f29847f # v1.1.6
+            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               with:
                   authToken: "${{ secrets.GITHUB_TOKEN }}"
                   state: success
-                  description: Cypress skipped
-                  context: "${{ github.workflow }} / cypress"
+                  description: Playwright skipped
+
+                  # Keep in step with the `context` that is updated by `Sibz/github-status-action`
+                  # in matrix-org/matrix-react-sdk/.github/workflows/end-to-end-tests.yaml.
+                  context: "${{ github.workflow }} / end-to-end-tests"
+
                   sha: "${{ github.event.workflow_run.head_sha }}"

.github/workflows/pull_request.yaml

@@ -27,7 +27,7 @@ jobs:
             pull-requests: read
         steps:
             - name: Add notice
-              uses: actions/github-script@v6
+              uses: actions/github-script@v7
               if: contains(github.event.pull_request.labels.*.name, 'X-Blocked')
               with:
                   script: |
@@ -49,7 +49,7 @@ jobs:
 
             - name: Add label
               if: ${{ steps.teams.outputs.isTeamMember == 'false' }}
-              uses: actions/github-script@v6
+              uses: actions/github-script@v7
               with:
                   script: |
                       github.rest.issues.addLabels({
@@ -68,7 +68,7 @@ jobs:
             github.event.pull_request.head.repo.full_name != github.repository
         steps:
             - name: Close pull request
-              uses: actions/github-script@v6
+              uses: actions/github-script@v7
               with:
                   script: |
                       github.rest.issues.createComment({

.github/workflows/release-drafter-workflow.yml

@@ -0,0 +1,86 @@
+name: Release Drafter
+on:
+    workflow_call:
+        inputs:
+            include-changes:
+                description: Project to include changelog entries from in this release.
+                type: string
+                required: false
+concurrency: release-drafter-action
+jobs:
+    draft:
+        runs-on: ubuntu-latest
+        steps:
+            - name: 🧮 Checkout code
+              uses: actions/checkout@v4
+              with:
+                  ref: staging
+                  fetch-depth: 0
+
+            - uses: actions/setup-node@v4
+              with:
+                  node-version-file: package.json
+                  cache: "yarn"
+
+            - name: Install Deps
+              run: "yarn install --frozen-lockfile"
+
+            - uses: t3chguy/release-drafter@105e541c2c3d857f032bd522c0764694758fabad
+              id: draft-release
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                  disable-autolabeler: true
+
+            - name: Get actions scripts
+              uses: actions/checkout@v4
+              with:
+                  repository: matrix-org/matrix-js-sdk
+                  persist-credentials: false
+                  path: .action-repo
+                  sparse-checkout: |
+                      .github/actions
+                      scripts/release
+
+            - name: Ingest upstream changes
+              if: inputs.include-changes
+              uses: actions/github-script@v7
+              env:
+                  RELEASE_ID: ${{ steps.release.outputs.id }}
+                  DEPENDENCY: ${{ inputs.include-changes }}
+                  VERSION: ${{ steps.draft-release.outputs.tag_name }}
+              with:
+                  retries: 3
+                  script: |
+                      const { RELEASE_ID: releaseId, DEPENDENCY, VERSION } = process.env;
+                      const { owner, repo } = context.repo;
+                      const script = require("./.action-repo/scripts/release/merge-release-notes.js");
+
+                      let deps = [];
+                      if (DEPENDENCY.includes("/")) {
+                          deps.push(DEPENDENCY.replace("$VERSION", VERSION))
+                      } else {
+                          const fromVersion = JSON.parse(await exec.exec("git show origin/master:package.json")).dependencies[DEPENDENCY];
+                          const toVersion = require("./package.json").dependencies[DEPENDENCY];
+
+                          if (toVersion.endsWith("#develop")) {
+                              core.warning(`${DEPENDENCY} will be kept at ${fromVersion}`, { title: "Develop dependency found" });
+                          } else {
+                              deps.push([DEPENDENCY, fromVersion, toVersion]);
+                          }
+                      }
+
+                      if (deps.length) {
+                          const notes = await script({
+                              github,
+                              releaseId,
+                              dependencies: deps,
+                          });
+
+                          await github.rest.repos.updateRelease({
+                              owner,
+                              repo,
+                              release_id: releaseId,
+                              body: notes,
+                          });
+                      }

.github/workflows/release-drafter.yml

@@ -0,0 +1,9 @@
+name: Release Drafter
+on:
+    push:
+        branches: [staging]
+    workflow_dispatch: {}
+concurrency: ${{ github.workflow }}
+jobs:
+    draft:
+        uses: matrix-org/matrix-js-sdk/.github/workflows/release-drafter-workflow.yml@develop

.github/workflows/release-gitflow.yml

@@ -0,0 +1,85 @@
+# Gitflow merge-back master->develop
+name: Merge master -> develop
+on:
+    push:
+        branches: [master]
+    workflow_call:
+        secrets:
+            ELEMENT_BOT_TOKEN:
+                required: true
+        inputs:
+            dependencies:
+                description: List of dependencies to reset.
+                type: string
+                required: false
+concurrency: ${{ github.workflow }}
+jobs:
+    merge:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  fetch-depth: 0
+
+            - name: Get actions scripts
+              uses: actions/checkout@v4
+              with:
+                  repository: matrix-org/matrix-js-sdk
+                  persist-credentials: false
+                  path: .action-repo
+                  sparse-checkout: |
+                      scripts/release
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+
+            - name: Install Deps
+              run: "yarn install --frozen-lockfile"
+
+            - name: Set up git
+              run: |
+                  git config --global user.email "releases@riot.im"
+                  git config --global user.name "RiotRobot"
+
+            - name: Merge to develop
+              run: |
+                  git checkout develop
+                  git merge -X ours master
+
+            - name: Run post-merge-master script to revert package.json fields
+              run: ./.action-repo/scripts/release/post-merge-master.sh
+
+            - name: Reset dependencies
+              if: inputs.dependencies
+              run: |
+                  while IFS= read -r PACKAGE; do
+                      [ -z "$PACKAGE" ] && continue
+
+                      CURRENT_VERSION=$(cat package.json | jq -r .dependencies[\"$PACKAGE\"])
+                      echo "Current $PACKAGE version is $CURRENT_VERSION"
+
+                      if [ "$CURRENT_VERSION" == "null" ]
+                      then
+                          echo "Unable to find $PACKAGE in package.json"
+                          exit 1
+                      fi
+
+                      if [ "$CURRENT_VERSION" == "develop" ]
+                      then
+                          echo "Not updating dependency $PACKAGE"
+                          continue
+                      fi
+
+                      echo "Resetting $1 to develop branch..."
+                      yarn add "github:matrix-org/$PACKAGE#develop"
+                      git add -u
+                      git commit -m "Reset $PACKAGE back to develop branch"
+                  done <<< "$DEPENDENCIES"
+              env:
+                  DEPENDENCIES: ${{ inputs.dependencies }}
+                  FINAL: ${{ inputs.final }}
+
+            - name: Push changes
+              run: git push origin develop

.github/workflows/release-make.yml

@@ -0,0 +1,331 @@
+name: Release Make
+on:
+    workflow_call:
+        secrets:
+            ELEMENT_BOT_TOKEN:
+                required: true
+            NPM_TOKEN:
+                required: false
+            GPG_PASSPHRASE:
+                required: false
+            GPG_PRIVATE_KEY:
+                required: false
+        inputs:
+            final:
+                description: Make final release
+                required: true
+                default: false
+                type: boolean
+            npm:
+                description: Publish to npm
+                type: boolean
+                default: false
+            downstreams:
+                description: List of github projects (owner/repo) which should have their dependency bumped to the newly released version (in JSON string array string syntax)
+                type: string
+                required: false
+            include-changes:
+                description: Project to include changelog entries from in this release.
+                type: string
+                required: false
+            gpg-fingerprint:
+                description: Fingerprint of the GPG key to use for signing the git tag and assets, if any.
+                type: string
+                required: false
+            asset-path:
+                description: |
+                    The path to the asset you want to upload, if any. You can use glob patterns here.
+                    Will be GPG signed and an `.asc` file included in the release artifacts if `gpg-fingerprint` is set.
+                type: string
+                required: false
+            expected-asset-count:
+                description: The number of expected assets, including signatures, excluding generated zip & tarball.
+                type: number
+                required: false
+jobs:
+    release:
+        name: Release
+        runs-on: ubuntu-latest
+        environment: Release
+        steps:
+            - name: Load GPG key
+              id: gpg
+              if: inputs.gpg-fingerprint
+              uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4 # v6
+              with:
+                  gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+                  passphrase: ${{ secrets.GPG_PASSPHRASE }}
+                  fingerprint: ${{ inputs.gpg-fingerprint }}
+
+            - name: Get draft release
+              id: release
+              uses: cardinalby/git-get-release-action@cedef2faf69cb7c55b285bad07688d04430b7ada # v1
+              env:
+                  GITHUB_TOKEN: ${{ github.token }}
+              with:
+                  draft: true
+                  latest: true
+
+            - uses: actions/checkout@v4
+              with:
+                  ref: staging
+                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  fetch-depth: 0
+
+            - name: Get actions scripts
+              uses: actions/checkout@v4
+              with:
+                  repository: matrix-org/matrix-js-sdk
+                  persist-credentials: false
+                  path: .action-repo
+                  sparse-checkout: |
+                      .github/actions
+                      scripts/release
+
+            - name: Prepare variables
+              id: prepare
+              run: |
+                  echo "VERSION=$VERSION" >> $GITHUB_ENV
+
+                  HAS_DIST=0
+                  jq -e .scripts.dist package.json >/dev/null 2>&1 && HAS_DIST=1
+                  echo "has-dist-script=$HAS_DIST" >> $GITHUB_OUTPUT
+              env:
+                  VERSION: ${{ steps.release.outputs.tag_name }}
+
+            - name: Finalise version
+              if: inputs.final
+              run: echo "VERSION=$(echo $VERSION | cut -d- -f1)" >> $GITHUB_ENV
+
+            - name: Check version number not in use
+              uses: actions/github-script@v7
+              with:
+                  script: |
+                      const { VERSION } = process.env;
+                      github.rest.repos.getReleaseByTag({
+                          owner: context.repo.owner,
+                          repo: context.repo.repo,
+                          tag: VERSION,
+                      }).then(() => {
+                          core.setFailed(`Version ${VERSION} already exists`);
+                      }).catch(() => {
+                        // This is fine, we expect there to not be any release with this version yet
+                      });
+
+            - name: Set up git
+              run: |
+                  git config --global user.email "releases@riot.im"
+                  git config --global user.name "RiotRobot"
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+
+            - name: Install dependencies
+              run: "yarn install --frozen-lockfile"
+
+            - name: Handle develop dependencies
+              run: |
+                  ret=0
+                  cat package.json | jq -r '.dependencies | to_entries | .[] | "\(.key) \(.value)"' | grep '#develop$' | while read -r dep ; do
+                      IFS=" "
+                      PACKAGE=${dep[0]}
+                      VERSION=${dep[1]}
+
+                      echo "::warning title=Develop dependency found::$DEPENDENCY will be kept at $VERSION"
+                      yarn upgrade "$PACKAGE@$VERSION" --exact
+                      git add -u
+                      git commit -m "Keep $PACKAGE at $VERSION"
+                  done
+
+            - name: Bump package.json version
+              run: yarn version --no-git-tag-version --new-version "${VERSION#v}"
+
+            - name: Add to CHANGELOG.md
+              if: inputs.final
+              run: |
+                  mv CHANGELOG.md CHANGELOG.md.old
+                  HEADER="Changes in [${VERSION#v}](https://github.com/${{ github.repository }}/releases/tag/$VERSION) ($(date '+%Y-%m-%d'))"
+
+                  {
+                      echo "$HEADER"
+                      printf '=%.0s' $(seq ${#HEADER})
+                      echo ""
+                      echo "$RELEASE_NOTES"
+                      echo ""
+                  } > CHANGELOG.md
+
+                  cat CHANGELOG.md.old >> CHANGELOG.md
+                  rm CHANGELOG.md.old
+                  git add CHANGELOG.md
+              env:
+                  RELEASE_NOTES: ${{ steps.release.outputs.body }}
+
+            - name: Run pre-release script to update package.json fields
+              run: |
+                  ./.action-repo/scripts/release/pre-release.sh
+                  git add package.json
+
+            - name: Commit changes
+              run: git commit -m "$VERSION"
+
+            - name: Build assets
+              if: steps.prepare.outputs.has-dist-script == '1'
+              run: DIST_VERSION="$VERSION" yarn dist
+
+            - name: Upload release assets & signatures
+              if: inputs.asset-path
+              uses: ./.action-repo/.github/actions/upload-release-assets
+              with:
+                  gpg-fingerprint: ${{ inputs.gpg-fingerprint }}
+                  upload-url: ${{ steps.release.outputs.upload_url }}
+                  asset-path: ${{ inputs.asset-path }}
+
+            - name: Create signed tag
+              if: inputs.gpg-fingerprint
+              run: |
+                  GIT_COMMITTER_EMAIL="$SIGNING_ID" GPG_TTY=$(tty) git tag -u "$SIGNING_ID" -m "Release $VERSION" "$VERSION"
+              env:
+                  SIGNING_ID: ${{ steps.gpg.outputs.email }}
+
+            - name: Generate & upload tarball signature
+              if: inputs.gpg-fingerprint
+              uses: ./.action-repo/.github/actions/sign-release-tarball
+              with:
+                  gpg-fingerprint: ${{ inputs.gpg-fingerprint }}
+                  upload-url: ${{ steps.release.outputs.upload_url }}
+
+            # We defer pushing changes until after the release assets are built,
+            # signed & uploaded to improve the atomicity of this action.
+            - name: Push changes to staging
+              run: |
+                  git push origin staging $TAG
+                  git reset --hard
+              env:
+                  TAG: ${{ inputs.gpg-fingerprint && env.VERSION || '' }}
+
+            - name: Validate tarball signature
+              if: inputs.gpg-fingerprint
+              run: |
+                  wget https://github.com/$GITHUB_REPOSITORY/archive/refs/tags/$VERSION.tar.gz
+                  gpg --verify "$VERSION.tar.gz.asc" "$VERSION.tar.gz"
+
+            - name: Validate release has expected assets
+              if: inputs.expected-asset-count
+              uses: actions/github-script@v7
+              env:
+                  RELEASE_ID: ${{ steps.release.outputs.id }}
+                  EXPECTED_ASSET_COUNT: ${{ inputs.expected-asset-count }}
+              with:
+                  retries: 3
+                  script: |
+                      const { RELEASE_ID: release_id, EXPECTED_ASSET_COUNT } = process.env;
+                      const { owner, repo } = context.repo;
+
+                      const { data: release } = await github.rest.repos.getRelease({
+                          owner,
+                          repo,
+                          release_id,
+                      });
+
+                      if (release.assets.length !== parseInt(EXPECTED_ASSET_COUNT, 10)) {
+                          core.setFailed(`Found ${release.assets.length} assets but expected ${EXPECTED_ASSET_COUNT}`);
+                      }
+
+            - name: Merge to master
+              if: inputs.final
+              run: |
+                  git checkout master
+                  git merge -X theirs staging
+                  git push origin master
+
+            - name: Publish release
+              uses: actions/github-script@v7
+              env:
+                  RELEASE_ID: ${{ steps.release.outputs.id }}
+                  FINAL: ${{ inputs.final }}
+              with:
+                  retries: 3
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  script: |
+                      const { RELEASE_ID: release_id, RELEASE_NOTES, VERSION, FINAL } = process.env;
+                      const { owner, repo } = context.repo;
+
+                      const opts = {
+                          owner,
+                          repo,
+                          release_id,
+                          tag_name: VERSION,
+                          name: VERSION,
+                          draft: false,
+                          body: RELEASE_NOTES,
+                      };
+
+                      if (FINAL == "true") {
+                          opts.prerelease = false;
+                          opts.make_latest = true;
+                      }
+
+                      github.rest.repos.updateRelease(opts);
+
+    npm:
+        name: Publish to npm
+        needs: release
+        if: inputs.npm
+        uses: matrix-org/matrix-js-sdk/.github/workflows/release-npm.yml@develop
+        secrets:
+            NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+    post-release:
+        name: Post release steps
+        needs: release
+        runs-on: ubuntu-latest
+        steps:
+            - id: repository
+              run: echo "REPO=${GITHUB_REPOSITORY#*/}" >> $GITHUB_OUTPUT
+
+            - name: Advance release blocker labels
+              uses: garganshu/github-label-updater@3770d15ebfed2fe2cb06a241047bc340f774a7d1 # v1.0.0
+              with:
+                  owner: ${{ github.repository_owner }}
+                  repo: ${{ steps.repository.outputs.REPO }}
+                  token: ${{ secrets.GITHUB_TOKEN }}
+                  filter-labels: X-Upcoming-Release-Blocker
+                  remove-labels: X-Upcoming-Release-Blocker
+                  add-labels: X-Release-Blocker
+
+            - name: Wait for master->develop gitflow merge
+              if: inputs.final
+              uses: t3chguy/wait-on-check-action@18541021811b56544d90e0f073401c2b99e249d6 # fork
+              with:
+                  ref: master
+                  repo-token: ${{ secrets.GITHUB_TOKEN }}
+                  wait-interval: 10
+                  check-name: merge
+                  allowed-conclusions: success
+
+    bump-downstreams:
+        name: Update npm dependency in downstream projects
+        needs: npm
+        runs-on: ubuntu-latest
+        if: inputs.downstreams
+        strategy:
+            matrix:
+                repo: ${{ fromJSON(inputs.downstreams) }}
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  repository: ${{ matrix.repo }}
+                  ref: staging
+                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+
+            - name: Bump dependency
+              env:
+                  DEPENDENCY: ${{ needs.npm.outputs.id }}
+              run: |
+                  git config --global user.email "releases@riot.im"
+                  git config --global user.name "RiotRobot"
+                  yarn upgrade "$DEPENDENCY" --exact
+                  git add package.json yarn.lock
+                  git commit -am"Upgrade dependency to $DEPENDENCY"
+                  git push origin staging

.github/workflows/release-npm.yml

@@ -1,4 +1,3 @@
-# Must only be called from `release#published` triggers
 name: Publish to npm
 on:
     workflow_call:
@@ -9,12 +8,16 @@ jobs:
     npm:
         name: Publish to npm
         runs-on: ubuntu-latest
+        outputs:
+            id: ${{ steps.npm-publish.outputs.id }}
         steps:
             - name: 🧮 Checkout code
               uses: actions/checkout@v4
+              with:
+                  ref: staging
 
             - name: 🔧 Yarn cache
-              uses: actions/setup-node@v3
+              uses: actions/setup-node@v4
               with:
                   cache: "yarn"
                   registry-url: "https://registry.npmjs.org"
@@ -24,7 +27,7 @@ jobs:
 
             - name: 🚀 Publish to npm
               id: npm-publish
-              uses: JS-DevTools/npm-publish@fe72237be0920f7a0cafd6a966c9b929c9466e9b # v2.2.2
+              uses: JS-DevTools/npm-publish@4b07b26a2f6e0a51846e1870223e545bae91c552 # v3.0.1
               with:
                   token: ${{ secrets.NPM_TOKEN }}
                   access: public
@@ -32,7 +35,7 @@ jobs:
                   ignore-scripts: false
 
             - name: 🎖️ Add `latest` dist-tag to final releases
-              if: github.event.release.prerelease == false && steps.npm-publish.outputs.id
+              if: steps.npm-publish.outputs.id && !contains(steps.npm-publish.outputs.id, '-rc.')
               run: npm dist-tag add "$release" latest
               env:
                   NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/release.yml

@@ -1,11 +1,39 @@
 name: Release Process
 on:
-    release:
-        types: [published]
-concurrency: ${{ github.workflow }}-${{ github.ref }}
+    workflow_dispatch:
+        inputs:
+            mode:
+                description: What type of release
+                required: true
+                default: rc
+                type: choice
+                options:
+                    - rc
+                    - final
+            docs:
+                description: Publish docs
+                required: true
+                type: boolean
+                default: true
+            npm:
+                description: Publish to npm
+                required: true
+                type: boolean
+                default: true
+concurrency: ${{ github.workflow }}
 jobs:
-    jsdoc:
+    release:
+        uses: matrix-org/matrix-js-sdk/.github/workflows/release-make.yml@develop
+        secrets: inherit
+        with:
+            final: ${{ inputs.mode == 'final' }}
+            npm: ${{ inputs.npm }}
+            downstreams: '["matrix-org/matrix-react-sdk", "element-hq/element-web"]'
+
+    docs:
         name: Publish Documentation
+        needs: release
+        if: inputs.docs
         runs-on: ubuntu-latest
         steps:
             - name: 🧮 Checkout code
@@ -18,7 +46,7 @@ jobs:
                   path: _docs
 
             - name: 🔧 Yarn cache
-              uses: actions/setup-node@v3
+              uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
@@ -36,17 +64,14 @@ jobs:
                   yarn gendoc
                   symlinks -rc _docs
 
-            - name: 🚀 Deploy
+            - name: 🔨 Set up git
               run: |
                   git config --global user.email "releases@riot.im"
                   git config --global user.name "RiotRobot"
+
+            - name: 🚀 Deploy
+              run: |
                   git add . --all
                   git commit -m "Update docs"
                   git push
               working-directory: _docs
-
-    npm:
-        name: Publish
-        uses: matrix-org/matrix-js-sdk/.github/workflows/release-npm.yml@develop
-        secrets:
-            NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/sonarcloud.yml

@@ -5,7 +5,13 @@ on:
         secrets:
             SONAR_TOKEN:
                 required: true
+            ELEMENT_BOT_TOKEN:
+                required: true
         inputs:
+            sharded:
+                type: boolean
+                required: false
+                description: "Whether to combine multiple LCOV and jest-sonar-report files in coverage artifact"
             extra_args:
                 type: string
                 required: false
@@ -13,11 +19,13 @@ on:
 jobs:
     sonarqube:
         runs-on: ubuntu-latest
-        if: github.event.workflow_run.conclusion == 'success'
+        if: |
+            github.event.workflow_run.conclusion == 'success' &&
+            github.event.workflow_run.event != 'merge_group'
         steps:
             # We create the status here and then update it to success/failure in the `report` stage
-            # This provides an easy link to this workflow_run from the PR before Cypress is done.
-            - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+            # This provides an easy link to this workflow_run from the PR before Sonarcloud is done.
+            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               with:
                   authToken: ${{ secrets.GITHUB_TOKEN }}
                   state: pending
@@ -25,24 +33,57 @@ jobs:
                   sha: ${{ github.event.workflow_run.head_sha }}
                   target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
 
+            - name: "🧮 Checkout code"
+              uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+              with:
+                  repository: ${{ github.event.workflow_run.head_repository.full_name }}
+                  ref: ${{ github.event.workflow_run.head_branch }} # checkout commit that triggered this workflow
+                  fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
+
+            - name: 📥 Download artifact
+              uses: actions/download-artifact@v4
+              if: ${{ !inputs.sharded }}
+              with:
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  run-id: ${{ github.event.workflow_run.id }}
+                  name: coverage
+                  path: coverage
+            - name: 📥 Download sharded artifacts
+              uses: actions/download-artifact@v4
+              if: inputs.sharded
+              with:
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  run-id: ${{ github.event.workflow_run.id }}
+                  pattern: coverage-*
+                  path: coverage
+                  merge-multiple: true
+
+            - id: extra_args
+              run: |
+                  coverage=$(find coverage -type f -name '*lcov.info' | tr '\n' ',' | sed 's/,$//g')
+                  echo "reportPaths=$coverage" >> $GITHUB_OUTPUT
+                  reports=$(find coverage -type f -name 'jest-sonar-report*.xml' | tr '\n' ',' | sed 's/,$//g')
+                  echo "testExecutionReportPaths=$reports" >> $GITHUB_OUTPUT
+
             - name: "🩻 SonarCloud Scan"
               id: sonarcloud
-              uses: matrix-org/sonarcloud-workflow-action@v2.6
+              uses: matrix-org/sonarcloud-workflow-action@v2.7
               # workflow_run fails report against the develop commit always, we don't want that for PRs
               continue-on-error: ${{ github.event.workflow_run.head_branch != 'develop' }}
               with:
+                  skip_checkout: true
                   repository: ${{ github.event.workflow_run.head_repository.full_name }}
                   is_pr: ${{ github.event.workflow_run.event == 'pull_request' }}
                   version_cmd: "cat package.json | jq -r .version"
                   branch: ${{ github.event.workflow_run.head_branch }}
                   revision: ${{ github.event.workflow_run.head_sha }}
                   token: ${{ secrets.SONAR_TOKEN }}
-                  coverage_run_id: ${{ github.event.workflow_run.id }}
-                  coverage_workflow_name: tests.yml
-                  coverage_extract_path: coverage
-                  extra_args: ${{ inputs.extra_args }}
+                  extra_args: |
+                      ${{ inputs.extra_args }}
+                      -Dsonar.javascript.lcov.reportPaths=${{ steps.extra_args.outputs.reportPaths }}
+                      -Dsonar.testExecutionReportPaths=${{ steps.extra_args.outputs.testExecutionReportPaths }}
 
-            - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               if: always()
               with:
                   authToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/sonarqube.yml

@@ -8,38 +8,12 @@ concurrency:
     group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch }}
     cancel-in-progress: true
 jobs:
-    # This is a workaround for https://github.com/SonarSource/SonarJS/issues/578
-    prepare:
-        name: Prepare
-        if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event != 'merge_group'
-        runs-on: ubuntu-latest
-        outputs:
-            reportPaths: ${{ steps.extra_args.outputs.reportPaths }}
-            testExecutionReportPaths: ${{ steps.extra_args.outputs.testExecutionReportPaths }}
-        steps:
-            # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
-            # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
-            - name: 📥 Download artifact
-              uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 # v2
-              with:
-                  workflow: tests.yaml
-                  run_id: ${{ github.event.workflow_run.id }}
-                  name: coverage
-                  path: coverage
-
-            - id: extra_args
-              run: |
-                  coverage=$(find coverage -type f -name '*lcov.info' | tr '\n' ',' | sed 's/,$//g')
-                  echo "reportPaths=$coverage" >> $GITHUB_OUTPUT
-                  reports=$(find coverage -type f -name 'jest-sonar-report*.xml' | tr '\n' ',' | sed 's/,$//g')
-                  echo "testExecutionReportPaths=$reports" >> $GITHUB_OUTPUT
-
     sonarqube:
         name: 🩻 SonarQube
         if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event != 'merge_group'
-        needs: prepare
         uses: matrix-org/matrix-js-sdk/.github/workflows/sonarcloud.yml@develop
         secrets:
             SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+            ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
         with:
-            extra_args: -Dsonar.javascript.lcov.reportPaths=${{ needs.prepare.outputs.reportPaths }} -Dsonar.testExecutionReportPaths=${{ needs.prepare.outputs.testExecutionReportPaths }}
+            sharded: true

.github/workflows/static_analysis.yml

@@ -15,7 +15,7 @@ jobs:
         steps:
             - uses: actions/checkout@v4
 
-            - uses: actions/setup-node@v3
+            - uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
@@ -41,7 +41,7 @@ jobs:
         steps:
             - uses: actions/checkout@v4
 
-            - uses: actions/setup-node@v3
+            - uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
@@ -51,13 +51,29 @@ jobs:
             - name: Run Linter
               run: "yarn run lint:js"
 
+    workflow_lint:
+        name: "Workflow Lint"
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+
+            - name: Install Deps
+              run: "yarn install --frozen-lockfile"
+
+            - name: Run Linter
+              run: "yarn lint:workflows"
+
     docs:
         name: "JSDoc Checker"
         runs-on: ubuntu-latest
         steps:
             - uses: actions/checkout@v4
 
-            - uses: actions/setup-node@v3
+            - uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
@@ -75,7 +91,7 @@ jobs:
                   find _docs -mindepth 1 -maxdepth 1 -type d -execdir mv {} stable \; -quit
 
             - name: Upload Artifact
-              uses: actions/upload-artifact@v3
+              uses: actions/upload-artifact@v4
               with:
                   name: docs
                   path: _docs

.github/workflows/tests.yml

@@ -18,14 +18,14 @@ jobs:
         strategy:
             matrix:
                 specs: [integ, unit]
-                node: [18, "*"]
+                node: [18, "lts/*", 21]
         steps:
             - name: Checkout code
               uses: actions/checkout@v4
 
             - name: Setup Node
               id: setupNode
-              uses: actions/setup-node@v3
+              uses: actions/setup-node@v4
               with:
                   cache: "yarn"
                   node-version: ${{ matrix.node }}
@@ -52,13 +52,13 @@ jobs:
 
             - name: Move coverage files into place
               if: env.ENABLE_COVERAGE == 'true'
-              run: mv coverage/lcov.info coverage/${{ steps.setupNode.output.node-version }}-${{ matrix.specs }}.lcov.info
+              run: mv coverage/lcov.info coverage/${{ steps.setupNode.outputs.node-version }}-${{ matrix.specs }}.lcov.info
 
             - name: Upload Artifact
               if: env.ENABLE_COVERAGE == 'true'
-              uses: actions/upload-artifact@v3
+              uses: actions/upload-artifact@v4
               with:
-                  name: coverage
+                  name: coverage-${{ matrix.specs }}-${{ matrix.node == 'lts/*' && 'lts' || matrix.node }}
                   path: |
                       coverage
                       !coverage/lcov-report
@@ -82,7 +82,7 @@ jobs:
         steps:
             - name: Skip SonarCloud on merge queues
               if: env.ENABLE_COVERAGE == 'false'
-              uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+              uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               with:
                   authToken: ${{ secrets.GITHUB_TOKEN }}
                   state: success

.github/workflows/triage-labelled.yml

@@ -0,0 +1,11 @@
+name: Move labelled issues to correct projects
+
+on:
+    issues:
+        types: [labeled]
+
+jobs:
+    call-triage-labelled:
+        uses: vector-im/element-web/.github/workflows/triage-labelled.yml@develop
+        secrets:
+            ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}

.prettierignore

@@ -25,5 +25,6 @@ out
 # This file is owned, parsed, and generated by allchange, which doesn't comply with prettier
 /CHANGELOG.md
 
-# This file is also autogenerated
+# These files are also autogenerated
 /spec/test-utils/test-data/index.ts
+/spec/test-utils/test_indexeddb_cryptostore_dump/dump.json

CHANGELOG.md

@@ -1,8 +1,96 @@
+Changes in [31.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.2.0) (2024-01-31)
+==================================================================================================
+## ✨ Features
+
+* Emit events during migration from libolm ([#3982](https://github.com/matrix-org/matrix-js-sdk/pull/3982)). Contributed by @richvdh.
+* Support for migration from from libolm ([#3978](https://github.com/matrix-org/matrix-js-sdk/pull/3978)). Contributed by @richvdh.
+
+## 🐛 Bug Fixes
+
+* ElementR | backup: call expensive `roomKeyCounts` less often ([#4015](https://github.com/matrix-org/matrix-js-sdk/pull/4015)). Contributed by @BillCarsonFr.
+* Decrypt and Import full backups in chunk with progress ([#4005](https://github.com/matrix-org/matrix-js-sdk/pull/4005)). Contributed by @BillCarsonFr.
+* Fix new threads not appearing. ([#4009](https://github.com/matrix-org/matrix-js-sdk/pull/4009)). Contributed by @dbkr.
+
+
+Changes in [31.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.1.0) (2024-01-19)
+==================================================================================================
+## ✨ Features
+
+* Broaden spec version support ([#4016](https://github.com/matrix-org/matrix-js-sdk/pull/4016)). Contributed by @RiotRobot.
+
+
+Changes in [31.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.0.0) (2024-01-16)
+==================================================================================================
+## 🚨 BREAKING CHANGES
+
+* Bump minimum spec version to v1.5 ([#3970](https://github.com/matrix-org/matrix-js-sdk/pull/3970)). Contributed by @richvdh.
+
+## ✨ Features
+
+* Bump minimum spec version to v1.5 ([#3970](https://github.com/matrix-org/matrix-js-sdk/pull/3970)). Contributed by @richvdh.
+* Send authenticated /versions request ([#3968](https://github.com/matrix-org/matrix-js-sdk/pull/3968)). Contributed by @dbkr.
+
+## 🐛 Bug Fixes
+
+* Revert "Bump matrix-sdk-crypto-wasm to 3.6.0" ([#3991](https://github.com/matrix-org/matrix-js-sdk/pull/3991)). Contributed by @andybalaam.
+* #22606 Fix "Remove" button  to users without "m.room.redaction" ([#3981](https://github.com/matrix-org/matrix-js-sdk/pull/3981)). Contributed by @rashmitpankhania.
+* ElementR: Ensure Encryption order per room ([#3973](https://github.com/matrix-org/matrix-js-sdk/pull/3973)). Contributed by @BillCarsonFr.
+* Element-R: fix `bootstrapSecretStorage` not resetting key backup when requested ([#3976](https://github.com/matrix-org/matrix-js-sdk/pull/3976)). Contributed by @uhoreg.
+
+
+Changes in [30.3.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.3.0) (2023-12-19)
+==================================================================================================
+## ✨ Features
+
+* Element-R: disable sending room key requests ([#3939](https://github.com/matrix-org/matrix-js-sdk/pull/3939)). Contributed by @richvdh.
+
+## 🐛 Bug Fixes
+
+* Fix notifications appearing for old events ([#3946](https://github.com/matrix-org/matrix-js-sdk/pull/3946)). Contributed by @dbkr.
+* Don't back up keys that we got from backup ([#3934](https://github.com/matrix-org/matrix-js-sdk/pull/3934)). Contributed by @uhoreg.
+* Fix upload with empty Content-Type ([#3918](https://github.com/matrix-org/matrix-js-sdk/pull/3918)). Contributed by @JakubOnderka.
+* Prevent phantom notifications from events not in a room's timeline ([#3942](https://github.com/matrix-org/matrix-js-sdk/pull/3942)). Contributed by @dbkr.
+
+
+Changes in [30.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.2.0) (2023-12-05)
+==================================================================================================
+## ✨ Features
+
+* Only await key query after lazy members resolved ([#3902](https://github.com/matrix-org/matrix-js-sdk/pull/3902)). Contributed by @BillCarsonFr.
+
+## 🐛 Bug Fixes
+
+* Rewrite receipt-handling code ([#3901](https://github.com/matrix-org/matrix-js-sdk/pull/3901)). Contributed by @andybalaam.
+* Explicitly free some Rust-side objects ([#3911](https://github.com/matrix-org/matrix-js-sdk/pull/3911)). Contributed by @richvdh.
+* Fix type for TimestampToEventResponse.origin\_server\_ts ([#3906](https://github.com/matrix-org/matrix-js-sdk/pull/3906)). Contributed by @Half-Shot.
+
+
+Changes in [30.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.1.0) (2023-11-21)
+==================================================================================================
+## ✨ Features
+
+* Rotate per-participant keys when a member leaves ([#3833](https://github.com/matrix-org/matrix-js-sdk/pull/3833)). Contributed by @dbkr.
+* Add E2EE for embedded mode of Element Call ([#3667](https://github.com/matrix-org/matrix-js-sdk/pull/3667)). Contributed by @SimonBrandner.
+
+## 🐛 Bug Fixes
+
+* Shorten TimelineWindow when an event is removed ([#3862](https://github.com/matrix-org/matrix-js-sdk/pull/3862)). Contributed by @andybalaam.
+* Ignore receipts pointing at missing or invalid events ([#3817](https://github.com/matrix-org/matrix-js-sdk/pull/3817)). Contributed by @andybalaam.
+* Fix members being loaded from server on initial sync (defeating lazy loading) ([#3830](https://github.com/matrix-org/matrix-js-sdk/pull/3830)). Contributed by @BillCarsonFr.
+
+
+Changes in [30.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.0.1) (2023-11-13)
+==================================================================================================
+
+## 🐛 Bug Fixes
+ * Ensure `setUserCreator` is called when a store is assigned ([\#3867](https://github.com/matrix-org/matrix-js-sdk/pull/3867)). Fixes vector-im/element-web#26520. Contributed by @MidhunSureshR.
+
 Changes in [30.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.0.0) (2023-11-07)
 ==================================================================================================
 
 ## 🚨 BREAKING CHANGES
  * Refactor & make base64 functions browser-safe ([\#3818](https://github.com/matrix-org/matrix-js-sdk/pull/3818)).
+ * `IndexedDBStore.startup()` must be called after using it on `sdk.createClient` now.
 
 ## 🦖 Deprecations
  * Deprecate `MatrixEvent.toJSON` ([\#3801](https://github.com/matrix-org/matrix-js-sdk/pull/3801)).

README.md

@@ -58,7 +58,7 @@ await client.startClient({ initialSyncLimit: 10 });
 You can perform a call to `/sync` to get the current state of the client:
 
 ```javascript
-client.once("sync", function (state, prevState, res) {
+client.once(ClientEvent.sync, function (state, prevState, res) {
     if (state === "PREPARED") {
         console.log("prepared");
     } else {
@@ -83,7 +83,7 @@ client.sendEvent("roomId", "m.room.message", content, "", (err, res) => {
 To listen for message events:
 
 ```javascript
-client.on("Room.timeline", function (event, room, toStartOfTimeline) {
+client.on(RoomEvent.Timeline, function (event, room, toStartOfTimeline) {
     if (event.getType() !== "m.room.message") {
         return; // only use messages
     }
@@ -144,12 +144,12 @@ are updated.
 
 ```javascript
 // Listen for low-level MatrixEvents
-client.on("event", function (event) {
+client.on(ClientEvent.Event, function (event) {
     console.log(event.getType());
 });
 
 // Listen for typing changes
-client.on("RoomMember.typing", function (event, member) {
+client.on(RoomMemberEvent.Typing, function (event, member) {
     if (member.typing) {
         console.log(member.name + " is typing...");
     } else {
@@ -211,7 +211,7 @@ const matrixClient = sdk.createClient({
 ### Automatically join rooms when invited
 
 ```javascript
-matrixClient.on("RoomMember.membership", function (event, member) {
+matrixClient.on(RoomMemberEvent.Membership, function (event, member) {
     if (member.membership === "invite" && member.userId === myUserId) {
         matrixClient.joinRoom(member.roomId).then(function () {
             console.log("Auto-joined %s", member.roomId);
@@ -225,7 +225,7 @@ matrixClient.startClient();
 ### Print out messages for all rooms
 
 ```javascript
-matrixClient.on("Room.timeline", function (event, room, toStartOfTimeline) {
+matrixClient.on(RoomEvent.Timeline, function (event, room, toStartOfTimeline) {
     if (toStartOfTimeline) {
         return; // don't print paginated results
     }
@@ -257,7 +257,7 @@ Output:
 ### Print out membership lists whenever they are changed
 
 ```javascript
-matrixClient.on("RoomState.members", function (event, state, member) {
+matrixClient.on(RoomStateEvent.Members, function (event, state, member) {
     const room = matrixClient.getRoom(state.roomId);
     if (!room) {
         return;

package.json

@@ -1,12 +1,12 @@
 {
     "name": "matrix-js-sdk",
-    "version": "30.0.0",
+    "version": "31.3.0-rc.4",
     "description": "Matrix Client-Server SDK for Javascript",
     "engines": {
         "node": ">=18.0.0"
     },
     "scripts": {
-        "prepublishOnly": "yarn build",
+        "prepack": "yarn build",
         "start": "echo THIS IS FOR LEGACY PURPOSES ONLY. && babel src -w -s -d lib --verbose --extensions \".ts,.js\"",
         "clean": "rimraf lib",
         "build": "yarn build:dev",
@@ -14,10 +14,11 @@
         "build:types": "tsc -p tsconfig-build.json --emitDeclarationOnly",
         "build:compile": "babel -d lib --verbose --extensions \".ts,.js\" src",
         "gendoc": "typedoc",
-        "lint": "yarn lint:types && yarn lint:js",
+        "lint": "yarn lint:types && yarn lint:js && yarn lint:workflows",
         "lint:js": "eslint --max-warnings 0 src spec && prettier --check .",
-        "lint:js-fix": "prettier --loglevel=warn --write . && eslint --fix src spec",
+        "lint:js-fix": "prettier --log-level=warn --write . && eslint --fix src spec",
         "lint:types": "tsc --noEmit",
+        "lint:workflows": "find .github/workflows -type f \\( -iname '*.yaml' -o -iname '*.yml' \\) | xargs -I {} sh -c 'echo \"Linting {}\"; action-validator \"{}\"'",
         "test": "jest",
         "test:watch": "jest --watch",
         "coverage": "yarn test --coverage"
@@ -51,7 +52,7 @@
     ],
     "dependencies": {
         "@babel/runtime": "^7.12.5",
-        "@matrix-org/matrix-sdk-crypto-wasm": "^2.2.0",
+        "@matrix-org/matrix-sdk-crypto-wasm": "^4.3.0",
         "another-json": "^0.2.0",
         "bs58": "^5.0.0",
         "content-type": "^1.0.4",
@@ -66,6 +67,8 @@
         "uuid": "9"
     },
     "devDependencies": {
+        "@action-validator/cli": "^0.5.3",
+        "@action-validator/core": "^0.5.3",
         "@babel/cli": "^7.12.10",
         "@babel/core": "^7.12.10",
         "@babel/eslint-parser": "^7.12.10",
@@ -88,25 +91,24 @@
         "@types/node": "18",
         "@types/sdp-transform": "^2.4.5",
         "@types/uuid": "9",
-        "@typescript-eslint/eslint-plugin": "^5.45.0",
-        "@typescript-eslint/parser": "^5.45.0",
-        "allchange": "^1.0.6",
+        "@typescript-eslint/eslint-plugin": "^6.0.0",
+        "@typescript-eslint/parser": "^6.0.0",
+        "allchange": "^1.3.0",
         "babel-jest": "^29.0.0",
-        "babelify": "^10.0.0",
         "debug": "^4.3.4",
         "domexception": "^4.0.0",
-        "eslint": "8.51.0",
+        "eslint": "8.56.0",
         "eslint-config-google": "^0.14.0",
         "eslint-config-prettier": "^9.0.0",
         "eslint-import-resolver-typescript": "^3.5.1",
         "eslint-plugin-import": "^2.26.0",
         "eslint-plugin-jest": "^27.1.6",
-        "eslint-plugin-jsdoc": "^46.0.0",
+        "eslint-plugin-jsdoc": "^48.0.0",
         "eslint-plugin-matrix-org": "^1.0.0",
         "eslint-plugin-tsdoc": "^0.2.17",
-        "eslint-plugin-unicorn": "^48.0.0",
-        "exorcist": "^2.0.0",
-        "fake-indexeddb": "^5.0.0",
+        "eslint-plugin-unicorn": "^50.0.0",
+        "fake-indexeddb": "^5.0.2",
+        "fetch-mock": "9.11.0",
         "fetch-mock-jest": "^1.5.1",
         "husky": "^8.0.3",
         "jest": "^29.0.0",
@@ -115,11 +117,9 @@
         "jest-mock": "^29.0.0",
         "lint-staged": "^15.0.2",
         "matrix-mock-request": "^2.5.0",
-        "prettier": "2.8.8",
+        "prettier": "3.2.4",
         "rimraf": "^5.0.0",
-        "terser": "^5.5.1",
         "ts-node": "^10.9.1",
-        "tsify": "^5.0.2",
         "typedoc": "^0.24.0",
         "typedoc-plugin-coverage": "^2.1.0",
         "typedoc-plugin-mdn-links": "^3.0.3",

post-release.sh

@@ -10,28 +10,6 @@ set -e
 jq --version > /dev/null || (echo "jq is required: please install it"; kill $$)
 
 if [ "$(git branch -lr | grep origin/develop -c)" -ge 1 ]; then
-    # When merging to develop, we need revert the `main` and `typings` fields if we adjusted them previously.
-    for i in main typings browser
-    do
-        # If a `lib` prefixed value is present, it means we adjusted the field
-        # earlier at publish time, so we should revert it now.
-        if [ "$(jq -r ".matrix_lib_$i" package.json)" != "null" ]; then
-            # If there's a `src` prefixed value, use that, otherwise delete.
-            # This is used to delete the `typings` field and reset `main` back
-            # to the TypeScript source.
-            src_value=$(jq -r ".matrix_src_$i" package.json)
-            if [ "$src_value" != "null" ]; then
-                jq ".$i = .matrix_src_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
-            else
-                jq "del(.$i)" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
-            fi
-        fi
-    done
-
-    if [ -n "$(git ls-files --modified package.json)" ]; then
-        echo "Committing develop package.json"
-        git commit package.json -m "Resetting package fields for development"
-    fi
-
+    "$(dirname "$0")/scripts/release/post-merge-master.sh"
     git push origin develop
 fi

release.sh

@@ -175,18 +175,7 @@ echo "yarn version"
 # manually commit the result.
 yarn version --no-git-tag-version --new-version "$release"
 
-# For the published and dist versions of the package, we copy the
-# `matrix_lib_main` and `matrix_lib_typings` fields to `main` and `typings` (if
-# they exist). This small bit of gymnastics allows us to use the TypeScript
-# source directly for development without needing to build before linting or
-# testing.
-for i in main typings browser
-do
-    lib_value=$(jq -r ".matrix_lib_$i" package.json)
-    if [ "$lib_value" != "null" ]; then
-        jq ".$i = .matrix_lib_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
-    fi
-done
+"$(dirname "$0")/scripts/release/pre-release.sh"
 
 # commit yarn.lock if it exists, is versioned, and is modified
 if [[ -f yarn.lock && $(git status --porcelain yarn.lock | grep '^ M') ]];

scripts/release/merge-release-notes.js

@@ -0,0 +1,135 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+
+// Dependency can be the name of an entry in package.json, in which case the owner, repo & version will be looked up in its own package.json
+// Or it can be a string in the form owner/repo@tag
+// Or it can be a tuple of dependency, from version, to version, in which case a list of releases in that range (to inclusive) will be returned
+async function getReleases(github, dependency) {
+    if (Array.isArray(dependency)) {
+        const [dep, fromVersion, toVersion] = dependency;
+        const upstreamPackageJson = getDependencyPackageJson(dep);
+        const [owner, repo] = upstreamPackageJson.repository.url.split("/").slice(-2);
+
+        const response = await github.rest.repos.listReleases({
+            owner,
+            repo,
+            per_page: 100,
+        });
+        const releases = response.data.filter((release) => !release.draft && !release.prerelease);
+
+        const fromVersionIndex = releases.findIndex((release) => release.tag_name === `v${fromVersion}`);
+        const toVersionIndex = releases.findIndex((release) => release.tag_name === `v${toVersion}`);
+
+        return releases.slice(toVersionIndex, fromVersionIndex);
+    }
+
+    return [await getRelease(github, dependency)];
+}
+
+async function getRelease(github, dependency) {
+    let owner;
+    let repo;
+    let tag;
+    if (dependency.includes("/") && dependency.includes("@")) {
+        owner = dependency.split("/")[0];
+        repo = dependency.split("/")[1].split("@")[0];
+        tag = dependency.split("@")[1];
+    } else {
+        const upstreamPackageJson = getDependencyPackageJson(dependency);
+        [owner, repo] = upstreamPackageJson.repository.url.split("/").slice(-2);
+        tag = `v${upstreamPackageJson.version}`;
+    }
+
+    const response = await github.rest.repos.getReleaseByTag({
+        owner,
+        repo,
+        tag,
+    });
+    return response.data;
+}
+
+function getDependencyPackageJson(dependency) {
+    return JSON.parse(fs.readFileSync(`./node_modules/${dependency}/package.json`, "utf8"));
+}
+
+const HEADING_PREFIX = "## ";
+
+const categories = [
+    "🔒 SECURITY FIXES",
+    "🚨 BREAKING CHANGESd",
+    "🦖 Deprecations",
+    "✨ Features",
+    "🐛 Bug Fixes",
+    "🧰 Maintenance",
+];
+
+const parseReleaseNotes = (body, sections) => {
+    let heading = null;
+    for (const line of body.split("\n")) {
+        const trimmed = line.trim();
+        if (trimmed.startsWith(HEADING_PREFIX)) {
+            heading = trimmed.slice(HEADING_PREFIX.length);
+            if (!categories.includes(heading)) heading = null;
+            continue;
+        }
+        if (heading && trimmed) {
+            sections[heading].push(trimmed);
+        }
+    }
+};
+
+const main = async ({ github, releaseId, dependencies }) => {
+    const { GITHUB_REPOSITORY } = process.env;
+    const [owner, repo] = GITHUB_REPOSITORY.split("/");
+
+    const sections = Object.fromEntries(categories.map((cat) => [cat, []]));
+    for (const dependency of dependencies) {
+        const releases = await getReleases(github, dependency);
+        for (const release of releases) {
+            parseReleaseNotes(release.body, sections);
+        }
+    }
+
+    const { data: release } = await github.rest.repos.getRelease({
+        owner,
+        repo,
+        release_id: releaseId,
+    });
+
+    const intro = release.body.split(HEADING_PREFIX, 2)[0].trim();
+
+    let output = "";
+    if (intro) {
+        output = intro + "\n\n";
+    }
+
+    for (const section in sections) {
+        const lines = sections[section];
+        if (!lines.length) continue;
+        output += HEADING_PREFIX + section + "\n\n";
+        output += lines.join("\n");
+        output += "\n\n";
+    }
+
+    return output;
+};
+
+// This is just for testing locally
+// Needs environment variables GITHUB_TOKEN & GITHUB_REPOSITORY
+if (require.main === module) {
+    const { Octokit } = require("@octokit/rest");
+    const github = new Octokit({ auth: process.env.GITHUB_TOKEN });
+    if (process.argv.length < 4) {
+        // eslint-disable-next-line no-console
+        console.error("Usage: node merge-release-notes.js owner/repo:release_id npm-package-name ...");
+        process.exit(1);
+    }
+    const [releaseId, ...dependencies] = process.argv.slice(2);
+    main({ github, releaseId, dependencies }).then((output) => {
+        // eslint-disable-next-line no-console
+        console.log(output);
+    });
+}
+
+module.exports = main;

scripts/release/post-merge-master.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# When merging to develop, we need revert the `main` and `typings` fields if we adjusted them previously.
+for i in main typings browser
+do
+    # If a `lib` prefixed value is present, it means we adjusted the field earlier at publish time, so we should revert it now.
+    if [ "$(jq -r ".matrix_lib_$i" package.json)" != "null" ]; then
+        # If there's a `src` prefixed value, use that, otherwise delete.
+        # This is used to delete the `typings` field and reset `main` back to the TypeScript source.
+        src_value=$(jq -r ".matrix_src_$i" package.json)
+        if [ "$src_value" != "null" ]; then
+            jq ".$i = .matrix_src_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
+        else
+            jq "del(.$i)" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
+        fi
+    fi
+done
+
+if [ -n "$(git ls-files --modified package.json)" ]; then
+    echo "Committing develop package.json"
+    git commit package.json -m "Resetting package fields for development"
+fi

scripts/release/pre-release.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# For the published and dist versions of the package,
+# we copy the `matrix_lib_main` and `matrix_lib_typings` fields to `main` and `typings` (if they exist).
+# This small bit of gymnastics allows us to use the TypeScript source directly for development without
+# needing to build before linting or testing.
+
+for i in main typings browser
+do
+    lib_value=$(jq -r ".matrix_lib_$i" package.json)
+    if [ "$lib_value" != "null" ]; then
+        jq ".$i = .matrix_lib_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
+    fi
+done

spec/integ/crypto/cross-signing.spec.ts

@@ -31,9 +31,12 @@ import {
     SELF_CROSS_SIGNING_PRIVATE_KEY_BASE64,
     SELF_CROSS_SIGNING_PUBLIC_KEY_BASE64,
     SIGNED_CROSS_SIGNING_KEYS_DATA,
+    SIGNED_TEST_DEVICE_DATA,
     USER_CROSS_SIGNING_PRIVATE_KEY_BASE64,
 } from "../../test-utils/test-data";
+import * as testData from "../../test-utils/test-data";
 import { E2EKeyResponder } from "../../test-utils/E2EKeyResponder";
+import { AccountDataAccumulator } from "../../test-utils/AccountDataAccumulator";
 
 afterEach(() => {
     // reset fake-indexeddb after each test, to make sure we don't leak connections
@@ -97,6 +100,12 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
         /** an object which intercepts `/keys/upload` requests on the test homeserver */
         new E2EKeyReceiver(homeserverUrl);
 
+        // Silence warnings from the backup manager
+        fetchMock.getOnce(new URL("/_matrix/client/v3/room_keys/version", homeserverUrl).toString(), {
+            status: 404,
+            body: { errcode: "M_NOT_FOUND" },
+        });
+
         await initCrypto(aliceClient);
     });
 
@@ -236,6 +245,53 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
                 `[${TEST_USER_ID}].[${TEST_DEVICE_ID}].signatures.[${TEST_USER_ID}].[ed25519:${SELF_CROSS_SIGNING_PUBLIC_KEY_BASE64}]`,
             );
         });
+
+        it("can bootstrapCrossSigning twice", async () => {
+            mockSetupCrossSigningRequests();
+
+            const authDict = { type: "test" };
+            await bootstrapCrossSigning(authDict);
+
+            // a second call should do nothing except GET requests
+            fetchMock.mockClear();
+            await bootstrapCrossSigning(authDict);
+            const calls = fetchMock.calls((url, opts) => opts.method != "GET");
+            expect(calls.length).toEqual(0);
+        });
+
+        newBackendOnly("will upload existing cross-signing keys to an established secret storage", async () => {
+            // This rather obscure codepath covers the case that:
+            //   - 4S is set up and working
+            //   - our device has private cross-signing keys, but has not published them to 4S
+            //
+            // To arrange that, we call `bootstrapCrossSigning` on our main device, and then (pretend to) set up 4S from
+            // a *different* device. Then, when we call `bootstrapCrossSigning` again, it should do the honours.
+
+            mockSetupCrossSigningRequests();
+            const accountDataAccumulator = new AccountDataAccumulator();
+            accountDataAccumulator.interceptGetAccountData();
+
+            const authDict = { type: "test" };
+            await bootstrapCrossSigning(authDict);
+
+            // Pretend that another device has uploaded a 4S key
+            accountDataAccumulator.accountDataEvents.set("m.secret_storage.default_key", { key: "key_id" });
+            accountDataAccumulator.accountDataEvents.set("m.secret_storage.key.key_id", {
+                key: "keykeykey",
+                algorithm: SECRET_STORAGE_ALGORITHM_V1_AES,
+            });
+
+            // Prepare for the cross-signing keys
+            const p = accountDataAccumulator.interceptSetAccountData(":type(m.cross_signing..*)");
+
+            await bootstrapCrossSigning(authDict);
+            await p;
+
+            // The cross-signing keys should have been uploaded
+            expect(accountDataAccumulator.accountDataEvents.has("m.cross_signing.master")).toBeTruthy();
+            expect(accountDataAccumulator.accountDataEvents.has("m.cross_signing.self_signing")).toBeTruthy();
+            expect(accountDataAccumulator.accountDataEvents.has("m.cross_signing.user_signing")).toBeTruthy();
+        });
     });
 
     describe("getCrossSigningStatus()", () => {
@@ -339,4 +395,49 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
             expect(userSigningKeyId).toBe(getPubKey(crossSigningKeys.user_signing_key));
         });
     });
+
+    describe("crossSignDevice", () => {
+        beforeEach(async () => {
+            // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
+
+            // make sure that there is another device which we can sign
+            e2eKeyResponder.addDeviceKeys(SIGNED_TEST_DEVICE_DATA);
+
+            // Complete initialsync, to get the outgoing requests going
+            mockInitialApiRequests(aliceClient.getHomeserverUrl());
+            syncResponder.sendOrQueueSyncResponse({ next_batch: 1 });
+            await aliceClient.startClient();
+            await syncPromise(aliceClient);
+
+            // Wait for legacy crypto to find the device
+            await jest.advanceTimersByTimeAsync(10);
+
+            const devices = await aliceClient.getCrypto()!.getUserDeviceInfo([aliceClient.getSafeUserId()]);
+            expect(devices.get(aliceClient.getSafeUserId())!.has(testData.TEST_DEVICE_ID)).toBeTruthy();
+        });
+
+        afterEach(async () => {
+            jest.useRealTimers();
+        });
+
+        it("fails for an unknown device", async () => {
+            await expect(aliceClient.getCrypto()!.crossSignDevice("unknown")).rejects.toThrow("Unknown device");
+        });
+
+        it("cross-signs the device", async () => {
+            mockSetupCrossSigningRequests();
+            await aliceClient.getCrypto()!.bootstrapCrossSigning({});
+
+            fetchMock.mockClear();
+            await aliceClient.getCrypto()!.crossSignDevice(testData.TEST_DEVICE_ID);
+
+            // check that a sig for the device was uploaded
+            const calls = fetchMock.calls("upload-sigs");
+            expect(calls.length).toEqual(1);
+            const body = JSON.parse(calls[0][1]!.body as string);
+            const deviceSig = body[aliceClient.getSafeUserId()][testData.TEST_DEVICE_ID];
+            expect(deviceSig).toHaveProperty("signatures");
+        });
+    });
 });

spec/integ/crypto/crypto.spec.ts

@@ -29,6 +29,7 @@ import {
     getSyncResponse,
     InitCrypto,
     mkEventCustom,
+    mkMembershipCustom,
     syncPromise,
 } from "../../test-utils/test-utils";
 import * as testData from "../../test-utils/test-data";
@@ -38,6 +39,7 @@ import {
     BOB_TEST_USER_ID,
     SIGNED_CROSS_SIGNING_KEYS_DATA,
     SIGNED_TEST_DEVICE_DATA,
+    TEST_ROOM_ID,
     TEST_ROOM_ID as ROOM_ID,
     TEST_USER_ID,
 } from "../../test-utils/test-data";
@@ -74,7 +76,7 @@ import {
     mockSetupCrossSigningRequests,
     mockSetupMegolmBackupRequests,
 } from "../../test-utils/mockEndpoints";
-import { AddSecretStorageKeyOpts } from "../../../src/secret-storage";
+import { SecretStorageKeyDescription } from "../../../src/secret-storage";
 import {
     CrossSigningKey,
     CryptoCallbacks,
@@ -96,6 +98,7 @@ import {
     getTestOlmAccountKeys,
 } from "./olm-utils";
 import { ToDevicePayload } from "../../../src/models/ToDeviceMessage";
+import { AccountDataAccumulator } from "../../test-utils/AccountDataAccumulator";
 
 afterEach(() => {
     // reset fake-indexeddb after each test, to make sure we don't leak connections
@@ -229,9 +232,6 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
     /** an object which intercepts `/keys/upload` requests from {@link #aliceClient} to catch the uploaded keys */
     let keyReceiver: E2EKeyReceiver;
 
-    /** an object which intercepts `/keys/query` requests on the test homeserver */
-    let keyResponder: E2EKeyResponder;
-
     /** an object which intercepts `/sync` requests from {@link #aliceClient} */
     let syncResponder: ISyncResponder;
 
@@ -339,7 +339,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
     function createCryptoCallbacks(): CryptoCallbacks {
         // Store the cached secret storage key and return it when `getSecretStorageKey` is called
         let cachedKey: { keyId: string; key: Uint8Array };
-        const cacheSecretStorageKey = (keyId: string, keyInfo: AddSecretStorageKeyOpts, key: Uint8Array) => {
+        const cacheSecretStorageKey = (keyId: string, keyInfo: SecretStorageKeyDescription, key: Uint8Array) => {
             cachedKey = {
                 keyId,
                 key,
@@ -367,6 +367,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 accessToken: "akjgkrgjs",
                 deviceId: "xzcvb",
                 cryptoCallbacks: createCryptoCallbacks(),
+                logger: logger.getChild("aliceClient"),
             });
 
             /* set up listeners for /keys/upload and /sync */
@@ -397,6 +398,13 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
         expect(aliceClient.getCrypto()).toHaveProperty("globalBlacklistUnverifiedDevices");
     });
 
+    it("CryptoAPI.getOwnDeviceKeys returns plausible values", async () => {
+        const deviceKeys = await aliceClient.getCrypto()!.getOwnDeviceKeys();
+        // We just check for a 43-character base64 string
+        expect(deviceKeys.curve25519).toMatch(/^[A-Za-z0-9+/]{43}$/);
+        expect(deviceKeys.ed25519).toMatch(/^[A-Za-z0-9+/]{43}$/);
+    });
+
     it("Alice receives a megolm message", async () => {
         expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
         await startClientAndAwaitFirstSync();
@@ -692,7 +700,13 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
     });
 
     it("prepareToEncrypt", async () => {
-        expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
+        const homeserverUrl = aliceClient.getHomeserverUrl();
+        const keyResponder = new E2EKeyResponder(homeserverUrl);
+        keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
+
+        const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
+        keyResponder.addDeviceKeys(testDeviceKeys);
+
         await startClientAndAwaitFirstSync();
         aliceClient.setGlobalErrorOnUnknownDevices(false);
 
@@ -700,10 +714,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
         syncResponder.sendOrQueueSyncResponse(getSyncResponse(["@bob:xyz"]));
         await syncPromise(aliceClient);
 
-        // we expect alice first to query bob's keys...
-        expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-
-        // ... and then claim one of his OTKs
+        // Alice should claim one of Bob's OTKs
         expectAliceKeyClaim(getTestKeysClaimResponse("@bob:xyz"));
 
         // fire off the prepare request
@@ -720,18 +731,20 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
     it("Alice sends a megolm message with GlobalErrorOnUnknownDevices=false", async () => {
         aliceClient.setGlobalErrorOnUnknownDevices(false);
-        expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
+        const homeserverUrl = aliceClient.getHomeserverUrl();
+        const keyResponder = new E2EKeyResponder(homeserverUrl);
+        keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
+
+        const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
+        keyResponder.addDeviceKeys(testDeviceKeys);
+
         await startClientAndAwaitFirstSync();
 
         // Alice shares a room with Bob
         syncResponder.sendOrQueueSyncResponse(getSyncResponse(["@bob:xyz"]));
         await syncPromise(aliceClient);
 
-        // Once we send the message, Alice will check Bob's device list (twice, because reasons) ...
-        expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-        expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-
-        // ... and claim one of his OTKs ...
+        // ... and claim one of Bob's OTKs ...
         expectAliceKeyClaim(getTestKeysClaimResponse("@bob:xyz"));
 
         // ... and send an m.room_key message
@@ -746,18 +759,20 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
     it("We should start a new megolm session after forceDiscardSession", async () => {
         aliceClient.setGlobalErrorOnUnknownDevices(false);
-        expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
+        const homeserverUrl = aliceClient.getHomeserverUrl();
+        const keyResponder = new E2EKeyResponder(homeserverUrl);
+        keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
+
+        const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
+        keyResponder.addDeviceKeys(testDeviceKeys);
+
         await startClientAndAwaitFirstSync();
 
         // Alice shares a room with Bob
         syncResponder.sendOrQueueSyncResponse(getSyncResponse(["@bob:xyz"]));
         await syncPromise(aliceClient);
 
-        // Once we send the message, Alice will check Bob's device list (twice, because reasons) ...
-        expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-        expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-
-        // ... and claim one of his OTKs ...
+        // ... and claim one of Bob's OTKs ...
         expectAliceKeyClaim(getTestKeysClaimResponse("@bob:xyz"));
 
         // ... and send an m.room_key message
@@ -1048,8 +1063,9 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             await startClientAndAwaitFirstSync();
             const p2pSession = await establishOlmSession(aliceClient, keyReceiver, syncResponder, testOlmAccount);
 
-            // We need to fake the timers to advance the time
-            jest.useFakeTimers();
+            // We need to fake the timers to advance the time, but the wasm bindings of matrix-sdk-crypto rely on a
+            // working `queueMicrotask`
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
             const syncResponse = getSyncResponse(["@bob:xyz"]);
 
@@ -2052,13 +2068,17 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             });
         }
 
-        oldBackendOnly("Sending an event initiates a member list sync", async () => {
+        it("Sending an event initiates a member list sync", async () => {
+            const homeserverUrl = aliceClient.getHomeserverUrl();
+            const keyResponder = new E2EKeyResponder(homeserverUrl);
+            keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
+
+            const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
+            keyResponder.addDeviceKeys(testDeviceKeys);
+
             // we expect a call to the /members list...
             const memberListPromise = expectMembershipRequest(ROOM_ID, ["@bob:xyz"]);
 
-            // then a request for bob's devices...
-            expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-
             // then a to-device with the room_key
             const inboundGroupSessionPromise = expectSendRoomKey("@bob:xyz", testOlmAccount, p2pSession);
 
@@ -2071,13 +2091,17 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             await Promise.all([sendPromise, megolmMessagePromise, memberListPromise]);
         });
 
-        oldBackendOnly("loading the membership list inhibits a later load", async () => {
+        it("loading the membership list inhibits a later load", async () => {
+            const homeserverUrl = aliceClient.getHomeserverUrl();
+            const keyResponder = new E2EKeyResponder(homeserverUrl);
+            keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
+
+            const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
+            keyResponder.addDeviceKeys(testDeviceKeys);
+
             const room = aliceClient.getRoom(ROOM_ID)!;
             await Promise.all([room.loadMembersIfNeeded(), expectMembershipRequest(ROOM_ID, ["@bob:xyz"])]);
 
-            // expect a request for bob's devices...
-            expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-
             // then a to-device with the room_key
             const inboundGroupSessionPromise = expectSendRoomKey("@bob:xyz", testOlmAccount, p2pSession);
 
@@ -2166,7 +2190,8 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
     describe("key upload request", () => {
         beforeEach(() => {
-            jest.useFakeTimers();
+            // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
         });
 
         afterEach(() => {
@@ -2366,8 +2391,9 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             expect(devicesInfo.get(user)?.size).toBeFalsy();
         });
 
-        it("Get devices from tacked users", async () => {
-            jest.useFakeTimers();
+        it("Get devices from tracked users", async () => {
+            // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
             expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
             await startClientAndAwaitFirstSync();
@@ -2410,12 +2436,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
     });
 
     describe("Secret Storage and Key Backup", () => {
-        /**
-         * The account data events to be returned by the sync.
-         * Will be updated when fecthMock intercepts calls to PUT `/_matrix/client/v3/user/:userId/account_data/`.
-         * Will be used by `sendSyncResponseWithUpdatedAccountData`
-         */
-        let accountDataEvents: Map<String, any>;
+        let accountDataAccumulator: AccountDataAccumulator;
 
         /**
          * Create a fake secret storage key
@@ -2428,76 +2449,19 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
         beforeEach(async () => {
             createSecretStorageKey.mockClear();
-            accountDataEvents = new Map();
+            accountDataAccumulator = new AccountDataAccumulator();
             expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
             await startClientAndAwaitFirstSync();
         });
 
-        function mockGetAccountData() {
-            fetchMock.get(
-                `path:/_matrix/client/v3/user/:userId/account_data/:type`,
-                (url) => {
-                    const type = url.split("/").pop();
-                    const existing = accountDataEvents.get(type!);
-                    if (existing) {
-                        // return it
-                        return {
-                            status: 200,
-                            body: existing.content,
-                        };
-                    } else {
-                        // 404
-                        return {
-                            status: 404,
-                            body: { errcode: "M_NOT_FOUND", error: "Account data not found." },
-                        };
-                    }
-                },
-                { overwriteRoutes: true },
-            );
-        }
-
         /**
          * Create a mock to respond to the PUT request `/_matrix/client/v3/user/:userId/account_data/m.cross_signing.${key}`
          * Resolved when the cross signing key is uploaded
          * https://spec.matrix.org/v1.6/client-server-api/#put_matrixclientv3useruseridaccount_datatype
          */
-        function awaitCrossSigningKeyUpload(key: string): Promise<Record<string, {}>> {
-            return new Promise((resolve) => {
-                // Called when the cross signing key is uploaded
-                fetchMock.put(
-                    `express:/_matrix/client/v3/user/:userId/account_data/m.cross_signing.${key}`,
-                    (url: string, options: RequestInit) => {
-                        const content = JSON.parse(options.body as string);
-                        const type = url.split("/").pop();
-                        // update account data for sync response
-                        accountDataEvents.set(type!, content);
-                        resolve(content.encrypted);
-                        return {};
-                    },
-                );
-            });
-        }
-
-        /**
-         * Send in the sync response the current account data events, as stored by `accountDataEvents`.
-         */
-        function sendSyncResponseWithUpdatedAccountData() {
-            try {
-                syncResponder.sendOrQueueSyncResponse({
-                    next_batch: 1,
-                    account_data: {
-                        events: Array.from(accountDataEvents, ([type, content]) => ({
-                            type: type,
-                            content: content,
-                        })),
-                    },
-                });
-            } catch (err) {
-                // Might fail with "Cannot queue more than one /sync response" if called too often.
-                // It's ok if it fails here, the sync response is cumulative and will contain
-                // the latest account data.
-            }
+        async function awaitCrossSigningKeyUpload(key: string): Promise<Record<string, {}>> {
+            const content = await accountDataAccumulator.interceptSetAccountData(`m.cross_signing.${key}`);
+            return content.encrypted;
         }
 
         /**
@@ -2505,28 +2469,18 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
          * Resolved when a key is uploaded (ie in `body.content.key`)
          * https://spec.matrix.org/v1.6/client-server-api/#put_matrixclientv3useruseridaccount_datatype
          */
-        function awaitSecretStorageKeyStoredInAccountData(): Promise<string> {
-            return new Promise((resolve) => {
-                // This url is called multiple times during the secret storage bootstrap process
-                // When we received the newly generated key, we return it
-                fetchMock.put(
-                    "express:/_matrix/client/v3/user/:userId/account_data/:type(m.secret_storage.*)",
-                    (url: string, options: RequestInit) => {
-                        const type = url.split("/").pop();
-                        const content = JSON.parse(options.body as string);
-
-                        // update account data for sync response
-                        accountDataEvents.set(type!, content);
-
-                        if (content.key) {
-                            resolve(content.key);
-                        }
-                        sendSyncResponseWithUpdatedAccountData();
-                        return {};
-                    },
-                    { overwriteRoutes: true },
-                );
-            });
+        async function awaitSecretStorageKeyStoredInAccountData(): Promise<string> {
+            // eslint-disable-next-line no-constant-condition
+            while (true) {
+                const content = await accountDataAccumulator.interceptSetAccountData(":type(m.secret_storage.*)", {
+                    repeat: 1,
+                    overwriteRoutes: true,
+                });
+                accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
+                if (content.key) {
+                    return content.key;
+                }
+            }
         }
 
         function awaitMegolmBackupKeyUpload(): Promise<Record<string, {}>> {
@@ -2537,7 +2491,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                     (url: string, options: RequestInit) => {
                         const content = JSON.parse(options.body as string);
                         // update account data for sync response
-                        accountDataEvents.set("m.megolm_backup.v1", content);
+                        accountDataAccumulator.accountDataEvents.set("m.megolm_backup.v1", content);
                         resolve(content.encrypted);
                         return {};
                     },
@@ -2602,7 +2556,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             await bootstrapPromise;
 
             // Return the newly created key in the sync response
-            sendSyncResponseWithUpdatedAccountData();
+            accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
 
             // Finally ensure backup is working
             await aliceClient.getCrypto()!.checkKeyBackupAndEnable();
@@ -2610,6 +2564,30 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             await backupStatusUpdate;
         }
 
+        describe("Generate 4S recovery keys", () => {
+            it("should create a random recovery key", async () => {
+                const generatedKey = await aliceClient.getCrypto()!.createRecoveryKeyFromPassphrase();
+                expect(generatedKey.privateKey).toBeDefined();
+                expect(generatedKey.privateKey).toBeInstanceOf(Uint8Array);
+                expect(generatedKey.privateKey.length).toBe(32);
+                expect(generatedKey.keyInfo?.passphrase).toBeUndefined();
+                expect(generatedKey.encodedPrivateKey).toBeDefined();
+                expect(generatedKey.encodedPrivateKey!.indexOf("Es")).toBe(0);
+            });
+
+            it("should create a recovery key from passphrase", async () => {
+                const generatedKey = await aliceClient.getCrypto()!.createRecoveryKeyFromPassphrase("mypassphrase");
+                expect(generatedKey.privateKey).toBeDefined();
+                expect(generatedKey.privateKey).toBeInstanceOf(Uint8Array);
+                expect(generatedKey.privateKey.length).toBe(32);
+                expect(generatedKey.keyInfo?.passphrase?.algorithm).toBe("m.pbkdf2");
+                expect(generatedKey.keyInfo?.passphrase?.iterations).toBe(500000);
+
+                expect(generatedKey.encodedPrivateKey).toBeDefined();
+                expect(generatedKey.encodedPrivateKey!.indexOf("Es")).toBe(0);
+            });
+        });
+
         describe("bootstrapSecretStorage", () => {
             // Doesn't work with legacy crypto, which will try to bootstrap even without private key, which is buggy.
             newBackendOnly(
@@ -2624,7 +2602,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             );
 
             it("Should create a 4S key", async () => {
-                mockGetAccountData();
+                accountDataAccumulator.interceptGetAccountData();
 
                 const awaitAccountData = awaitAccountDataUpdate("m.secret_storage.default_key");
 
@@ -2635,8 +2613,16 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 // Wait for the key to be uploaded in the account data
                 const secretStorageKey = await awaitSecretStorageKeyStoredInAccountData();
 
+                // check that the key content contains the key check info
+                const keyContent = accountDataAccumulator.accountDataEvents.get(
+                    `m.secret_storage.key.${secretStorageKey}`,
+                )!;
+                // In order to verify if the key is valid, a zero secret is encrypted with the key
+                expect(keyContent.iv).toBeDefined();
+                expect(keyContent.mac).toBeDefined();
+
                 // Return the newly created key in the sync response
-                sendSyncResponseWithUpdatedAccountData();
+                accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
 
                 // Finally, wait for bootstrapSecretStorage to finished
                 await bootstrapPromise;
@@ -2660,7 +2646,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 await awaitSecretStorageKeyStoredInAccountData();
 
                 // Return the newly created key in the sync response
-                sendSyncResponseWithUpdatedAccountData();
+                accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
 
                 // Wait for bootstrapSecretStorage to finished
                 await bootstrapPromise;
@@ -2684,7 +2670,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 await awaitSecretStorageKeyStoredInAccountData();
 
                 // Return the newly created key in the sync response
-                sendSyncResponseWithUpdatedAccountData();
+                accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
 
                 // Wait for bootstrapSecretStorage to finished
                 await bootstrapPromise;
@@ -2698,7 +2684,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 await awaitSecretStorageKeyStoredInAccountData();
 
                 // Return the newly created key in the sync response
-                sendSyncResponseWithUpdatedAccountData();
+                accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
 
                 // Wait for bootstrapSecretStorage to finished
                 await bootstrapPromise;
@@ -2722,7 +2708,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 const secretStorageKey = await awaitSecretStorageKeyStoredInAccountData();
 
                 // Return the newly created key in the sync response
-                sendSyncResponseWithUpdatedAccountData();
+                accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
 
                 // Wait for the cross signing keys to be uploaded
                 const [masterKey, userSigningKey, selfSigningKey] = await Promise.all([
@@ -2762,7 +2748,8 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
         describe("Manage Key Backup", () => {
             beforeEach(async () => {
-                jest.useFakeTimers();
+                // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+                jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
             });
 
             afterEach(() => {
@@ -2875,6 +2862,19 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
                 const newBackupUploadPromise = awaitMegolmBackupKeyUpload();
 
+                // Track calls to scheduleAllGroupSessionsForBackup. This is
+                // only relevant on legacy encryption.
+                const scheduleAllGroupSessionsForBackup = jest.fn();
+                if (backend === "libolm") {
+                    aliceClient.crypto!.backupManager.scheduleAllGroupSessionsForBackup =
+                        scheduleAllGroupSessionsForBackup;
+                } else {
+                    // With Rust crypto, we don't need to call this function, so
+                    // we call the dummy value here so we pass our later
+                    // expectation.
+                    scheduleAllGroupSessionsForBackup();
+                }
+
                 await aliceClient.getCrypto()!.resetKeyBackup();
                 await awaitDeleteCalled;
                 await newBackupStatusUpdate;
@@ -2886,6 +2886,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 expect(nextVersion).toBeDefined();
                 expect(nextVersion).not.toEqual(currentVersion);
                 expect(nextKey).not.toEqual(currentBackupKey);
+                expect(scheduleAllGroupSessionsForBackup).toHaveBeenCalled();
 
                 // The `deleteKeyBackupVersion` API is deprecated but has been modified to work with both crypto backend
                 // ensure that it works anyhow
@@ -2902,7 +2903,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             // anything that we don't have a specific matcher for silently returns a 404
             fetchMock.catch(404);
 
-            keyResponder = new E2EKeyResponder(aliceClient.getHomeserverUrl());
+            const keyResponder = new E2EKeyResponder(aliceClient.getHomeserverUrl());
             keyResponder.addCrossSigningData(SIGNED_CROSS_SIGNING_KEYS_DATA);
             keyResponder.addDeviceKeys(SIGNED_TEST_DEVICE_DATA);
             keyResponder.addKeyReceiver(BOB_TEST_USER_ID, keyReceiver);
@@ -2938,4 +2939,180 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             expect(hasCrossSigningKeysForUser).toBe(false);
         });
     });
+
+    /** Guards against downgrade attacks from servers hiding or manipulating the crypto settings. */
+    describe("Persistent encryption settings", () => {
+        let persistentStoreClient: MatrixClient;
+        let client2: MatrixClient;
+
+        beforeEach(async () => {
+            const homeserverurl = "https://alice-server.com";
+            const userId = "@alice:localhost";
+
+            const keyResponder = new E2EKeyResponder(homeserverurl);
+            keyResponder.addKeyReceiver(userId, keyReceiver);
+
+            // For legacy crypto, these tests only work properly with a proper (indexeddb-based) CryptoStore, so
+            // rather than using the existing `aliceClient`, create a new client. Once we drop legacy crypto, we can
+            // just use `aliceClient` here.
+            persistentStoreClient = await makeNewClient(homeserverurl, userId, "persistentStoreClient");
+            await persistentStoreClient.startClient({});
+        });
+
+        afterEach(async () => {
+            persistentStoreClient.stopClient();
+            client2?.stopClient();
+        });
+
+        test("Sending a message in a room where the server is hiding the state event does not send a plaintext event", async () => {
+            // Alice is in an encrypted room
+            const encryptionState = mkEncryptionEvent({ algorithm: "m.megolm.v1.aes-sha2" });
+            syncResponder.sendOrQueueSyncResponse(getSyncResponseWithState([encryptionState]));
+            await syncPromise(persistentStoreClient);
+
+            // Send a message, and expect to get an `m.room.encrypted` event.
+            await Promise.all([persistentStoreClient.sendTextMessage(ROOM_ID, "test"), expectEncryptedSendMessage()]);
+
+            // We now replace the client, and allow the new one to resync, *without* the encryption event.
+            client2 = await replaceClient(persistentStoreClient);
+            syncResponder.sendOrQueueSyncResponse(getSyncResponseWithState([]));
+            await client2.startClient({});
+            await syncPromise(client2);
+            logger.log(client2.getUserId() + ": restarted");
+
+            await expectSendMessageToFail(client2);
+        });
+
+        test("Changes to the rotation period should be ignored", async () => {
+            // Alice is in an encrypted room, where the rotation period is set to 2 messages
+            const encryptionState = mkEncryptionEvent({ algorithm: "m.megolm.v1.aes-sha2", rotation_period_msgs: 2 });
+            syncResponder.sendOrQueueSyncResponse(getSyncResponseWithState([encryptionState]));
+            await syncPromise(persistentStoreClient);
+
+            // Send a message, and expect to get an `m.room.encrypted` event.
+            const [, msg1Content] = await Promise.all([
+                persistentStoreClient.sendTextMessage(ROOM_ID, "test1"),
+                expectEncryptedSendMessage(),
+            ]);
+
+            // Replace the state with one which bumps the rotation period. This should be ignored, though it's not
+            // clear that is correct behaviour (see https://github.com/element-hq/element-meta/issues/69)
+            const encryptionState2 = mkEncryptionEvent({
+                algorithm: "m.megolm.v1.aes-sha2",
+                rotation_period_msgs: 100,
+            });
+            syncResponder.sendOrQueueSyncResponse({
+                next_batch: "1",
+                rooms: { join: { [TEST_ROOM_ID]: { timeline: { events: [encryptionState2], prev_batch: "" } } } },
+            });
+            await syncPromise(persistentStoreClient);
+
+            // Send two more messages. The first should use the same megolm session as the first; the second should
+            // use a different one.
+            const [, msg2Content] = await Promise.all([
+                persistentStoreClient.sendTextMessage(ROOM_ID, "test2"),
+                expectEncryptedSendMessage(),
+            ]);
+            expect(msg2Content.session_id).toEqual(msg1Content.session_id);
+            const [, msg3Content] = await Promise.all([
+                persistentStoreClient.sendTextMessage(ROOM_ID, "test3"),
+                expectEncryptedSendMessage(),
+            ]);
+            expect(msg3Content.session_id).not.toEqual(msg1Content.session_id);
+        });
+
+        test("Changes to the rotation period should be ignored after a client restart", async () => {
+            // Alice is in an encrypted room, where the rotation period is set to 2 messages
+            const encryptionState = mkEncryptionEvent({ algorithm: "m.megolm.v1.aes-sha2", rotation_period_msgs: 2 });
+            syncResponder.sendOrQueueSyncResponse(getSyncResponseWithState([encryptionState]));
+            await syncPromise(persistentStoreClient);
+
+            // Send a message, and expect to get an `m.room.encrypted` event.
+            await Promise.all([persistentStoreClient.sendTextMessage(ROOM_ID, "test1"), expectEncryptedSendMessage()]);
+
+            // We now replace the client, and allow the new one to resync with a *different* encryption event.
+            client2 = await replaceClient(persistentStoreClient);
+            const encryptionState2 = mkEncryptionEvent({
+                algorithm: "m.megolm.v1.aes-sha2",
+                rotation_period_msgs: 100,
+            });
+            syncResponder.sendOrQueueSyncResponse(getSyncResponseWithState([encryptionState2]));
+            await client2.startClient({});
+            await syncPromise(client2);
+            logger.log(client2.getUserId() + ": restarted");
+
+            // Now send another message, which should (for now) be rejected.
+            await expectSendMessageToFail(client2);
+        });
+
+        /** Shut down `oldClient`, and build a new MatrixClient for the same user. */
+        async function replaceClient(oldClient: MatrixClient) {
+            oldClient.stopClient();
+            syncResponder.sendOrQueueSyncResponse({}); // flush pending request from old client
+            return makeNewClient(oldClient.getHomeserverUrl(), oldClient.getSafeUserId(), "client2");
+        }
+
+        async function makeNewClient(
+            homeserverUrl: string,
+            userId: string,
+            loggerPrefix: string,
+        ): Promise<MatrixClient> {
+            const client = createClient({
+                baseUrl: homeserverUrl,
+                userId: userId,
+                accessToken: "akjgkrgjs",
+                deviceId: "xzcvb",
+                cryptoCallbacks: createCryptoCallbacks(),
+                logger: logger.getChild(loggerPrefix),
+
+                // For legacy crypto, these tests only work with a proper persistent cryptoStore.
+                cryptoStore: new IndexedDBCryptoStore(indexedDB, "test"),
+            });
+            await initCrypto(client);
+            mockInitialApiRequests(client.getHomeserverUrl());
+            return client;
+        }
+
+        function mkEncryptionEvent(content: Object) {
+            return mkEventCustom({
+                sender: persistentStoreClient.getSafeUserId(),
+                type: "m.room.encryption",
+                state_key: "",
+                content: content,
+            });
+        }
+
+        /** Sync response which includes `TEST_ROOM_ID`, where alice is a member
+         *
+         * @param stateEvents - Additional state events for the test room
+         */
+        function getSyncResponseWithState(stateEvents: Array<Object>) {
+            const roomResponse = {
+                state: {
+                    events: [
+                        mkMembershipCustom({ membership: "join", sender: persistentStoreClient.getSafeUserId() }),
+                        ...stateEvents,
+                    ],
+                },
+                timeline: {
+                    events: [],
+                    prev_batch: "",
+                },
+            };
+
+            return {
+                next_batch: "1",
+                rooms: { join: { [TEST_ROOM_ID]: roomResponse } },
+            };
+        }
+
+        /** Send a message with the given client, and check that it is not sent in plaintext */
+        async function expectSendMessageToFail(aliceClient2: MatrixClient) {
+            // The precise failure mode here is somewhat up for debate (https://github.com/element-hq/element-meta/issues/69).
+            // For now, the attempt to send is rejected with an exception. The text is different between old and new stacks.
+            await expect(aliceClient2.sendTextMessage(ROOM_ID, "test")).rejects.toThrow(
+                /unconfigured room !room:id|Room !room:id was previously configured to use encryption/,
+            );
+        }
+    });
 });

spec/integ/crypto/megolm-backup.spec.ts

@@ -17,8 +17,18 @@ limitations under the License.
 import fetchMock from "fetch-mock-jest";
 import "fake-indexeddb/auto";
 import { IDBFactory } from "fake-indexeddb";
+import { Mocked } from "jest-mock";
 
-import { createClient, CryptoEvent, ICreateClientOpts, MatrixClient, TypedEventEmitter } from "../../../src";
+import {
+    createClient,
+    CryptoApi,
+    CryptoEvent,
+    ICreateClientOpts,
+    IEvent,
+    IMegolmSessionData,
+    MatrixClient,
+    TypedEventEmitter,
+} from "../../../src";
 import { SyncResponder } from "../../test-utils/SyncResponder";
 import { E2EKeyReceiver } from "../../test-utils/E2EKeyReceiver";
 import { E2EKeyResponder } from "../../test-utils/E2EKeyResponder";
@@ -31,9 +41,10 @@ import {
     syncPromise,
 } from "../../test-utils/test-utils";
 import * as testData from "../../test-utils/test-data";
-import { KeyBackupInfo } from "../../../src/crypto-api/keybackup";
+import { KeyBackupInfo, KeyBackupSession } from "../../../src/crypto-api/keybackup";
 import { IKeyBackup } from "../../../src/crypto/backup";
 import { flushPromises } from "../../test-utils/flushPromises";
+import { defer, IDeferred } from "../../../src/utils";
 
 const ROOM_ID = testData.TEST_ROOM_ID;
 
@@ -118,7 +129,8 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
     let e2eKeyResponder: E2EKeyResponder;
 
     beforeEach(async () => {
-        jest.useFakeTimers();
+        // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+        jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
         // anything that we don't have a specific matcher for silently returns a 404
         fetchMock.catch(404);
@@ -285,17 +297,21 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
     });
 
     describe("recover from backup", () => {
-        it("can restore from backup (Curve25519 version)", async function () {
+        let aliceCrypto: CryptoApi;
+
+        beforeEach(async () => {
             fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
 
             aliceClient = await initTestClient();
-            const aliceCrypto = aliceClient.getCrypto()!;
+            aliceCrypto = aliceClient.getCrypto()!;
             await aliceClient.startClient();
 
             // tell Alice to trust the dummy device that signed the backup
             await waitForDeviceList();
             await aliceCrypto.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
+        });
 
+        it("can restore from backup (Curve25519 version)", async function () {
             const fullBackup = {
                 rooms: {
                     [ROOM_ID]: {
@@ -339,17 +355,179 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
             expect(afterCache.imported).toStrictEqual(1);
         });
 
+        /**
+         * Creates a mock backup response of a GET `room_keys/keys` with a given number of keys per room.
+         * @param keysPerRoom The number of keys per room
+         */
+        function createBackupDownloadResponse(keysPerRoom: number[]) {
+            const response: {
+                rooms: {
+                    [roomId: string]: {
+                        sessions: {
+                            [sessionId: string]: KeyBackupSession;
+                        };
+                    };
+                };
+            } = { rooms: {} };
+
+            const expectedTotal = keysPerRoom.reduce((a, b) => a + b, 0);
+            for (let i = 0; i < keysPerRoom.length; i++) {
+                const roomId = `!room${i}:example.com`;
+                response.rooms[roomId] = { sessions: {} };
+                for (let j = 0; j < keysPerRoom[i]; j++) {
+                    const sessionId = `session${j}`;
+                    // Put the same fake session data, not important for that test
+                    response.rooms[roomId].sessions[sessionId] = testData.CURVE25519_KEY_BACKUP_DATA;
+                }
+            }
+            return { response, expectedTotal };
+        }
+
+        it("Should import full backup in chunks", async function () {
+            const importMockImpl = jest.fn();
+            // @ts-ignore - mock a private method for testing purpose
+            aliceCrypto.importBackedUpRoomKeys = importMockImpl;
+
+            // We need several rooms with several sessions to test chunking
+            const { response, expectedTotal } = createBackupDownloadResponse([45, 300, 345, 12, 130]);
+
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys", response);
+
+            const check = await aliceCrypto.checkKeyBackupAndEnable();
+
+            const progressCallback = jest.fn();
+            const result = await aliceClient.restoreKeyBackupWithRecoveryKey(
+                testData.BACKUP_DECRYPTION_KEY_BASE58,
+                undefined,
+                undefined,
+                check!.backupInfo!,
+                {
+                    progressCallback,
+                },
+            );
+
+            expect(result.imported).toStrictEqual(expectedTotal);
+            // Should be called 5 times: 200*4 plus one chunk with the remaining 32
+            expect(importMockImpl).toHaveBeenCalledTimes(5);
+            for (let i = 0; i < 4; i++) {
+                expect(importMockImpl.mock.calls[i][0].length).toEqual(200);
+            }
+            expect(importMockImpl.mock.calls[4][0].length).toEqual(32);
+
+            expect(progressCallback).toHaveBeenCalledWith({
+                stage: "fetch",
+            });
+
+            // Should be called 4 times and report 200/400/600/800
+            for (let i = 0; i < 4; i++) {
+                expect(progressCallback).toHaveBeenCalledWith({
+                    total: expectedTotal,
+                    successes: (i + 1) * 200,
+                    stage: "load_keys",
+                    failures: 0,
+                });
+            }
+
+            // The last chunk
+            expect(progressCallback).toHaveBeenCalledWith({
+                total: expectedTotal,
+                successes: 832,
+                stage: "load_keys",
+                failures: 0,
+            });
+        });
+
+        it("Should continue to process backup if a chunk import fails and report failures", async function () {
+            // @ts-ignore - mock a private method for testing purpose
+            aliceCrypto.importBackedUpRoomKeys = jest
+                .fn()
+                .mockImplementationOnce(() => {
+                    // Fail to import first chunk
+                    throw new Error("test error");
+                })
+                // Ok for other chunks
+                .mockResolvedValue(undefined);
+
+            const { response, expectedTotal } = createBackupDownloadResponse([100, 300]);
+
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys", response);
+
+            const check = await aliceCrypto.checkKeyBackupAndEnable();
+
+            const progressCallback = jest.fn();
+            const result = await aliceClient.restoreKeyBackupWithRecoveryKey(
+                testData.BACKUP_DECRYPTION_KEY_BASE58,
+                undefined,
+                undefined,
+                check!.backupInfo!,
+                {
+                    progressCallback,
+                },
+            );
+
+            expect(result.total).toStrictEqual(expectedTotal);
+            // A chunk failed to import
+            expect(result.imported).toStrictEqual(200);
+
+            expect(progressCallback).toHaveBeenCalledWith({
+                total: expectedTotal,
+                successes: 0,
+                stage: "load_keys",
+                failures: 200,
+            });
+
+            expect(progressCallback).toHaveBeenCalledWith({
+                total: expectedTotal,
+                successes: 200,
+                stage: "load_keys",
+                failures: 200,
+            });
+        });
+
+        it("Should continue if some keys fails to decrypt", async function () {
+            // @ts-ignore - mock a private method for testing purpose
+            aliceCrypto.importBackedUpRoomKeys = jest.fn();
+
+            const decryptionFailureCount = 2;
+
+            const mockDecryptor = {
+                // DecryptSessions does not reject on decryption failure, but just skip the key
+                decryptSessions: jest.fn().mockImplementation((sessions) => {
+                    // simulate fail to decrypt 2 keys out of all
+                    const decrypted = [];
+                    const keys = Object.keys(sessions);
+                    for (let i = 0; i < keys.length - decryptionFailureCount; i++) {
+                        decrypted.push({
+                            session_id: keys[i],
+                        } as unknown as Mocked<IMegolmSessionData>);
+                    }
+                    return decrypted;
+                }),
+                free: jest.fn(),
+            };
+
+            // @ts-ignore - mock a private method for testing purpose
+            aliceCrypto.getBackupDecryptor = jest.fn().mockResolvedValue(mockDecryptor);
+
+            const { response, expectedTotal } = createBackupDownloadResponse([100]);
+
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys", response);
+
+            const check = await aliceCrypto.checkKeyBackupAndEnable();
+
+            const result = await aliceClient.restoreKeyBackupWithRecoveryKey(
+                testData.BACKUP_DECRYPTION_KEY_BASE58,
+                undefined,
+                undefined,
+                check!.backupInfo!,
+            );
+
+            expect(result.total).toStrictEqual(expectedTotal);
+            // A chunk failed to import
+            expect(result.imported).toStrictEqual(expectedTotal - decryptionFailureCount);
+        });
+
         it("recover specific session from backup", async function () {
-            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
-
-            aliceClient = await initTestClient();
-            const aliceCrypto = aliceClient.getCrypto()!;
-            await aliceClient.startClient();
-
-            // tell Alice to trust the dummy device that signed the backup
-            await waitForDeviceList();
-            await aliceCrypto.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
-
             fetchMock.get(
                 "express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id",
                 testData.CURVE25519_KEY_BACKUP_DATA,
@@ -370,16 +548,6 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
         });
 
         it("Fails on bad recovery key", async function () {
-            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
-
-            aliceClient = await initTestClient();
-            const aliceCrypto = aliceClient.getCrypto()!;
-            await aliceClient.startClient();
-
-            // tell Alice to trust the dummy device that signed the backup
-            await waitForDeviceList();
-            await aliceCrypto.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
-
             const fullBackup = {
                 rooms: {
                     [ROOM_ID]: {
@@ -888,6 +1056,146 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
         });
     });
 
+    describe("Backup Changed from other sessions", () => {
+        beforeEach(async () => {
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+
+            // ignore requests to send room key requests
+            fetchMock.put("express:/_matrix/client/v3/sendToDevice/m.room_key_request/:request_id", {});
+
+            aliceClient = await initTestClient();
+            const aliceCrypto = aliceClient.getCrypto()!;
+            await aliceCrypto.storeSessionBackupPrivateKey(
+                Buffer.from(testData.BACKUP_DECRYPTION_KEY_BASE64, "base64"),
+                testData.SIGNED_BACKUP_DATA.version!,
+            );
+
+            // start after saving the private key
+            await aliceClient.startClient();
+
+            // tell Alice to trust the dummy device that signed the backup, and re-check the backup.
+            // XXX: should we automatically re-check after a device becomes verified?
+            await waitForDeviceList();
+            await aliceClient.getCrypto()!.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
+            await aliceClient.getCrypto()!.checkKeyBackupAndEnable();
+        });
+
+        // let aliceClient: MatrixClient;
+
+        const SYNC_RESPONSE = {
+            next_batch: 1,
+            rooms: { join: { [ROOM_ID]: { timeline: { events: [testData.ENCRYPTED_EVENT] } } } },
+        };
+
+        it("If current backup has changed, the manager should switch to the new one on UTD", async () => {
+            // =====
+            // First ensure that the client checks for keys using the backup version 1
+            /// =====
+
+            fetchMock.get(
+                "express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id",
+                (url, request) => {
+                    // check that the version is correct
+                    const version = new URLSearchParams(new URL(url).search).get("version");
+                    if (version == "1") {
+                        return testData.CURVE25519_KEY_BACKUP_DATA;
+                    } else {
+                        return {
+                            status: 403,
+                            body: {
+                                current_version: "1",
+                                errcode: "M_WRONG_ROOM_KEYS_VERSION",
+                                error: "Wrong backup version.",
+                            },
+                        };
+                    }
+                },
+                { overwriteRoutes: true },
+            );
+
+            // Send Alice a message that she won't be able to decrypt, and check that she fetches the key from the backup.
+            syncResponder.sendOrQueueSyncResponse(SYNC_RESPONSE);
+            await syncPromise(aliceClient);
+
+            const room = aliceClient.getRoom(ROOM_ID)!;
+            const event = room.getLiveTimeline().getEvents()[0];
+            await advanceTimersUntil(awaitDecryption(event, { waitOnDecryptionFailure: true }));
+
+            expect(event.getContent()).toEqual(testData.CLEAR_EVENT.content);
+
+            // =====
+            // Second suppose now that the backup has changed to version 2
+            /// =====
+
+            const newBackup = {
+                ...testData.SIGNED_BACKUP_DATA,
+                version: "2",
+            };
+
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", newBackup, { overwriteRoutes: true });
+            // suppose the new key is now known
+            const aliceCrypto = aliceClient.getCrypto()!;
+            await aliceCrypto.storeSessionBackupPrivateKey(
+                Buffer.from(testData.BACKUP_DECRYPTION_KEY_BASE64, "base64"),
+                newBackup.version,
+            );
+
+            // A check backup should happen at some point
+            await aliceCrypto.checkKeyBackupAndEnable();
+
+            const awaitHasQueriedNewBackup: IDeferred<void> = defer<void>();
+
+            fetchMock.get(
+                "express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id",
+                (url, request) => {
+                    // check that the version is correct
+                    const version = new URLSearchParams(new URL(url).search).get("version");
+                    if (version == newBackup.version) {
+                        awaitHasQueriedNewBackup.resolve();
+                        return testData.CURVE25519_KEY_BACKUP_DATA;
+                    } else {
+                        // awaitHasQueriedOldBackup.resolve();
+                        return {
+                            status: 403,
+                            body: {
+                                current_version: "2",
+                                errcode: "M_WRONG_ROOM_KEYS_VERSION",
+                                error: "Wrong backup version.",
+                            },
+                        };
+                    }
+                },
+                { overwriteRoutes: true },
+            );
+
+            // Send Alice a message that she won't be able to decrypt, and check that she fetches the key from the new backup.
+            const newMessage: Partial<IEvent> = {
+                type: "m.room.encrypted",
+                room_id: "!room:id",
+                sender: "@alice:localhost",
+                content: {
+                    algorithm: "m.megolm.v1.aes-sha2",
+                    ciphertext:
+                        "AwgAEpABKvf9FqPW52zeHfeVTn90a3jlBLlx7g6VDEkc2089RQUJoWpSJRiK13E83rN41wgGFJccyfoCr7ZDGJeuGYMGETTrgnLQhLs6JmyPf37JYkzxW8uS8rGUKEqTFQriKhibHVLvVacOlSIObUiKU/V3r176XuixqZF/4eyK9A22JNpInbgI10ZUT6LnApH9LR3FpZbE2zImf1uNPuvp7r0xQbW7CcJjqpH+qTPBD5zFdFnMkc2SnbXCsIOaX11Dm0krWfQz7iA26ZnI1nyZnyh7XPrCnJCRsuQH",
+                    device_id: "WVMJGTSSVB",
+                    sender_key: "E5RiY/YCIrHWaF4u416CqvblC6udK2jt9SJ/h1QeLS0",
+                    session_id: "ybnW+LGdUhoS4fHm1DAEphukO3sZ1GCqZD7UQz7L+GA",
+                },
+                event_id: "$event2",
+                origin_server_ts: 1507753887000,
+            };
+
+            const nextSyncResponse = {
+                next_batch: 2,
+                rooms: { join: { [ROOM_ID]: { timeline: { events: [newMessage] } } } },
+            };
+            syncResponder.sendOrQueueSyncResponse(nextSyncResponse);
+            await syncPromise(aliceClient);
+
+            await awaitHasQueriedNewBackup.promise;
+        });
+    });
+
     /** make sure that the client knows about the dummy device */
     async function waitForDeviceList(): Promise<void> {
         // Completing the initial sync will make the device list download outdated device lists (of which our own

spec/integ/crypto/rust-crypto.spec.ts

@@ -16,8 +16,12 @@ limitations under the License.
 
 import "fake-indexeddb/auto";
 import { IDBFactory } from "fake-indexeddb";
+import fetchMock from "fetch-mock-jest";
 
-import { createClient } from "../../../src";
+import { createClient, CryptoEvent, IndexedDBCryptoStore } from "../../../src";
+import { populateStore } from "../../test-utils/test_indexeddb_cryptostore_dump";
+
+jest.setTimeout(15000);
 
 afterEach(() => {
     // reset fake-indexeddb after each test, to make sure we don't leak connections
@@ -88,6 +92,70 @@ describe("MatrixClient.initRustCrypto", () => {
         await matrixClient.initRustCrypto();
         await matrixClient.initRustCrypto();
     });
+
+    it("should migrate from libolm", async () => {
+        fetchMock.get("path:/_matrix/client/v3/room_keys/version", {
+            auth_data: {
+                public_key: "q+HZiJdHl2Yopv9GGvv7EYSzDMrAiRknK4glSdoaomI",
+                signatures: {
+                    "@vdhtest200713:matrix.org": {
+                        "ed25519:gh9fGr39eNZUdWynEMJ/q/WZq/Pk/foFxHXFBFm18ZI":
+                            "reDp6Mu+j+tfUL3/T6f5OBT3N825Lzpc43vvG+RvjX6V+KxXzodBQArgCoeEHLtL9OgSBmNrhTkSOX87MWCKAw",
+                        "ed25519:KMFSTJSMLB":
+                            "F8tyV5W6wNi0GXTdSg+gxSCULQi0EYxdAAqfkyNq58KzssZMw5i+PRA0aI2b+D7NH/aZaJrtiYNHJ0gWLSQvAw",
+                    },
+                },
+            },
+            version: "7",
+            algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+            etag: "1",
+            count: 79,
+        });
+
+        const testStoreName = "test-store";
+        await populateStore(testStoreName);
+        const cryptoStore = new IndexedDBCryptoStore(indexedDB, testStoreName);
+
+        const matrixClient = createClient({
+            baseUrl: "http://test.server",
+            userId: "@vdhtest200713:matrix.org",
+            deviceId: "KMFSTJSMLB",
+            cryptoStore,
+            pickleKey: "+1k2Ppd7HIisUY824v7JtV3/oEE4yX0TqtmNPyhaD7o",
+        });
+
+        const progressListener = jest.fn();
+        matrixClient.addListener(CryptoEvent.LegacyCryptoStoreMigrationProgress, progressListener);
+
+        await matrixClient.initRustCrypto();
+
+        // Do some basic checks on the imported data
+        const deviceKeys = await matrixClient.getCrypto()!.getOwnDeviceKeys();
+        expect(deviceKeys.curve25519).toEqual("LKv0bKbc0EC4h0jknbemv3QalEkeYvuNeUXVRgVVTTU");
+        expect(deviceKeys.ed25519).toEqual("qK70DEqIXq7T+UU3v/al47Ab4JkMEBLpNrTBMbS5rrw");
+
+        expect(await matrixClient.getCrypto()!.getActiveSessionBackupVersion()).toEqual("7");
+
+        expect(await matrixClient.getCrypto()!.isEncryptionEnabledInRoom("!CWLUCoEWXSFyTCOtfL:matrix.org")).toBe(true);
+
+        // check the progress callback
+        expect(progressListener.mock.calls.length).toBeGreaterThan(50);
+
+        // The first call should have progress == 0
+        const [firstProgress, totalSteps] = progressListener.mock.calls[0];
+        expect(totalSteps).toBeGreaterThan(3000);
+        expect(firstProgress).toEqual(0);
+
+        for (let i = 1; i < progressListener.mock.calls.length - 1; i++) {
+            const [progress, total] = progressListener.mock.calls[i];
+            expect(total).toEqual(totalSteps);
+            expect(progress).toBeGreaterThan(progressListener.mock.calls[i - 1][0]);
+            expect(progress).toBeLessThanOrEqual(totalSteps);
+        }
+
+        // The final call should have progress == total == -1
+        expect(progressListener).toHaveBeenLastCalledWith(-1, -1);
+    }, 60000);
 });
 
 describe("MatrixClient.clearStores", () => {

spec/integ/crypto/verification.spec.ts

@@ -85,7 +85,8 @@ import { encodeBase64 } from "../../../src/base64";
 
 // The verification flows use javascript timers to set timeouts. We tell jest to use mock timer implementations
 // to ensure that we don't end up with dangling timeouts.
-jest.useFakeTimers();
+// But the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
 beforeAll(async () => {
     // we use the libolm primitives in the test, so init the Olm library
@@ -743,6 +744,8 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
             expect(toDeviceMessage.transaction_id).toEqual(transactionId);
             expect(toDeviceMessage.code).toEqual("m.user");
             expect(request.phase).toEqual(VerificationPhase.Cancelled);
+            expect(request.cancellationCode).toEqual("m.user");
+            expect(request.cancellingUserId).toEqual("@alice:localhost");
         });
 
         it("can cancel during the SAS phase", async () => {
@@ -1259,14 +1262,11 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
 
             const requestId = await requestPromises.get("m.megolm_backup.v1");
 
+            const keyBackupIsCached = emitPromise(aliceClient, CryptoEvent.KeyBackupDecryptionKeyCached);
+
             await sendBackupGossipAndExpectVersion(requestId!, BACKUP_DECRYPTION_KEY_BASE64, matchingBackupInfo);
 
-            // We are lacking a way to signal that the secret has been received, so we wait a bit..
-            jest.useRealTimers();
-            await new Promise((resolve) => {
-                setTimeout(resolve, 500);
-            });
-            jest.useFakeTimers();
+            await keyBackupIsCached;
 
             // the backup secret should be cached
             const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
@@ -1288,7 +1288,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
             await new Promise((resolve) => {
                 setTimeout(resolve, 500);
             });
-            jest.useFakeTimers();
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
             // the backup secret should not be cached
             const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
@@ -1312,7 +1312,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
             await new Promise((resolve) => {
                 setTimeout(resolve, 500);
             });
-            jest.useFakeTimers();
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
             // the backup secret should not be cached
             const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
@@ -1337,7 +1337,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
             await new Promise((resolve) => {
                 setTimeout(resolve, 500);
             });
-            jest.useFakeTimers();
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
             // the backup secret should not be cached
             const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
@@ -1358,7 +1358,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
             await new Promise((resolve) => {
                 setTimeout(resolve, 500);
             });
-            jest.useFakeTimers();
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
             // the backup secret should not be cached
             const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();

spec/integ/matrix-client-event-timeline.spec.ts

@@ -33,7 +33,7 @@ import {
 import { logger } from "../../src/logger";
 import { encodeParams, encodeUri, QueryDict, replaceParam } from "../../src/utils";
 import { TestClient } from "../TestClient";
-import { FeatureSupport, Thread, THREAD_RELATION_TYPE, ThreadEvent } from "../../src/models/thread";
+import { FeatureSupport, Thread, ThreadEvent } from "../../src/models/thread";
 import { emitPromise } from "../test-utils/test-utils";
 import { Feature, ServerSupport } from "../../src/feature";
 
@@ -623,9 +623,7 @@ describe("MatrixClient event timelines", function () {
                     "GET",
                     "/rooms/!foo%3Abar/relations/" +
                         encodeURIComponent(THREAD_ROOT.event_id!) +
-                        "/" +
-                        encodeURIComponent(THREAD_RELATION_TYPE.name) +
-                        buildRelationPaginationQuery({ dir: Direction.Backward, limit: 1 }),
+                        buildRelationPaginationQuery({ dir: Direction.Backward }),
                 )
                 .respond(200, function () {
                     return {
@@ -1154,10 +1152,7 @@ describe("MatrixClient event timelines", function () {
         httpBackend
             .when(
                 "GET",
-                "/_matrix/client/v1/rooms/!foo%3Abar/relations/" +
-                    encodeURIComponent(THREAD_ROOT_UPDATED.event_id!) +
-                    "/" +
-                    encodeURIComponent(THREAD_RELATION_TYPE.name),
+                "/_matrix/client/v1/rooms/!foo%3Abar/relations/" + encodeURIComponent(THREAD_ROOT_UPDATED.event_id!),
             )
             .respond(200, {
                 chunk: [THREAD_REPLY3.event, THREAD_REPLY2.event, THREAD_REPLY],
@@ -1262,11 +1257,8 @@ describe("MatrixClient event timelines", function () {
                 "GET",
                 "/_matrix/client/v1/rooms/!foo%3Abar/relations/" +
                     encodeURIComponent(THREAD_ROOT_UPDATED.event_id!) +
-                    "/" +
-                    encodeURIComponent(THREAD_RELATION_TYPE.name) +
                     buildRelationPaginationQuery({
                         dir: Direction.Backward,
-                        limit: 3,
                         recurse: true,
                     }),
             )
@@ -1321,11 +1313,7 @@ describe("MatrixClient event timelines", function () {
         function respondToThread(root: Partial<IEvent>, replies: Partial<IEvent>[]): ExpectedHttpRequest {
             const request = httpBackend.when(
                 "GET",
-                "/_matrix/client/v1/rooms/!foo%3Abar/relations/" +
-                    encodeURIComponent(root.event_id!) +
-                    "/" +
-                    encodeURIComponent(THREAD_RELATION_TYPE.name) +
-                    "?dir=b&limit=1",
+                "/_matrix/client/v1/rooms/!foo%3Abar/relations/" + encodeURIComponent(root.event_id!) + "?dir=b",
             );
             request.respond(200, function () {
                 return {
@@ -1567,7 +1555,7 @@ describe("MatrixClient event timelines", function () {
                 expect(threadIds).toContain(THREAD2_ROOT.event_id);
                 const [allThreads] = timelineSets!;
                 const timeline = allThreads.getLiveTimeline()!;
-                // Test threads are in chronological order
+                // Test threads are in chronological order (first thread should be first because it has a more recent reply)
                 expect(timeline.getEvents().map((it) => it.event.event_id)).toEqual([
                     THREAD_ROOT.event_id,
                     THREAD2_ROOT.event_id,
@@ -2034,9 +2022,7 @@ describe("MatrixClient event timelines", function () {
                     "GET",
                     "/_matrix/client/v1/rooms/!foo%3Abar/relations/" +
                         encodeURIComponent(THREAD_ROOT.event_id!) +
-                        "/" +
-                        encodeURIComponent(THREAD_RELATION_TYPE.name) +
-                        buildRelationPaginationQuery({ dir: Direction.Backward, limit: 1 }),
+                        buildRelationPaginationQuery({ dir: Direction.Backward }),
                 )
                 .respond(200, function () {
                     return {

spec/integ/matrix-client-methods.spec.ts

@@ -1204,73 +1204,20 @@ describe("MatrixClient", function () {
 
     describe("requestLoginToken", () => {
         it("should hit the expected API endpoint with UIA", async () => {
-            jest.spyOn(client.http, "getUrl");
-            httpBackend
-                .when("GET", "/capabilities")
-                .respond(200, { capabilities: { "m.get_login_token": { enabled: true } } });
             const response = {};
             const uiaData = {};
             const prom = client.requestLoginToken(uiaData);
             httpBackend.when("POST", "/v1/login/get_token", { auth: uiaData }).respond(200, response);
             await httpBackend.flush("");
             expect(await prom).toStrictEqual(response);
-            expect(client.http.getUrl).toHaveLastReturnedWith(
-                expect.objectContaining({
-                    href: "http://alice.localhost.test.server/_matrix/client/v1/login/get_token",
-                }),
-            );
         });
 
         it("should hit the expected API endpoint without UIA", async () => {
-            jest.spyOn(client.http, "getUrl");
-            httpBackend
-                .when("GET", "/capabilities")
-                .respond(200, { capabilities: { "m.get_login_token": { enabled: true } } });
             const response = { login_token: "xyz", expires_in_ms: 5000 };
             const prom = client.requestLoginToken();
             httpBackend.when("POST", "/v1/login/get_token", {}).respond(200, response);
             await httpBackend.flush("");
-            // check that expires_in has been populated for compatibility with r0
-            expect(await prom).toStrictEqual({ ...response, expires_in: 5 });
-            expect(client.http.getUrl).toHaveLastReturnedWith(
-                expect.objectContaining({
-                    href: "http://alice.localhost.test.server/_matrix/client/v1/login/get_token",
-                }),
-            );
-        });
-
-        it("should still hit the stable endpoint when capability is disabled (but present)", async () => {
-            jest.spyOn(client.http, "getUrl");
-            httpBackend
-                .when("GET", "/capabilities")
-                .respond(200, { capabilities: { "m.get_login_token": { enabled: false } } });
-            const response = { login_token: "xyz", expires_in_ms: 5000 };
-            const prom = client.requestLoginToken();
-            httpBackend.when("POST", "/v1/login/get_token", {}).respond(200, response);
-            await httpBackend.flush("");
-            // check that expires_in has been populated for compatibility with r0
-            expect(await prom).toStrictEqual({ ...response, expires_in: 5 });
-            expect(client.http.getUrl).toHaveLastReturnedWith(
-                expect.objectContaining({
-                    href: "http://alice.localhost.test.server/_matrix/client/v1/login/get_token",
-                }),
-            );
-        });
-
-        it("should hit the r0 endpoint for fallback", async () => {
-            jest.spyOn(client.http, "getUrl");
-            httpBackend.when("GET", "/capabilities").respond(200, {});
-            const response = { login_token: "xyz", expires_in: 5 };
-            const prom = client.requestLoginToken();
-            httpBackend.when("POST", "/unstable/org.matrix.msc3882/login/token", {}).respond(200, response);
-            await httpBackend.flush("");
-            // check that expires_in has been populated for compatibility with r1
-            expect(await prom).toStrictEqual({ ...response, expires_in_ms: 5000 });
-            expect(client.http.getUrl).toHaveLastReturnedWith(
-                expect.objectContaining({
-                    href: "http://alice.localhost.test.server/_matrix/client/unstable/org.matrix.msc3882/login/token",
-                }),
-            );
+            expect(await prom).toStrictEqual(response);
         });
     });
 

spec/integ/matrix-client-unread-notifications.spec.ts

@@ -28,32 +28,70 @@ import {
     NotificationCountType,
     RelationType,
     Room,
+    fixNotificationCountOnDecryption,
 } from "../../src";
 import { TestClient } from "../TestClient";
 import { ReceiptType } from "../../src/@types/read_receipts";
 import { mkThread } from "../test-utils/thread";
 import { SyncState } from "../../src/sync";
 
+const userA = "@alice:localhost";
+const userB = "@bob:localhost";
+const selfUserId = userA;
+const selfAccessToken = "aseukfgwef";
+
+function setupTestClient(): [MatrixClient, HttpBackend] {
+    const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
+    const httpBackend = testClient.httpBackend;
+    const client = testClient.client;
+    httpBackend!.when("GET", "/versions").respond(200, {});
+    httpBackend!.when("GET", "/pushrules").respond(200, {});
+    httpBackend!.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
+    return [client, httpBackend];
+}
+
+describe("Notification count fixing", () => {
+    let client: MatrixClient | undefined;
+
+    beforeEach(() => {
+        [client] = setupTestClient();
+    });
+
+    it("doesn't increment notification count for events that can't be found in a room", async () => {
+        const roomId = "!room:localhost";
+
+        client!.startClient({ threadSupport: true });
+        const room = new Room(roomId, client!, selfUserId);
+        jest.spyOn(client!, "getRoom").mockImplementation((id) => (id === roomId ? room : null));
+
+        const event = new MatrixEvent({
+            room_id: roomId,
+            type: "m.reaction",
+            event_id: "$foo",
+            content: {
+                "m.relates_to": {
+                    rel_type: RelationType.Annotation,
+                    event_id: "$foo",
+                    key: "x",
+                },
+            },
+        });
+
+        jest.spyOn(event, "getPushActions").mockReturnValue({
+            notify: true,
+            tweaks: {},
+        });
+
+        fixNotificationCountOnDecryption(client!, event);
+
+        expect(room.getUnreadNotificationCount(NotificationCountType.Total)).toBe(0);
+    });
+});
+
 describe("MatrixClient syncing", () => {
-    const userA = "@alice:localhost";
-    const userB = "@bob:localhost";
-
-    const selfUserId = userA;
-    const selfAccessToken = "aseukfgwef";
-
     let client: MatrixClient | undefined;
     let httpBackend: HttpBackend | undefined;
 
-    const setupTestClient = (): [MatrixClient, HttpBackend] => {
-        const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
-        const httpBackend = testClient.httpBackend;
-        const client = testClient.client;
-        httpBackend!.when("GET", "/versions").respond(200, {});
-        httpBackend!.when("GET", "/pushrules").respond(200, {});
-        httpBackend!.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
-        return [client, httpBackend];
-    };
-
     beforeEach(() => {
         [client, httpBackend] = setupTestClient();
     });

spec/test-utils/AccountDataAccumulator.ts

@@ -0,0 +1,108 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import fetchMock from "fetch-mock-jest";
+import { MockOptionsMethodPut } from "fetch-mock";
+
+import { ISyncResponder } from "./SyncResponder";
+
+/**
+ *  An object which intercepts `account_data` get and set requests via fetch-mock.
+ */
+export class AccountDataAccumulator {
+    /**
+     * The account data events to be returned by the sync.
+     * Will be updated when fetchMock intercepts calls to PUT `/_matrix/client/v3/user/:userId/account_data/`.
+     * Will be used by `sendSyncResponseWithUpdatedAccountData`
+     */
+    public accountDataEvents: Map<String, any> = new Map();
+
+    /**
+     * Intercept requests to set a particular type of account data.
+     *
+     * Once it is set, its data is stored (for future return by `interceptGetAccountData` etc) and the resolved promise is
+     * resolved.
+     *
+     * @param accountDataType - type of account data to be intercepted
+     * @param opts - options to pass to fetchMock
+     * @returns a Promise which will resolve (with the content of the account data) once it is set.
+     */
+    public interceptSetAccountData(accountDataType: string, opts?: MockOptionsMethodPut): Promise<any> {
+        return new Promise((resolve) => {
+            // Called when the cross signing key is uploaded
+            fetchMock.put(
+                `express:/_matrix/client/v3/user/:userId/account_data/${accountDataType}`,
+                (url: string, options: RequestInit) => {
+                    const content = JSON.parse(options.body as string);
+                    const type = url.split("/").pop();
+                    // update account data for sync response
+                    this.accountDataEvents.set(type!, content);
+                    resolve(content);
+                    return {};
+                },
+                opts,
+            );
+        });
+    }
+
+    /**
+     * Intercept all requests to get account data
+     */
+    public interceptGetAccountData(): void {
+        fetchMock.get(
+            `express:/_matrix/client/v3/user/:userId/account_data/:type`,
+            (url) => {
+                const type = url.split("/").pop();
+                const existing = this.accountDataEvents.get(type!);
+                if (existing) {
+                    // return it
+                    return {
+                        status: 200,
+                        body: existing,
+                    };
+                } else {
+                    // 404
+                    return {
+                        status: 404,
+                        body: { errcode: "M_NOT_FOUND", error: "Account data not found." },
+                    };
+                }
+            },
+            { overwriteRoutes: true },
+        );
+    }
+
+    /**
+     * Send a sync response the current account data events.
+     */
+    public sendSyncResponseWithUpdatedAccountData(syncResponder: ISyncResponder): void {
+        try {
+            syncResponder.sendOrQueueSyncResponse({
+                next_batch: 1,
+                account_data: {
+                    events: Array.from(this.accountDataEvents, ([type, content]) => ({
+                        type: type,
+                        content: content,
+                    })),
+                },
+            });
+        } catch (err) {
+            // Might fail with "Cannot queue more than one /sync response" if called too often.
+            // It's ok if it fails here, the sync response is cumulative and will contain
+            // the latest account data.
+        }
+    }
+}

spec/test-utils/mockEndpoints.ts

@@ -24,11 +24,21 @@ import { KeyBackupInfo } from "../../src/crypto-api";
  * @param homeserverUrl - the homeserver url for the client under test
  */
 export function mockInitialApiRequests(homeserverUrl: string) {
-    fetchMock.getOnce(new URL("/_matrix/client/versions", homeserverUrl).toString(), { versions: ["v1.1"] });
-    fetchMock.getOnce(new URL("/_matrix/client/v3/pushrules/", homeserverUrl).toString(), {});
-    fetchMock.postOnce(new URL("/_matrix/client/v3/user/%40alice%3Alocalhost/filter", homeserverUrl).toString(), {
-        filter_id: "fid",
-    });
+    fetchMock.getOnce(
+        new URL("/_matrix/client/versions", homeserverUrl).toString(),
+        { versions: ["v1.1"] },
+        { overwriteRoutes: true },
+    );
+    fetchMock.getOnce(
+        new URL("/_matrix/client/v3/pushrules/", homeserverUrl).toString(),
+        {},
+        { overwriteRoutes: true },
+    );
+    fetchMock.postOnce(
+        new URL("/_matrix/client/v3/user/%40alice%3Alocalhost/filter", homeserverUrl).toString(),
+        { filter_id: "fid" },
+        { overwriteRoutes: true },
+    );
 }
 
 /**

spec/test-utils/test-utils.ts

@@ -315,6 +315,7 @@ export interface IMessageOpts {
     event?: boolean;
     relatesTo?: IEventRelation;
     ts?: number;
+    unsigned?: IUnsigned;
 }
 
 /**

spec/test-utils/test_indexeddb_cryptostore_dump/README.md

@@ -0,0 +1,53 @@
+## Dump of libolm indexeddb cryptostore
+
+This directory contains a dump of a real indexeddb store from a session using
+libolm crypto.
+
+The corresponding pickle key is `+1k2Ppd7HIisUY824v7JtV3/oEE4yX0TqtmNPyhaD7o`.
+
+It was created by pasting the following into the browser console:
+
+```javascript
+async function exportIndexedDb(name) {
+    const db = await new Promise((resolve, reject) => {
+        const dbReq = indexedDB.open(name);
+        dbReq.onerror = reject;
+        dbReq.onsuccess = () => resolve(dbReq.result);
+    });
+
+    const storeNames = db.objectStoreNames;
+    const exports = {};
+    for (const store of storeNames) {
+        exports[store] = [];
+        const txn = db.transaction(store, "readonly");
+        const objectStore = txn.objectStore(store);
+        await new Promise((resolve, reject) => {
+            const cursorReq = objectStore.openCursor();
+            cursorReq.onerror = reject;
+            cursorReq.onsuccess = (event) => {
+                const cursor = event.target.result;
+                if (cursor) {
+                    const entry = { value: cursor.value };
+                    if (!objectStore.keyPath) {
+                        entry.key = cursor.key;
+                    }
+                    exports[store].push(entry);
+                    cursor.continue();
+                } else {
+                    resolve();
+                }
+            };
+        });
+    }
+    return exports;
+}
+
+window.saveAs(
+    new Blob([JSON.stringify(await exportIndexedDb("matrix-js-sdk:crypto"), null, 2)], {
+        type: "application/json;charset=utf-8",
+    }),
+    "dump.json",
+);
+```
+
+The pickle key is extracted via `mxMatrixClientPeg.get().crypto.olmDevice.pickleKey`.

spec/test-utils/test_indexeddb_cryptostore_dump/index.ts

@@ -0,0 +1,136 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+
+/**
+ * Populate an IndexedDB store with the test data from this directory.
+ *
+ * @param name - Name of the IndexedDB database to create.
+ */
+export async function populateStore(name: string): Promise<IDBDatabase> {
+    const req = indexedDB.open(name, 11);
+
+    const db = await new Promise<IDBDatabase>((resolve, reject) => {
+        req.onupgradeneeded = (ev): void => {
+            const db = req.result;
+            const oldVersion = ev.oldVersion;
+            upgradeDatabase(oldVersion, db);
+        };
+
+        req.onerror = (ev): void => {
+            reject(req.error);
+        };
+
+        req.onsuccess = (): void => {
+            const db = req.result;
+            resolve(db);
+        };
+    });
+
+    await importData(db);
+
+    return db;
+}
+
+/** Create the schema for the indexed db store */
+function upgradeDatabase(oldVersion: number, db: IDBDatabase) {
+    if (oldVersion < 1) {
+        const outgoingRoomKeyRequestsStore = db.createObjectStore("outgoingRoomKeyRequests", { keyPath: "requestId" });
+        outgoingRoomKeyRequestsStore.createIndex("session", ["requestBody.room_id", "requestBody.session_id"]);
+        outgoingRoomKeyRequestsStore.createIndex("state", "state");
+    }
+
+    if (oldVersion < 2) {
+        db.createObjectStore("account");
+    }
+
+    if (oldVersion < 3) {
+        const sessionsStore = db.createObjectStore("sessions", { keyPath: ["deviceKey", "sessionId"] });
+        sessionsStore.createIndex("deviceKey", "deviceKey");
+    }
+
+    if (oldVersion < 4) {
+        db.createObjectStore("inbound_group_sessions", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 5) {
+        db.createObjectStore("device_data");
+    }
+
+    if (oldVersion < 6) {
+        db.createObjectStore("rooms");
+    }
+
+    if (oldVersion < 7) {
+        db.createObjectStore("sessions_needing_backup", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 8) {
+        db.createObjectStore("inbound_group_sessions_withheld", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 9) {
+        const problemsStore = db.createObjectStore("session_problems", { keyPath: ["deviceKey", "time"] });
+        problemsStore.createIndex("deviceKey", "deviceKey");
+
+        db.createObjectStore("notified_error_devices", { keyPath: ["userId", "deviceId"] });
+    }
+
+    if (oldVersion < 10) {
+        db.createObjectStore("shared_history_inbound_group_sessions", { keyPath: ["roomId"] });
+    }
+
+    if (oldVersion < 11) {
+        db.createObjectStore("parked_shared_history", { keyPath: ["roomId"] });
+    }
+}
+
+async function importData(db: IDBDatabase) {
+    const path = resolve("spec/test-utils/test_indexeddb_cryptostore_dump/dump.json");
+    const json: Record<string, Array<{ key?: any; value: any }>> = JSON.parse(
+        await readFile(path, { encoding: "utf8" }),
+    );
+
+    for (const [storeName, data] of Object.entries(json)) {
+        await new Promise((resolve, reject) => {
+            const store = db.transaction(storeName, "readwrite").objectStore(storeName);
+
+            function putEntry(idx: number) {
+                if (idx >= data.length) {
+                    resolve(undefined);
+                    return;
+                }
+
+                const { key, value } = data[idx];
+                try {
+                    const putReq = store.put(value, key);
+                    putReq.onsuccess = (_) => putEntry(idx + 1);
+                    putReq.onerror = (_) => reject(putReq.error);
+                } catch (e) {
+                    throw new Error(
+                        `Error populating '${storeName}' with key ${JSON.stringify(key)}, value ${JSON.stringify(
+                            value,
+                        )}: ${e}`,
+                    );
+                }
+            }
+
+            putEntry(0);
+        });
+    }
+}

spec/test-utils/thread.ts

@@ -161,3 +161,23 @@ export const mkThread = ({
 
     return { thread, rootEvent, events };
 };
+
+/**
+ * Create a thread, and make sure the events are added to the thread and the
+ * room's timeline as if they came in via sync.
+ *
+ * Note that mkThread doesn't actually add the events properly to the room.
+ */
+export const populateThread = ({
+    room,
+    client,
+    authorId,
+    participantUserIds,
+    length = 2,
+    ts = 1,
+}: MakeThreadProps): MakeThreadResult => {
+    const ret = mkThread({ room, client, authorId, participantUserIds, length, ts });
+    ret.thread.initialEventsFetched = true;
+    room.addLiveEvents(ret.events);
+    return ret;
+};

spec/test-utils/webrtc.ts

@@ -269,7 +269,11 @@ export class MockRTCRtpTransceiver {
 }
 
 export class MockMediaStreamTrack {
-    constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) {}
+    constructor(
+        public readonly id: string,
+        public readonly kind: "audio" | "video",
+        public enabled = true,
+    ) {}
 
     public stop = jest.fn<void, []>();
 
@@ -306,7 +310,10 @@ export class MockMediaStreamTrack {
 // XXX: Using EventTarget in jest doesn't seem to work, so we write our own
 // implementation
 export class MockMediaStream {
-    constructor(public id: string, private tracks: MockMediaStreamTrack[] = []) {}
+    constructor(
+        public id: string,
+        private tracks: MockMediaStreamTrack[] = [],
+    ) {}
 
     public listeners: [string, (...args: any[]) => any][] = [];
     public isStopped = false;
@@ -435,7 +442,11 @@ type EmittedEventMap = CallEventHandlerEventHandlerMap &
 export class MockCallMatrixClient extends TypedEventEmitter<EmittedEvents, EmittedEventMap> {
     public mediaHandler = new MockMediaHandler();
 
-    constructor(public userId: string, public deviceId: string, public sessionId: string) {
+    constructor(
+        public userId: string,
+        public deviceId: string,
+        public sessionId: string,
+    ) {
         super();
     }
 
@@ -502,7 +513,10 @@ export class MockCallMatrixClient extends TypedEventEmitter<EmittedEvents, Emitt
 }
 
 export class MockMatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap> {
-    constructor(public roomId: string, public groupCallId?: string) {
+    constructor(
+        public roomId: string,
+        public groupCallId?: string,
+    ) {
         super();
     }
 
@@ -550,7 +564,11 @@ export class MockMatrixCall extends TypedEventEmitter<CallEvent, CallEventHandle
 }
 
 export class MockCallFeed {
-    constructor(public userId: string, public deviceId: string | undefined, public stream: MockMediaStream) {}
+    constructor(
+        public userId: string,
+        public deviceId: string | undefined,
+        public stream: MockMediaStream,
+    ) {}
 
     public measureVolumeActivity(val: boolean) {}
     public dispose() {}

spec/unit/autodiscovery.spec.ts

@@ -351,7 +351,7 @@ describe("AutoDiscovery", function () {
         function () {
             const httpBackend = getHttpBackend();
             httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
-                not_matrix_versions: ["v1.1"],
+                not_matrix_versions: ["v1.5"],
             });
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
                 "m.homeserver": {
@@ -388,7 +388,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["v1.1"],
+                versions: ["v1.5"],
             });
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {
@@ -428,7 +428,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["v1.1"],
+                versions: ["v1.5"],
             });
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {
@@ -469,7 +469,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["v1.1"],
+                versions: ["v1.5"],
             });
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {
@@ -515,7 +515,7 @@ describe("AutoDiscovery", function () {
                     expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
                 })
                 .respond(200, {
-                    versions: ["v1.1"],
+                    versions: ["v1.5"],
                 });
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
                 "m.homeserver": {
@@ -560,7 +560,7 @@ describe("AutoDiscovery", function () {
                     expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
                 })
                 .respond(200, {
-                    versions: ["v1.1"],
+                    versions: ["v1.5"],
                 });
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
                 "m.homeserver": {
@@ -606,7 +606,7 @@ describe("AutoDiscovery", function () {
                     expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
                 })
                 .respond(200, {
-                    versions: ["v1.1"],
+                    versions: ["v1.5"],
                 });
             httpBackend.when("GET", "/_matrix/identity/v2").respond(404, {});
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
@@ -653,7 +653,7 @@ describe("AutoDiscovery", function () {
                     expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
                 })
                 .respond(200, {
-                    versions: ["v1.1"],
+                    versions: ["v1.5"],
                 });
             httpBackend.when("GET", "/_matrix/identity/v2").respond(500, {});
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
@@ -697,7 +697,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["v1.1"],
+                versions: ["v1.5"],
             });
         httpBackend
             .when("GET", "/_matrix/identity/v2")
@@ -747,7 +747,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["v1.1"],
+                versions: ["v1.5"],
             });
         httpBackend
             .when("GET", "/_matrix/identity/v2")
@@ -910,7 +910,7 @@ describe("AutoDiscovery", function () {
 
         beforeEach(() => {
             fetchMock.resetBehavior();
-            fetchMock.get(`${homeserverUrl}_matrix/client/versions`, { versions: ["v1.1"] });
+            fetchMock.get(`${homeserverUrl}_matrix/client/versions`, { versions: ["v1.5"] });
 
             fetchMock.get("https://example.org/.well-known/matrix/client", {
                 "m.homeserver": {

spec/unit/base64.spec.ts

@@ -17,7 +17,7 @@ limitations under the License.
 import { TextEncoder, TextDecoder } from "util";
 import NodeBuffer from "node:buffer";
 
-import { decodeBase64, encodeBase64, encodeUnpaddedBase64 } from "../../src/base64";
+import { decodeBase64, encodeBase64, encodeUnpaddedBase64, encodeUnpaddedBase64Url } from "../../src/base64";
 
 describe.each(["browser", "node"])("Base64 encoding (%s)", (env) => {
     let origBuffer = Buffer;
@@ -43,19 +43,27 @@ describe.each(["browser", "node"])("Base64 encoding (%s)", (env) => {
         global.btoa = undefined;
     });
 
-    it("Should decode properly encoded data", async () => {
+    it("Should decode properly encoded data", () => {
         const decoded = new TextDecoder().decode(decodeBase64("ZW5jb2RpbmcgaGVsbG8gd29ybGQ="));
 
         expect(decoded).toStrictEqual("encoding hello world");
     });
 
-    it("Should decode URL-safe base64", async () => {
+    it("Should encode unpadded URL-safe base64", () => {
+        const toEncode = "?????";
+        const data = new TextEncoder().encode(toEncode);
+
+        const encoded = encodeUnpaddedBase64Url(data);
+        expect(encoded).toEqual("Pz8_Pz8");
+    });
+
+    it("Should decode URL-safe base64", () => {
         const decoded = new TextDecoder().decode(decodeBase64("Pz8_Pz8="));
 
         expect(decoded).toStrictEqual("?????");
     });
 
-    it("Encode unpadded should not have padding", async () => {
+    it("Encode unpadded should not have padding", () => {
         const toEncode = "encoding hello world";
         const data = new TextEncoder().encode(toEncode);
 
@@ -68,7 +76,7 @@ describe.each(["browser", "node"])("Base64 encoding (%s)", (env) => {
         expect(padding).toStrictEqual("=");
     });
 
-    it("Decode should be indifferent to padding", async () => {
+    it("Decode should be indifferent to padding", () => {
         const withPadding = "ZW5jb2RpbmcgaGVsbG8gd29ybGQ=";
         const withoutPadding = "ZW5jb2RpbmcgaGVsbG8gd29ybGQ";
 

spec/unit/content-repo.spec.ts

@@ -37,6 +37,21 @@ describe("ContentRepo", function () {
             );
         });
 
+        it("should allow redirects when requested on download URLs", function () {
+            const mxcUri = "mxc://server.name/resourceid";
+            expect(getHttpUriForMxc(baseUrl, mxcUri, undefined, undefined, undefined, false, true)).toEqual(
+                baseUrl + "/_matrix/media/v3/download/server.name/resourceid?allow_redirect=true",
+            );
+        });
+
+        it("should allow redirects when requested on thumbnail URLs", function () {
+            const mxcUri = "mxc://server.name/resourceid";
+            expect(getHttpUriForMxc(baseUrl, mxcUri, 32, 32, "scale", false, true)).toEqual(
+                baseUrl +
+                    "/_matrix/media/v3/thumbnail/server.name/resourceid?width=32&height=32&method=scale&allow_redirect=true",
+            );
+        });
+
         it("should return the empty string for null input", function () {
             expect(getHttpUriForMxc(null as any, "")).toEqual("");
         });

spec/unit/crypto.spec.ts

@@ -356,7 +356,6 @@ describe("Crypto", function () {
 
         let crypto: Crypto;
         let mockBaseApis: MatrixClient;
-        let mockRoomList: RoomList;
 
         let fakeEmitter: EventEmitter;
 
@@ -390,19 +389,10 @@ describe("Crypto", function () {
                 isGuest: jest.fn(),
                 emit: jest.fn(),
             } as unknown as MatrixClient;
-            mockRoomList = {} as unknown as RoomList;
 
             fakeEmitter = new EventEmitter();
 
-            crypto = new Crypto(
-                mockBaseApis,
-                "@alice:home.server",
-                "FLIBBLE",
-                clientStore,
-                cryptoStore,
-                mockRoomList,
-                [],
-            );
+            crypto = new Crypto(mockBaseApis, "@alice:home.server", "FLIBBLE", clientStore, cryptoStore, []);
             crypto.registerEventHandlers(fakeEmitter as any);
             await crypto.init();
         });
@@ -1273,7 +1263,7 @@ describe("Crypto", function () {
                     ({
                         init_with_private_key: jest.fn(),
                         free,
-                    } as unknown as PkDecryption),
+                    }) as unknown as PkDecryption,
             );
             client.client.checkSecretStoragePrivateKey(new Uint8Array(), "");
             expect(free).toHaveBeenCalled();
@@ -1299,7 +1289,7 @@ describe("Crypto", function () {
                     ({
                         init_with_seed: jest.fn(),
                         free,
-                    } as unknown as PkSigning),
+                    }) as unknown as PkSigning,
             );
             client.client.checkCrossSigningPrivateKey(new Uint8Array(), "");
             expect(free).toHaveBeenCalled();
@@ -1341,15 +1331,9 @@ describe("Crypto", function () {
                 setRoomEncryption: jest.fn().mockResolvedValue(undefined),
             } as unknown as RoomList;
 
-            crypto = new Crypto(
-                mockClient,
-                "@alice:home.server",
-                "FLIBBLE",
-                clientStore,
-                cryptoStore,
-                mockRoomList,
-                [],
-            );
+            crypto = new Crypto(mockClient, "@alice:home.server", "FLIBBLE", clientStore, cryptoStore, []);
+            // @ts-ignore we are injecting a mock into a private property
+            crypto.roomList = mockRoomList;
         });
 
         it("should set the algorithm if called for a known room", async () => {

spec/unit/crypto/cross-signing.spec.ts

@@ -106,7 +106,7 @@ describe("Cross Signing", function () {
         });
         alice.uploadKeySignatures = async () => ({ failures: {} });
         alice.setAccountData = async () => ({});
-        alice.getAccountDataFromServer = async <T>() => ({} as T);
+        alice.getAccountDataFromServer = async <T>() => ({}) as T;
         // set Alice's cross-signing key
         await alice.bootstrapCrossSigning({
             authUploadDeviceSigningKeys: async (func) => {
@@ -146,7 +146,7 @@ describe("Cross Signing", function () {
         };
         alice.uploadKeySignatures = async () => ({ failures: {} });
         alice.setAccountData = async () => ({});
-        alice.getAccountDataFromServer = async <T extends { [k: string]: any }>(): Promise<T | null> => ({} as T);
+        alice.getAccountDataFromServer = async <T extends { [k: string]: any }>(): Promise<T | null> => ({}) as T;
         const authUploadDeviceSigningKeys: BootstrapCrossSigningOpts["authUploadDeviceSigningKeys"] = async (func) => {
             await func({});
         };

spec/unit/crypto/crypto-utils.ts

@@ -33,12 +33,10 @@ export async function resetCrossSigningKeys(
 
 export async function createSecretStorageKey(): Promise<IRecoveryKey> {
     const decryption = new global.Olm.PkDecryption();
-    const storagePublicKey = decryption.generate_key();
+    decryption.generate_key();
     const storagePrivateKey = decryption.get_private_key();
     decryption.free();
     return {
-        // `pubkey` not used anymore with symmetric 4S
-        keyInfo: { pubkey: storagePublicKey, key: undefined! },
         privateKey: storagePrivateKey,
     };
 }

spec/unit/crypto/secrets.spec.ts

@@ -190,10 +190,7 @@ describe("Secrets", function () {
         };
         resetCrossSigningKeys(alice);
 
-        const { keyId: newKeyId } = await alice.addSecretStorageKey(SECRET_STORAGE_ALGORITHM_V1_AES, {
-            pubkey: undefined,
-            key: undefined,
-        });
+        const { keyId: newKeyId } = await alice.addSecretStorageKey(SECRET_STORAGE_ALGORITHM_V1_AES, { key });
         // we don't await on this because it waits for the event to come down the sync
         // which won't happen in the test setup
         alice.setDefaultSecretStorageKeyId(newKeyId);
@@ -335,7 +332,6 @@ describe("Secrets", function () {
 
         it("bootstraps when cross-signing keys in secret storage", async function () {
             const decryption = new global.Olm.PkDecryption();
-            const storagePublicKey = decryption.generate_key();
             const storagePrivateKey = decryption.get_private_key();
 
             const bob: MatrixClient = await makeTestClient(
@@ -378,8 +374,6 @@ describe("Secrets", function () {
             });
             await bob.bootstrapSecretStorage({
                 createSecretStorageKey: async () => ({
-                    // `pubkey` not used anymore with symmetric 4S
-                    keyInfo: { pubkey: storagePublicKey },
                     privateKey: storagePrivateKey,
                 }),
             });

spec/unit/crypto/store/CryptoStore.spec.ts

@@ -0,0 +1,226 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import "fake-indexeddb/auto";
+import "jest-localstorage-mock";
+import { IndexedDBCryptoStore, LocalStorageCryptoStore, MemoryCryptoStore } from "../../../../src";
+import { CryptoStore, MigrationState, SESSION_BATCH_SIZE } from "../../../../src/crypto/store/base";
+
+describe.each([
+    ["IndexedDBCryptoStore", () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+    ["LocalStorageCryptoStore", () => new LocalStorageCryptoStore(localStorage)],
+    ["MemoryCryptoStore", () => new MemoryCryptoStore()],
+])("CryptoStore tests for %s", function (name, dbFactory) {
+    let store: CryptoStore;
+
+    beforeEach(async () => {
+        store = dbFactory();
+    });
+
+    describe("containsData", () => {
+        it("returns false at first", async () => {
+            expect(await store.containsData()).toBe(false);
+        });
+
+        it("returns true after startup and account setup", async () => {
+            await store.startup();
+            await store.doTxn("readwrite", [IndexedDBCryptoStore.STORE_ACCOUNT], (txn) => {
+                store.storeAccount(txn, "not a real account");
+            });
+            expect(await store.containsData()).toBe(true);
+        });
+    });
+
+    describe("migrationState", () => {
+        beforeEach(async () => {
+            await store.startup();
+        });
+
+        it("returns 0 at first", async () => {
+            expect(await store.getMigrationState()).toEqual(MigrationState.NOT_STARTED);
+        });
+
+        it("stores updates", async () => {
+            await store.setMigrationState(MigrationState.INITIAL_DATA_MIGRATED);
+            expect(await store.getMigrationState()).toEqual(MigrationState.INITIAL_DATA_MIGRATED);
+        });
+    });
+
+    describe("get/delete EndToEndSessionsBatch", () => {
+        beforeEach(async () => {
+            await store.startup();
+        });
+
+        it("returns null at first", async () => {
+            expect(await store.getEndToEndSessionsBatch()).toBe(null);
+        });
+
+        it("returns a batch of sessions", async () => {
+            // First store some sessions in the db
+            const N_DEVICES = 6;
+            const N_SESSIONS_PER_DEVICE = 6;
+            await createSessions(N_DEVICES, N_SESSIONS_PER_DEVICE);
+
+            let nSessions = 0;
+            await store.doTxn("readonly", [IndexedDBCryptoStore.STORE_SESSIONS], (txn) =>
+                store.countEndToEndSessions(txn, (n) => (nSessions = n)),
+            );
+            expect(nSessions).toEqual(N_DEVICES * N_SESSIONS_PER_DEVICE);
+
+            // Then, get a batch and check it looks right.
+            const batch = await store.getEndToEndSessionsBatch();
+            expect(batch!.length).toEqual(N_DEVICES * N_SESSIONS_PER_DEVICE);
+            for (let i = 0; i < N_DEVICES; i++) {
+                for (let j = 0; j < N_SESSIONS_PER_DEVICE; j++) {
+                    const r = batch![i * N_DEVICES + j];
+
+                    expect(r.deviceKey).toEqual(`device${i}`);
+                    expect(r.sessionId).toEqual(`session${j}`);
+                }
+            }
+        });
+
+        it("returns another batch of sessions after the first batch is deleted", async () => {
+            // First store some sessions in the db
+            const N_DEVICES = 8;
+            const N_SESSIONS_PER_DEVICE = 8;
+            await createSessions(N_DEVICES, N_SESSIONS_PER_DEVICE);
+
+            // Get the first batch
+            const batch = (await store.getEndToEndSessionsBatch())!;
+            expect(batch.length).toEqual(SESSION_BATCH_SIZE);
+
+            // ... and delete.
+            await store.deleteEndToEndSessionsBatch(batch);
+
+            // Fetch a second batch
+            const batch2 = (await store.getEndToEndSessionsBatch())!;
+            expect(batch2.length).toEqual(N_DEVICES * N_SESSIONS_PER_DEVICE - SESSION_BATCH_SIZE);
+
+            // ... and delete.
+            await store.deleteEndToEndSessionsBatch(batch2);
+
+            // the batch should now be null.
+            expect(await store.getEndToEndSessionsBatch()).toBe(null);
+        });
+
+        /** Create a bunch of fake Olm sessions and stash them in the DB. */
+        async function createSessions(nDevices: number, nSessionsPerDevice: number) {
+            await store.doTxn("readwrite", IndexedDBCryptoStore.STORE_SESSIONS, (txn) => {
+                for (let i = 0; i < nDevices; i++) {
+                    for (let j = 0; j < nSessionsPerDevice; j++) {
+                        store.storeEndToEndSession(
+                            `device${i}`,
+                            `session${j}`,
+                            {
+                                deviceKey: `device${i}`,
+                                sessionId: `session${j}`,
+                            },
+                            txn,
+                        );
+                    }
+                }
+            });
+        }
+    });
+
+    describe("get/delete EndToEndInboundGroupSessionsBatch", () => {
+        beforeEach(async () => {
+            await store.startup();
+        });
+
+        it("returns null at first", async () => {
+            expect(await store.getEndToEndInboundGroupSessionsBatch()).toBe(null);
+        });
+
+        it("returns a batch of sessions", async () => {
+            const N_DEVICES = 6;
+            const N_SESSIONS_PER_DEVICE = 6;
+            await createSessions(N_DEVICES, N_SESSIONS_PER_DEVICE);
+
+            // Mark one of the sessions as needing backup
+            await store.doTxn("readwrite", IndexedDBCryptoStore.STORE_BACKUP, async (txn) => {
+                await store.markSessionsNeedingBackup([{ senderKey: pad43("device5"), sessionId: "session5" }], txn);
+            });
+
+            expect(await store.countEndToEndInboundGroupSessions()).toEqual(N_DEVICES * N_SESSIONS_PER_DEVICE);
+
+            const batch = await store.getEndToEndInboundGroupSessionsBatch();
+            expect(batch!.length).toEqual(N_DEVICES * N_SESSIONS_PER_DEVICE);
+            for (let i = 0; i < N_DEVICES; i++) {
+                for (let j = 0; j < N_SESSIONS_PER_DEVICE; j++) {
+                    const r = batch![i * N_DEVICES + j];
+
+                    expect(r.senderKey).toEqual(pad43(`device${i}`));
+                    expect(r.sessionId).toEqual(`session${j}`);
+
+                    // only the last session needs backup
+                    expect(r.needsBackup).toBe(i === 5 && j === 5);
+                }
+            }
+        });
+
+        it("returns another batch of sessions after the first batch is deleted", async () => {
+            // First store some sessions in the db
+            const N_DEVICES = 8;
+            const N_SESSIONS_PER_DEVICE = 8;
+            await createSessions(N_DEVICES, N_SESSIONS_PER_DEVICE);
+
+            // Get the first batch
+            const batch = (await store.getEndToEndInboundGroupSessionsBatch())!;
+            expect(batch.length).toEqual(SESSION_BATCH_SIZE);
+
+            // ... and delete.
+            await store.deleteEndToEndInboundGroupSessionsBatch(batch);
+
+            // Fetch a second batch
+            const batch2 = (await store.getEndToEndInboundGroupSessionsBatch())!;
+            expect(batch2.length).toEqual(N_DEVICES * N_SESSIONS_PER_DEVICE - SESSION_BATCH_SIZE);
+
+            // ... and delete.
+            await store.deleteEndToEndInboundGroupSessionsBatch(batch2);
+
+            // the batch should now be null.
+            expect(await store.getEndToEndInboundGroupSessionsBatch()).toBe(null);
+        });
+
+        /** Create a bunch of fake megolm sessions and stash them in the DB. */
+        async function createSessions(nDevices: number, nSessionsPerDevice: number) {
+            await store.doTxn("readwrite", IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS, (txn) => {
+                for (let i = 0; i < nDevices; i++) {
+                    for (let j = 0; j < nSessionsPerDevice; j++) {
+                        store.storeEndToEndInboundGroupSession(
+                            pad43(`device${i}`),
+                            `session${j}`,
+                            {
+                                forwardingCurve25519KeyChain: [],
+                                keysClaimed: {},
+                                room_id: "",
+                                session: "",
+                            },
+                            txn,
+                        );
+                    }
+                }
+            });
+        }
+    });
+});
+
+/** Pad a string to 43 characters long */
+function pad43(x: string): string {
+    return x + ".".repeat(43 - x.length);
+}

spec/unit/crypto/store/IndexedDBCryptoStore.spec.ts

@@ -0,0 +1,73 @@
+/*
+Copyright 2024 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import "fake-indexeddb/auto";
+import { IndexedDBCryptoStore } from "../../../../src";
+import { MigrationState } from "../../../../src/crypto/store/base";
+
+describe("IndexedDBCryptoStore", () => {
+    describe("Test `existsAndIsNotMigrated`", () => {
+        beforeEach(async () => {
+            // eslint-disable-next-line no-global-assign
+            indexedDB = new IDBFactory();
+        });
+
+        it("Should be true if there is a legacy database", async () => {
+            // should detect a store that is not migrated
+            const store = new IndexedDBCryptoStore(global.indexedDB, "tests");
+            await store.startup();
+
+            const result = await IndexedDBCryptoStore.existsAndIsNotMigrated(global.indexedDB, "tests");
+
+            expect(result).toBe(true);
+        });
+
+        it("Should be true if there is a legacy database in non migrated state", async () => {
+            // should detect a store that is not migrated
+            const store = new IndexedDBCryptoStore(global.indexedDB, "tests");
+            await store.startup();
+            await store.setMigrationState(MigrationState.NOT_STARTED);
+
+            const result = await IndexedDBCryptoStore.existsAndIsNotMigrated(global.indexedDB, "tests");
+
+            expect(result).toBe(true);
+        });
+
+        describe.each([
+            MigrationState.INITIAL_DATA_MIGRATED,
+            MigrationState.OLM_SESSIONS_MIGRATED,
+            MigrationState.MEGOLM_SESSIONS_MIGRATED,
+            MigrationState.ROOM_SETTINGS_MIGRATED,
+        ])("Exists and Migration state is %s", (migrationState) => {
+            it("Should be false if migration has started", async () => {
+                // should detect a store that is not migrated
+                const store = new IndexedDBCryptoStore(global.indexedDB, "tests");
+                await store.startup();
+                await store.setMigrationState(migrationState);
+
+                const result = await IndexedDBCryptoStore.existsAndIsNotMigrated(global.indexedDB, "tests");
+
+                expect(result).toBe(false);
+            });
+        });
+
+        it("Should be false if there is no legacy database", async () => {
+            const result = await IndexedDBCryptoStore.existsAndIsNotMigrated(global.indexedDB, "tests");
+
+            expect(result).toBe(false);
+        });
+    });
+});

spec/unit/matrix-client.spec.ts

@@ -55,6 +55,7 @@ import {
     RuleId,
     IPushRule,
     ConditionKind,
+    getHttpUriForMxc,
 } from "../../src";
 import { supportsMatrixCall } from "../../src/webrtc/call";
 import { makeBeaconEvent } from "../test-utils/beacon";
@@ -64,7 +65,7 @@ import {
     PolicyScope,
 } from "../../src/models/invites-ignorer";
 import { IOlmDevice } from "../../src/crypto/algorithms/megolm";
-import { QueryDict } from "../../src/utils";
+import { defer, QueryDict } from "../../src/utils";
 import { SyncState } from "../../src/sync";
 import * as featureUtils from "../../src/feature";
 import { StubStore } from "../../src/store/stub";
@@ -369,6 +370,21 @@ describe("MatrixClient", function () {
         client.stopClient();
     });
 
+    describe("mxcUrlToHttp", () => {
+        it("should call getHttpUriForMxc", () => {
+            const mxc = "mxc://server/example";
+            expect(client.mxcUrlToHttp(mxc)).toBe(getHttpUriForMxc(client.baseUrl, mxc));
+            expect(client.mxcUrlToHttp(mxc, 32)).toBe(getHttpUriForMxc(client.baseUrl, mxc, 32));
+            expect(client.mxcUrlToHttp(mxc, 32, 46)).toBe(getHttpUriForMxc(client.baseUrl, mxc, 32, 46));
+            expect(client.mxcUrlToHttp(mxc, 32, 46, "scale")).toBe(
+                getHttpUriForMxc(client.baseUrl, mxc, 32, 46, "scale"),
+            );
+            expect(client.mxcUrlToHttp(mxc, 32, 46, "scale", false, true)).toBe(
+                getHttpUriForMxc(client.baseUrl, mxc, 32, 46, "scale", false, true),
+            );
+        });
+    });
+
     describe("timestampToEvent", () => {
         const roomId = "!room:server.org";
         const eventId = "$eventId:example.org";
@@ -1434,25 +1450,11 @@ describe("MatrixClient", function () {
         const mockRoom = {
             getMyMembership: () => "join",
             updatePendingEvent: (event: MatrixEvent, status: EventStatus) => event.setStatus(status),
-            currentState: {
-                getStateEvents: (eventType, stateKey) => {
-                    if (eventType === EventType.RoomCreate) {
-                        expect(stateKey).toEqual("");
-                        return new MatrixEvent({
-                            content: {
-                                [RoomCreateTypeField]: RoomType.Space,
-                            },
-                        });
-                    } else if (eventType === EventType.RoomEncryption) {
-                        expect(stateKey).toEqual("");
-                        return new MatrixEvent({ content: {} });
-                    } else {
-                        throw new Error("Unexpected event type or state key");
-                    }
-                },
-            } as Room["currentState"],
+            hasEncryptionStateEvent: jest.fn().mockReturnValue(true),
         } as unknown as Room;
 
+        let mockCrypto: Mocked<Crypto>;
+
         let event: MatrixEvent;
         beforeEach(async () => {
             event = new MatrixEvent({
@@ -1467,11 +1469,12 @@ describe("MatrixClient", function () {
                 expect(getRoomId).toEqual(roomId);
                 return mockRoom;
             };
-            client.crypto = client["cryptoBackend"] = {
-                // mock crypto
-                encryptEvent: () => new Promise(() => {}),
+            mockCrypto = {
+                isEncryptionEnabledInRoom: jest.fn().mockResolvedValue(true),
+                encryptEvent: jest.fn(),
                 stop: jest.fn(),
-            } as unknown as Crypto;
+            } as unknown as Mocked<Crypto>;
+            client.crypto = client["cryptoBackend"] = mockCrypto;
         });
 
         function assertCancelled() {
@@ -1488,12 +1491,21 @@ describe("MatrixClient", function () {
         });
 
         it("should cancel an event which is encrypting", async () => {
+            const encryptEventDefer = defer();
+            mockCrypto.encryptEvent.mockReturnValue(encryptEventDefer.promise);
+
+            const statusPromise = testUtils.emitPromise(event, "Event.status");
             // @ts-ignore protected method access
-            client.encryptAndSendEvent(mockRoom, event);
-            await testUtils.emitPromise(event, "Event.status");
+            const encryptAndSendPromise = client.encryptAndSendEvent(mockRoom, event);
+            await statusPromise;
             expect(event.status).toBe(EventStatus.ENCRYPTING);
             client.cancelPendingEvent(event);
             assertCancelled();
+
+            // now let the encryption complete, and check that the message is not sent.
+            encryptEventDefer.resolve();
+            await encryptAndSendPromise;
+            assertCancelled();
         });
 
         it("should cancel an event which is not sent", () => {
@@ -2211,8 +2223,7 @@ describe("MatrixClient", function () {
                     "org.matrix.msc3391": true,
                 },
             };
-            jest.spyOn(client.http, "request").mockResolvedValue(versionsResponse);
-            const requestSpy = jest.spyOn(client.http, "authedRequest").mockImplementation(() => Promise.resolve());
+            const requestSpy = jest.spyOn(client.http, "authedRequest").mockResolvedValue(versionsResponse);
             const unstablePrefix = "/_matrix/client/unstable/org.matrix.msc3391";
             const path = `/user/${encodeURIComponent(userId)}/account_data/${eventType}`;
 
@@ -2250,8 +2261,7 @@ describe("MatrixClient", function () {
                     "org.matrix.msc3391": false,
                 },
             };
-            jest.spyOn(client.http, "request").mockResolvedValue(versionsResponse);
-            const requestSpy = jest.spyOn(client.http, "authedRequest").mockImplementation(() => Promise.resolve());
+            const requestSpy = jest.spyOn(client.http, "authedRequest").mockResolvedValue(versionsResponse);
             const path = `/user/${encodeURIComponent(userId)}/account_data/${eventType}`;
 
             // populate version support

spec/unit/matrixrtc/CallMembership.spec.ts

@@ -34,9 +34,12 @@ function makeMockEvent(originTs = 0): MatrixEvent {
 }
 
 describe("CallMembership", () => {
-    it("rejects membership with no expiry", () => {
+    it("rejects membership with no expiry and no expires_ts", () => {
         expect(() => {
-            new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { expires: undefined }));
+            new CallMembership(
+                makeMockEvent(),
+                Object.assign({}, membershipTemplate, { expires: undefined, expires_ts: undefined }),
+            );
         }).toThrow();
     });
 
@@ -57,6 +60,16 @@ describe("CallMembership", () => {
             new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { scope: undefined }));
         }).toThrow();
     });
+    it("rejects with malformatted expires_ts", () => {
+        expect(() => {
+            new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { expires_ts: "string" }));
+        }).toThrow();
+    });
+    it("rejects with malformatted expires", () => {
+        expect(() => {
+            new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { expires: "string" }));
+        }).toThrow();
+    });
 
     it("uses event timestamp if no created_ts", () => {
         const membership = new CallMembership(makeMockEvent(12345), membershipTemplate);
@@ -71,11 +84,19 @@ describe("CallMembership", () => {
         expect(membership.createdTs()).toEqual(67890);
     });
 
-    it("computes absolute expiry time", () => {
+    it("computes absolute expiry time based on expires", () => {
         const membership = new CallMembership(makeMockEvent(1000), membershipTemplate);
         expect(membership.getAbsoluteExpiry()).toEqual(5000 + 1000);
     });
 
+    it("computes absolute expiry time based on expires_ts", () => {
+        const membership = new CallMembership(
+            makeMockEvent(1000),
+            Object.assign({}, membershipTemplate, { expires: undefined, expires_ts: 6000 }),
+        );
+        expect(membership.getAbsoluteExpiry()).toEqual(5000 + 1000);
+    });
+
     it("considers memberships unexpired if local age low enough", () => {
         const fakeEvent = makeMockEvent(1000);
         fakeEvent.getLocalAge = jest.fn().mockReturnValue(3000);
@@ -85,7 +106,7 @@ describe("CallMembership", () => {
 
     it("considers memberships expired when local age large", () => {
         const fakeEvent = makeMockEvent(1000);
-        fakeEvent.getLocalAge = jest.fn().mockReturnValue(6000);
+        fakeEvent.localTimestamp = Date.now() - 6000;
         const membership = new CallMembership(fakeEvent, membershipTemplate);
         expect(membership.isExpired()).toEqual(true);
     });

spec/unit/matrixrtc/MatrixRTCSession.spec.ts

@@ -14,11 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { EventTimeline, EventType, MatrixClient, Room } from "../../../src";
+import { EventTimeline, EventType, MatrixClient, MatrixError, MatrixEvent, Room } from "../../../src";
 import { CallMembershipData } from "../../../src/matrixrtc/CallMembership";
 import { MatrixRTCSession, MatrixRTCSessionEvent } from "../../../src/matrixrtc/MatrixRTCSession";
+import { EncryptionKeysEventContent } from "../../../src/matrixrtc/types";
 import { randomString } from "../../../src/randomstring";
-import { makeMockRoom, mockRTCEvent } from "./mocks";
+import { makeMockRoom, makeMockRoomState, mockRTCEvent } from "./mocks";
 
 const membershipTemplate: CallMembershipData = {
     call_id: "",
@@ -59,13 +60,14 @@ describe("MatrixRTCSession", () => {
         expect(sess?.memberships[0].deviceId).toEqual("AAAAAAA");
         expect(sess?.memberships[0].membershipID).toEqual("bloop");
         expect(sess?.memberships[0].isExpired()).toEqual(false);
+        expect(sess?.callId).toEqual("");
     });
 
     it("ignores expired memberships events", () => {
         const expiredMembership = Object.assign({}, membershipTemplate);
         expiredMembership.expires = 1000;
         expiredMembership.device_id = "EXPIRED";
-        const mockRoom = makeMockRoom([membershipTemplate, expiredMembership], () => 10000);
+        const mockRoom = makeMockRoom([membershipTemplate, expiredMembership], 10000);
 
         sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
         expect(sess?.memberships.length).toEqual(1);
@@ -184,8 +186,15 @@ describe("MatrixRTCSession", () => {
 
     describe("joining", () => {
         let mockRoom: Room;
+        let sendStateEventMock: jest.Mock;
+        let sendEventMock: jest.Mock;
 
         beforeEach(() => {
+            sendStateEventMock = jest.fn();
+            sendEventMock = jest.fn();
+            client.sendStateEvent = sendStateEventMock;
+            client.sendEvent = sendEventMock;
+
             mockRoom = makeMockRoom([]);
             sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
         });
@@ -205,10 +214,8 @@ describe("MatrixRTCSession", () => {
         });
 
         it("sends a membership event when joining a call", () => {
-            client.sendStateEvent = jest.fn();
-
+            jest.useFakeTimers();
             sess!.joinRoomSession([mockFocus]);
-
             expect(client.sendStateEvent).toHaveBeenCalledWith(
                 mockRoom!.roomId,
                 EventType.GroupCallMemberPrefix,
@@ -220,6 +227,7 @@ describe("MatrixRTCSession", () => {
                             call_id: "",
                             device_id: "AAAAAAA",
                             expires: 3600000,
+                            expires_ts: Date.now() + 3600000,
                             foci_active: [{ type: "mock" }],
                             membershipID: expect.stringMatching(".*"),
                         },
@@ -227,12 +235,10 @@ describe("MatrixRTCSession", () => {
                 },
                 "@alice:example.org",
             );
+            jest.useRealTimers();
         });
 
         it("does nothing if join called when already joined", () => {
-            const sendStateEventMock = jest.fn();
-            client.sendStateEvent = sendStateEventMock;
-
             sess!.joinRoomSession([mockFocus]);
 
             expect(client.sendStateEvent).toHaveBeenCalledTimes(1);
@@ -262,7 +268,7 @@ describe("MatrixRTCSession", () => {
                 const timeElapsed = 60 * 60 * 1000 - 1000;
                 mockRoom.getLiveTimeline().getState(EventTimeline.FORWARDS)!.getStateEvents = jest
                     .fn()
-                    .mockReturnValue(mockRTCEvent(eventContent.memberships, mockRoom.roomId, () => timeElapsed));
+                    .mockReturnValue(mockRTCEvent(eventContent.memberships, mockRoom.roomId, timeElapsed));
 
                 const eventReSentPromise = new Promise<Record<string, any>>((r) => {
                     resolveFn = (_roomId: string, _type: string, val: Record<string, any>) => {
@@ -287,6 +293,7 @@ describe("MatrixRTCSession", () => {
                                 call_id: "",
                                 device_id: "AAAAAAA",
                                 expires: 3600000 * 2,
+                                expires_ts: 1000 + 3600000 * 2,
                                 foci_active: [{ type: "mock" }],
                                 created_ts: 1000,
                                 membershipID: expect.stringMatching(".*"),
@@ -299,15 +306,244 @@ describe("MatrixRTCSession", () => {
                 jest.useRealTimers();
             }
         });
+
+        it("creates a key when joining", () => {
+            sess!.joinRoomSession([mockFocus], true);
+            const keys = sess?.getKeysForParticipant("@alice:example.org", "AAAAAAA");
+            expect(keys).toHaveLength(1);
+
+            const allKeys = sess!.getEncryptionKeys();
+            expect(allKeys).toBeTruthy();
+            expect(Array.from(allKeys)).toHaveLength(1);
+        });
+
+        it("sends keys when joining", async () => {
+            const eventSentPromise = new Promise((resolve) => {
+                sendEventMock.mockImplementation(resolve);
+            });
+
+            sess!.joinRoomSession([mockFocus], true);
+
+            await eventSentPromise;
+
+            expect(sendEventMock).toHaveBeenCalledWith(expect.stringMatching(".*"), "io.element.call.encryption_keys", {
+                call_id: "",
+                device_id: "AAAAAAA",
+                keys: [
+                    {
+                        index: 0,
+                        key: expect.stringMatching(".*"),
+                    },
+                ],
+            });
+        });
+
+        it("retries key sends", async () => {
+            jest.useFakeTimers();
+            let firstEventSent = false;
+
+            try {
+                const eventSentPromise = new Promise<void>((resolve) => {
+                    sendEventMock.mockImplementation(() => {
+                        if (!firstEventSent) {
+                            jest.advanceTimersByTime(10000);
+
+                            firstEventSent = true;
+                            const e = new Error() as MatrixError;
+                            e.data = {};
+                            throw e;
+                        } else {
+                            resolve();
+                        }
+                    });
+                });
+
+                sess!.joinRoomSession([mockFocus], true);
+                jest.advanceTimersByTime(10000);
+
+                await eventSentPromise;
+
+                expect(sendEventMock).toHaveBeenCalledTimes(2);
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+
+        it("cancels key send event that fail", async () => {
+            const eventSentinel = {} as unknown as MatrixEvent;
+
+            client.cancelPendingEvent = jest.fn();
+            sendEventMock.mockImplementation(() => {
+                const e = new Error() as MatrixError;
+                e.data = {};
+                e.event = eventSentinel;
+                throw e;
+            });
+
+            sess!.joinRoomSession([mockFocus], true);
+
+            expect(client.cancelPendingEvent).toHaveBeenCalledWith(eventSentinel);
+        });
+
+        it("Re-sends key if a new member joins", async () => {
+            jest.useFakeTimers();
+            try {
+                const mockRoom = makeMockRoom([membershipTemplate]);
+                sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+                const keysSentPromise1 = new Promise((resolve) => {
+                    sendEventMock.mockImplementation(resolve);
+                });
+
+                sess.joinRoomSession([mockFocus], true);
+                await keysSentPromise1;
+
+                sendEventMock.mockClear();
+                jest.advanceTimersByTime(10000);
+
+                const keysSentPromise2 = new Promise((resolve) => {
+                    sendEventMock.mockImplementation(resolve);
+                });
+
+                const onMembershipsChanged = jest.fn();
+                sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
+
+                const member2 = Object.assign({}, membershipTemplate, {
+                    device_id: "BBBBBBB",
+                });
+
+                mockRoom.getLiveTimeline().getState = jest
+                    .fn()
+                    .mockReturnValue(makeMockRoomState([membershipTemplate, member2], mockRoom.roomId, undefined));
+                sess.onMembershipUpdate();
+
+                await keysSentPromise2;
+
+                expect(sendEventMock).toHaveBeenCalled();
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+
+        it("Rotates key if a member leaves", async () => {
+            jest.useFakeTimers();
+            try {
+                const member2 = Object.assign({}, membershipTemplate, {
+                    device_id: "BBBBBBB",
+                });
+                const mockRoom = makeMockRoom([membershipTemplate, member2]);
+                sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+                const onMyEncryptionKeyChanged = jest.fn();
+                sess.on(
+                    MatrixRTCSessionEvent.EncryptionKeyChanged,
+                    (_key: Uint8Array, _idx: number, participantId: string) => {
+                        if (participantId === `${client.getUserId()}:${client.getDeviceId()}`) {
+                            onMyEncryptionKeyChanged();
+                        }
+                    },
+                );
+
+                const keysSentPromise1 = new Promise<EncryptionKeysEventContent>((resolve) => {
+                    sendEventMock.mockImplementation((_roomId, _evType, payload) => resolve(payload));
+                });
+
+                sess.joinRoomSession([mockFocus], true);
+                const firstKeysPayload = await keysSentPromise1;
+                expect(firstKeysPayload.keys).toHaveLength(1);
+
+                sendEventMock.mockClear();
+
+                const keysSentPromise2 = new Promise<EncryptionKeysEventContent>((resolve) => {
+                    sendEventMock.mockImplementation((_roomId, _evType, payload) => resolve(payload));
+                });
+
+                mockRoom.getLiveTimeline().getState = jest
+                    .fn()
+                    .mockReturnValue(makeMockRoomState([membershipTemplate], mockRoom.roomId, undefined));
+                sess.onMembershipUpdate();
+
+                jest.advanceTimersByTime(10000);
+
+                const secondKeysPayload = await keysSentPromise2;
+
+                expect(secondKeysPayload.keys).toHaveLength(2);
+                expect(onMyEncryptionKeyChanged).toHaveBeenCalledTimes(2);
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+
+        it("Doesn't re-send key immediately", async () => {
+            const realSetImmediate = setImmediate;
+            jest.useFakeTimers();
+            try {
+                const mockRoom = makeMockRoom([membershipTemplate]);
+                sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+                const keysSentPromise1 = new Promise((resolve) => {
+                    sendEventMock.mockImplementation(resolve);
+                });
+
+                sess.joinRoomSession([mockFocus], true);
+                await keysSentPromise1;
+
+                sendEventMock.mockClear();
+
+                const onMembershipsChanged = jest.fn();
+                sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
+
+                const member2 = Object.assign({}, membershipTemplate, {
+                    device_id: "BBBBBBB",
+                });
+
+                mockRoom.getLiveTimeline().getState = jest
+                    .fn()
+                    .mockReturnValue(makeMockRoomState([membershipTemplate, member2], mockRoom.roomId, undefined));
+                sess.onMembershipUpdate();
+
+                await new Promise((resolve) => {
+                    realSetImmediate(resolve);
+                });
+
+                expect(sendEventMock).not.toHaveBeenCalled();
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+    });
+
+    it("Does not emit if no membership changes", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+        const onMembershipsChanged = jest.fn();
+        sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
+        sess.onMembershipUpdate();
+
+        expect(onMembershipsChanged).not.toHaveBeenCalled();
+    });
+
+    it("Emits on membership changes", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+        const onMembershipsChanged = jest.fn();
+        sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
+
+        mockRoom.getLiveTimeline().getState = jest
+            .fn()
+            .mockReturnValue(makeMockRoomState([], mockRoom.roomId, undefined));
+        sess.onMembershipUpdate();
+
+        expect(onMembershipsChanged).toHaveBeenCalled();
     });
 
     it("emits an event at the time a membership event expires", () => {
         jest.useFakeTimers();
         try {
-            let eventAge = 0;
-
             const membership = Object.assign({}, membershipTemplate);
-            const mockRoom = makeMockRoom([membership], () => eventAge);
+            const mockRoom = makeMockRoom([membership], 0);
 
             sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
             const membershipObject = sess.memberships[0];
@@ -315,7 +551,6 @@ describe("MatrixRTCSession", () => {
             const onMembershipsChanged = jest.fn();
             sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
 
-            eventAge = 61 * 1000 * 1000;
             jest.advanceTimersByTime(61 * 1000 * 1000);
 
             expect(onMembershipsChanged).toHaveBeenCalledWith([membershipObject], []);
@@ -326,52 +561,55 @@ describe("MatrixRTCSession", () => {
     });
 
     it("prunes expired memberships on update", () => {
-        client.sendStateEvent = jest.fn();
+        jest.useFakeTimers();
+        try {
+            client.sendStateEvent = jest.fn();
 
-        let eventAge = 0;
-
-        const mockRoom = makeMockRoom(
-            [
+            const mockMemberships = [
                 Object.assign({}, membershipTemplate, {
                     device_id: "OTHERDEVICE",
                     expires: 1000,
                 }),
-            ],
-            () => eventAge,
-        );
-        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+            ];
+            const mockRoomNoExpired = makeMockRoom(mockMemberships, 0);
 
-        // sanity check
-        expect(sess.memberships).toHaveLength(1);
-        expect(sess.memberships[0].deviceId).toEqual("OTHERDEVICE");
+            sess = MatrixRTCSession.roomSessionForRoom(client, mockRoomNoExpired);
 
-        eventAge = 10000;
+            // sanity check
+            expect(sess.memberships).toHaveLength(1);
+            expect(sess.memberships[0].deviceId).toEqual("OTHERDEVICE");
 
-        sess.joinRoomSession([mockFocus]);
+            jest.advanceTimersByTime(10000);
 
-        expect(client.sendStateEvent).toHaveBeenCalledWith(
-            mockRoom!.roomId,
-            EventType.GroupCallMemberPrefix,
-            {
-                memberships: [
-                    {
-                        application: "m.call",
-                        scope: "m.room",
-                        call_id: "",
-                        device_id: "AAAAAAA",
-                        expires: 3600000,
-                        foci_active: [mockFocus],
-                        membershipID: expect.stringMatching(".*"),
-                    },
-                ],
-            },
-            "@alice:example.org",
-        );
+            sess.joinRoomSession([mockFocus]);
+
+            expect(client.sendStateEvent).toHaveBeenCalledWith(
+                mockRoomNoExpired!.roomId,
+                EventType.GroupCallMemberPrefix,
+                {
+                    memberships: [
+                        {
+                            application: "m.call",
+                            scope: "m.room",
+                            call_id: "",
+                            device_id: "AAAAAAA",
+                            expires: 3600000,
+                            expires_ts: Date.now() + 3600000,
+                            foci_active: [mockFocus],
+                            membershipID: expect.stringMatching(".*"),
+                        },
+                    ],
+                },
+                "@alice:example.org",
+            );
+        } finally {
+            jest.useRealTimers();
+        }
     });
 
     it("fills in created_ts for other memberships on update", () => {
         client.sendStateEvent = jest.fn();
-
+        jest.useFakeTimers();
         const mockRoom = makeMockRoom([
             Object.assign({}, membershipTemplate, {
                 device_id: "OTHERDEVICE",
@@ -401,6 +639,7 @@ describe("MatrixRTCSession", () => {
                         call_id: "",
                         device_id: "AAAAAAA",
                         expires: 3600000,
+                        expires_ts: Date.now() + 3600000,
                         foci_active: [mockFocus],
                         membershipID: expect.stringMatching(".*"),
                     },
@@ -408,5 +647,78 @@ describe("MatrixRTCSession", () => {
             },
             "@alice:example.org",
         );
+        jest.useRealTimers();
+    });
+
+    it("collects keys from encryption events", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        sess.onCallEncryption({
+            getType: jest.fn().mockReturnValue("io.element.call.encryption_keys"),
+            getContent: jest.fn().mockReturnValue({
+                device_id: "bobsphone",
+                call_id: "",
+                keys: [
+                    {
+                        index: 0,
+                        key: "dGhpcyBpcyB0aGUga2V5",
+                    },
+                ],
+            }),
+            getSender: jest.fn().mockReturnValue("@bob:example.org"),
+        } as unknown as MatrixEvent);
+
+        const bobKeys = sess.getKeysForParticipant("@bob:example.org", "bobsphone")!;
+        expect(bobKeys).toHaveLength(1);
+        expect(bobKeys[0]).toEqual(Buffer.from("this is the key", "utf-8"));
+    });
+
+    it("collects keys at non-zero indices", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        sess.onCallEncryption({
+            getType: jest.fn().mockReturnValue("io.element.call.encryption_keys"),
+            getContent: jest.fn().mockReturnValue({
+                device_id: "bobsphone",
+                call_id: "",
+                keys: [
+                    {
+                        index: 4,
+                        key: "dGhpcyBpcyB0aGUga2V5",
+                    },
+                ],
+            }),
+            getSender: jest.fn().mockReturnValue("@bob:example.org"),
+        } as unknown as MatrixEvent);
+
+        const bobKeys = sess.getKeysForParticipant("@bob:example.org", "bobsphone")!;
+        expect(bobKeys).toHaveLength(5);
+        expect(bobKeys[0]).toBeFalsy();
+        expect(bobKeys[1]).toBeFalsy();
+        expect(bobKeys[2]).toBeFalsy();
+        expect(bobKeys[3]).toBeFalsy();
+        expect(bobKeys[4]).toEqual(Buffer.from("this is the key", "utf-8"));
+    });
+
+    it("ignores keys event for the local participant", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        sess.onCallEncryption({
+            getType: jest.fn().mockReturnValue("io.element.call.encryption_keys"),
+            getContent: jest.fn().mockReturnValue({
+                device_id: client.getDeviceId(),
+                call_id: "",
+                keys: [
+                    {
+                        index: 4,
+                        key: "dGhpcyBpcyB0aGUga2V5",
+                    },
+                ],
+            }),
+            getSender: jest.fn().mockReturnValue(client.getUserId()),
+        } as unknown as MatrixEvent);
+
+        const myKeys = sess.getKeysForParticipant(client.getUserId()!, client.getDeviceId()!)!;
+        expect(myKeys).toBeFalsy();
     });
 });

spec/unit/matrixrtc/MatrixRTCSessionManager.spec.ts

@@ -14,7 +14,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { ClientEvent, EventTimeline, MatrixClient } from "../../../src";
+import {
+    ClientEvent,
+    EventTimeline,
+    EventType,
+    IRoomTimelineData,
+    MatrixClient,
+    MatrixEvent,
+    RoomEvent,
+} from "../../../src";
 import { RoomStateEvent } from "../../../src/models/room-state";
 import { CallMembershipData } from "../../../src/matrixrtc/CallMembership";
 import { MatrixRTCSessionManagerEvents } from "../../../src/matrixrtc/MatrixRTCSessionManager";
@@ -78,4 +86,26 @@ describe("MatrixRTCSessionManager", () => {
 
         expect(onEnded).toHaveBeenCalledWith(room1.roomId, client.matrixRTC.getActiveRoomSession(room1));
     });
+
+    it("Calls onCallEncryption on encryption keys event", () => {
+        const room1 = makeMockRoom([membershipTemplate]);
+        jest.spyOn(client, "getRooms").mockReturnValue([room1]);
+        jest.spyOn(client, "getRoom").mockReturnValue(room1);
+
+        client.emit(ClientEvent.Room, room1);
+        const onCallEncryptionMock = jest.fn();
+        client.matrixRTC.getRoomSession(room1).onCallEncryption = onCallEncryptionMock;
+
+        const timelineEvent = {
+            getType: jest.fn().mockReturnValue(EventType.CallEncryptionKeysPrefix),
+            getContent: jest.fn().mockReturnValue({}),
+            getSender: jest.fn().mockReturnValue("@mock:user.example"),
+            getRoomId: jest.fn().mockReturnValue("!room:id"),
+            sender: {
+                userId: "@mock:user.example",
+            },
+        } as unknown as MatrixEvent;
+        client.emit(RoomEvent.Timeline, timelineEvent, undefined, undefined, false, {} as IRoomTimelineData);
+        expect(onCallEncryptionMock).toHaveBeenCalled();
+    });
 });

spec/unit/matrixrtc/mocks.ts

@@ -18,37 +18,29 @@ import { EventType, MatrixEvent, Room } from "../../../src";
 import { CallMembershipData } from "../../../src/matrixrtc/CallMembership";
 import { randomString } from "../../../src/randomstring";
 
-export function makeMockRoom(
-    memberships: CallMembershipData[],
-    getLocalAge: (() => number) | undefined = undefined,
-): Room {
+export function makeMockRoom(memberships: CallMembershipData[], localAge: number | null = null): Room {
     const roomId = randomString(8);
+    // Caching roomState here so it does not get recreated when calling `getLiveTimeline.getState()`
+    const roomState = makeMockRoomState(memberships, roomId, localAge);
     return {
         roomId: roomId,
         getLiveTimeline: jest.fn().mockReturnValue({
-            getState: jest.fn().mockReturnValue(makeMockRoomState(memberships, roomId, getLocalAge)),
+            getState: jest.fn().mockReturnValue(roomState),
         }),
     } as unknown as Room;
 }
 
-function makeMockRoomState(memberships: CallMembershipData[], roomId: string, getLocalAge: (() => number) | undefined) {
+export function makeMockRoomState(memberships: CallMembershipData[], roomId: string, localAge: number | null = null) {
+    const event = mockRTCEvent(memberships, roomId, localAge);
     return {
         getStateEvents: (_: string, stateKey: string) => {
-            const event = mockRTCEvent(memberships, roomId, getLocalAge);
-
             if (stateKey !== undefined) return event;
             return [event];
         },
     };
 }
 
-export function mockRTCEvent(
-    memberships: CallMembershipData[],
-    roomId: string,
-    getLocalAge: (() => number) | undefined,
-): MatrixEvent {
-    const getLocalAgeFn = getLocalAge ?? (() => 10);
-
+export function mockRTCEvent(memberships: CallMembershipData[], roomId: string, localAge: number | null): MatrixEvent {
     return {
         getType: jest.fn().mockReturnValue(EventType.GroupCallMemberPrefix),
         getContent: jest.fn().mockReturnValue({
@@ -56,8 +48,7 @@ export function mockRTCEvent(
         }),
         getSender: jest.fn().mockReturnValue("@mock:user.example"),
         getTs: jest.fn().mockReturnValue(1000),
-        getLocalAge: getLocalAgeFn,
-        localTimestamp: Date.now(),
+        localTimestamp: Date.now() - (localAge ?? 10),
         getRoomId: jest.fn().mockReturnValue(roomId),
         sender: {
             userId: "@mock:user.example",

spec/unit/models/MSC3089Branch.spec.ts

@@ -213,7 +213,7 @@ describe("MSC3089Branch", () => {
                     expect(eventId).toEqual(fileEventId);
                     return fileEvent;
                 },
-            } as EventTimelineSet);
+            }) as EventTimelineSet;
         client.mxcUrlToHttp = (mxc: string) => {
             expect(mxc).toEqual("mxc://" + mxcLatter);
             return `https://example.org/_matrix/media/v1/download/${mxcLatter}`;
@@ -239,7 +239,7 @@ describe("MSC3089Branch", () => {
                     expect(eventId).toEqual(fileEventId);
                     return fileEvent;
                 },
-            } as EventTimelineSet);
+            }) as EventTimelineSet;
         client.mxcUrlToHttp = (mxc: string) => {
             expect(mxc).toEqual("mxc://" + mxcLatter);
             return `https://example.org/_matrix/media/v1/download/${mxcLatter}`;
@@ -332,7 +332,7 @@ describe("MSC3089Branch", () => {
                 getId: () => "$unknown",
             },
         ];
-        staticRoom.getLiveTimeline = () => ({ getEvents: () => events } as EventTimeline);
+        staticRoom.getLiveTimeline = () => ({ getEvents: () => events }) as EventTimeline;
 
         directory.getFile = (evId: string) => {
             expect(evId).toEqual(fileEventId);

spec/unit/models/MSC3089TreeSpace.spec.ts

@@ -399,7 +399,7 @@ describe("MSC3089TreeSpace", () => {
                 ];
             },
         };
-        client.getRoom = () => ({} as Room); // to appease the TreeSpace constructor
+        client.getRoom = () => ({}) as Room; // to appease the TreeSpace constructor
 
         const getFn = jest.fn().mockImplementation((roomId: string) => {
             if (roomId === thirdChildRoom) {
@@ -422,7 +422,7 @@ describe("MSC3089TreeSpace", () => {
     });
 
     it("should find specific directories", () => {
-        client.getRoom = () => ({} as Room); // to appease the TreeSpace constructor
+        client.getRoom = () => ({}) as Room; // to appease the TreeSpace constructor
 
         // Only mocking used API
         const firstSubdirectory = { roomId: "!first:example.org" } as any as MSC3089TreeSpace;

spec/unit/models/event.spec.ts

@@ -14,10 +14,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { MockedObject } from "jest-mock";
+
 import { MatrixEvent, MatrixEventEvent } from "../../../src/models/event";
 import { emitPromise } from "../../test-utils/test-utils";
 import { Crypto, IEventDecryptionResult } from "../../../src/crypto";
-import { IAnnotatedPushRule, PushRuleActionName, TweakName } from "../../../src";
+import {
+    IAnnotatedPushRule,
+    MatrixClient,
+    PushRuleActionName,
+    Room,
+    THREAD_RELATION_TYPE,
+    TweakName,
+} from "../../../src";
 
 describe("MatrixEvent", () => {
     it("should create copies of itself", () => {
@@ -61,31 +70,264 @@ describe("MatrixEvent", () => {
         expect(a.toSnapshot().isEquivalentTo(b)).toBe(false);
     });
 
-    it("should prune clearEvent when being redacted", () => {
-        const ev = new MatrixEvent({
-            type: "m.room.message",
-            content: {
-                body: "Test",
-            },
-            event_id: "$event1:server",
+    describe("redaction", () => {
+        it("should prune clearEvent when being redacted", () => {
+            const ev = createEvent("$event1:server", "Test");
+
+            expect(ev.getContent().body).toBe("Test");
+            expect(ev.getWireContent().body).toBe("Test");
+            ev.makeEncrypted("m.room.encrypted", { ciphertext: "xyz" }, "", "");
+            expect(ev.getContent().body).toBe("Test");
+            expect(ev.getWireContent().body).toBeUndefined();
+            expect(ev.getWireContent().ciphertext).toBe("xyz");
+
+            const mockClient = {} as unknown as MockedObject<MatrixClient>;
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const redaction = createRedaction(ev.getId()!);
+
+            ev.makeRedacted(redaction, room);
+            expect(ev.getContent().body).toBeUndefined();
+            expect(ev.getWireContent().body).toBeUndefined();
+            expect(ev.getWireContent().ciphertext).toBeUndefined();
         });
 
-        expect(ev.getContent().body).toBe("Test");
-        expect(ev.getWireContent().body).toBe("Test");
-        ev.makeEncrypted("m.room.encrypted", { ciphertext: "xyz" }, "", "");
-        expect(ev.getContent().body).toBe("Test");
-        expect(ev.getWireContent().body).toBeUndefined();
-        expect(ev.getWireContent().ciphertext).toBe("xyz");
+        it("should remain in the main timeline when redacted", async () => {
+            // Given an event in the main timeline
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const ev = createEvent("$event1:server");
 
-        const redaction = new MatrixEvent({
-            type: "m.room.redaction",
-            redacts: ev.getId(),
+            await room.addLiveEvents([ev]);
+            await room.createThreadsTimelineSets();
+            expect(ev.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([ev.getId()]);
+
+            // When I redact it
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then it remains in the main timeline
+            expect(ev.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([ev.getId()]);
         });
 
-        ev.makeRedacted(redaction);
-        expect(ev.getContent().body).toBeUndefined();
-        expect(ev.getWireContent().body).toBeUndefined();
-        expect(ev.getWireContent().ciphertext).toBeUndefined();
+        it("should keep thread roots in both timelines when redacted", async () => {
+            // Given a thread exists
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev]);
+            await room.createThreadsTimelineSets();
+            expect(threadRoot.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId()]);
+
+            // When I redact the thread root
+            const redaction = createRedaction(ev.getId()!);
+            threadRoot.makeRedacted(redaction, room);
+
+            // Then it remains in the main timeline and the thread
+            expect(threadRoot.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId()]);
+        });
+
+        it("should move into the main timeline when redacted", async () => {
+            // Given an event in a thread
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev]);
+            await room.createThreadsTimelineSets();
+            expect(ev.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId()]);
+
+            // When I redact it
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then it disappears from the thread and appears in the main timeline
+            expect(ev.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId(), ev.getId()]);
+            expect(threadLiveEventIds(room, 0)).not.toContain(ev.getId());
+        });
+
+        it("should move reactions to a redacted event into the main timeline", async () => {
+            // Given an event in a thread with a reaction
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+            const reaction = createReactionEvent("$reaction:server", ev.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev, reaction]);
+            await room.createThreadsTimelineSets();
+            expect(reaction.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId(), reaction.getId()]);
+
+            // When I redact the event
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then the reaction moves into the main timeline
+            expect(reaction.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId(), ev.getId(), reaction.getId()]);
+            expect(threadLiveEventIds(room, 0)).not.toContain(reaction.getId());
+        });
+
+        it("should move edits of a redacted event into the main timeline", async () => {
+            // Given an event in a thread with a reaction
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+            const edit = createEditEvent("$edit:server", ev.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev, edit]);
+            await room.createThreadsTimelineSets();
+            expect(edit.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId(), edit.getId()]);
+
+            // When I redact the event
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then the edit moves into the main timeline
+            expect(edit.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId(), ev.getId(), edit.getId()]);
+            expect(threadLiveEventIds(room, 0)).not.toContain(edit.getId());
+        });
+
+        it("should move reactions to replies to replies a redacted event into the main timeline", async () => {
+            // Given an event in a thread with a reaction
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+            const reply1 = createReplyEvent("$reply1:server", ev.getId()!);
+            const reply2 = createReplyEvent("$reply2:server", reply1.getId()!);
+            const reaction = createReactionEvent("$reaction:server", reply2.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev, reply1, reply2, reaction]);
+            await room.createThreadsTimelineSets();
+            expect(reaction.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([
+                threadRoot.getId(),
+                ev.getId(),
+                reply1.getId(),
+                reply2.getId(),
+                reaction.getId(),
+            ]);
+
+            // When I redact the event
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then the replies move to the main thread and the reaction disappears
+            expect(reaction.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([
+                threadRoot.getId(),
+                ev.getId(),
+                reply1.getId(),
+                reply2.getId(),
+                reaction.getId(),
+            ]);
+            expect(threadLiveEventIds(room, 0)).not.toContain(reply1.getId());
+            expect(threadLiveEventIds(room, 0)).not.toContain(reply2.getId());
+            expect(threadLiveEventIds(room, 0)).not.toContain(reaction.getId());
+        });
+
+        function createMockClient(): MatrixClient {
+            return {
+                supportsThreads: jest.fn().mockReturnValue(true),
+                decryptEventIfNeeded: jest.fn().mockReturnThis(),
+                getUserId: jest.fn().mockReturnValue("@user:server"),
+            } as unknown as MockedObject<MatrixClient>;
+        }
+
+        function createEvent(eventId: string, body?: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.message",
+                content: {
+                    body: body ?? eventId,
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createThreadedEvent(eventId: string, threadRootId: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.message",
+                content: {
+                    "body": eventId,
+                    "m.relates_to": {
+                        rel_type: THREAD_RELATION_TYPE.name,
+                        event_id: threadRootId,
+                    },
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createEditEvent(eventId: string, repliedToId: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.message",
+                content: {
+                    "body": "Edited",
+                    "m.new_content": {
+                        body: "Edited",
+                    },
+                    "m.relates_to": {
+                        event_id: repliedToId,
+                        rel_type: "m.replace",
+                    },
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createReplyEvent(eventId: string, repliedToId: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.message",
+                content: {
+                    "m.relates_to": {
+                        event_id: repliedToId,
+                        key: "x",
+                        rel_type: "m.in_reply_to",
+                    },
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createReactionEvent(eventId: string, reactedToId: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.reaction",
+                content: {
+                    "m.relates_to": {
+                        event_id: reactedToId,
+                        key: "x",
+                        rel_type: "m.annotation",
+                    },
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createRedaction(redactedEventid: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.redaction",
+                redacts: redactedEventid,
+            });
+        }
     });
 
     describe("applyVisibilityEvent", () => {
@@ -330,3 +572,19 @@ describe("MatrixEvent", () => {
         expect(stateEvent.threadRootId).toBeUndefined();
     });
 });
+
+function mainTimelineLiveEventIds(room: Room): Array<string> {
+    return room
+        .getLiveTimeline()
+        .getEvents()
+        .map((e) => e.getId()!);
+}
+
+function threadLiveEventIds(room: Room, threadIndex: number): Array<string> {
+    return room
+        .getThreads()
+        [threadIndex].getUnfilteredTimelineSet()
+        .getLiveTimeline()
+        .getEvents()
+        .map((e) => e.getId()!);
+}

spec/unit/models/room-receipts.spec.ts

@@ -0,0 +1,541 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { FeatureSupport, MatrixClient, MatrixEvent, ReceiptContent, THREAD_RELATION_TYPE, Thread } from "../../../src";
+import { Room } from "../../../src/models/room";
+
+/**
+ * Note, these tests check the functionality of the RoomReceipts class, but most
+ * of them access that functionality via the surrounding Room class, because a
+ * room is required for RoomReceipts to function, and this matches the pattern
+ * of how this code is used in the wild.
+ */
+describe("RoomReceipts", () => {
+    beforeAll(() => {
+        jest.replaceProperty(Thread, "hasServerSideSupport", FeatureSupport.Stable);
+    });
+
+    afterAll(() => {
+        jest.restoreAllMocks();
+    });
+
+    it("reports events unread if there are no receipts", () => {
+        // Given there are no receipts in the room
+        const room = createRoom();
+        const [event] = createEvent();
+        room.addLiveEvents([event]);
+
+        // When I ask about any event, then it is unread
+        expect(room.hasUserReadEvent(readerId, event.getId()!)).toBe(false);
+    });
+
+    it("reports events we sent as read even if there are no (real) receipts", () => {
+        // Given there are no receipts in the room
+        const room = createRoom();
+        const [event] = createEventSentBy(readerId);
+        room.addLiveEvents([event]);
+
+        // When I ask about an event I sent, it is read (because a synthetic
+        // receipt was created and stored in RoomReceipts)
+        expect(room.hasUserReadEvent(readerId, event.getId()!)).toBe(true);
+    });
+
+    it("reports read if we receive an unthreaded receipt for this event", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [event, eventId] = createEvent();
+        room.addLiveEvents([event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for this event+user
+        room.addReceipt(createReceipt(readerId, event));
+
+        // Then that event is read
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+    });
+
+    it("reports read if we receive an unthreaded receipt for a later event", () => {
+        // Given we have 2 events
+        const room = createRoom();
+        const [event1, event1Id] = createEvent();
+        const [event2] = createEvent();
+        room.addLiveEvents([event1, event2]);
+
+        // When we receive a receipt for the later event
+        room.addReceipt(createReceipt(readerId, event2));
+
+        // Then the earlier one is read
+        expect(room.hasUserReadEvent(readerId, event1Id)).toBe(true);
+    });
+
+    it("reports read for a non-live event if we receive an unthreaded receipt for a live one", () => {
+        // Given we have 2 events: one live and one old
+        const room = createRoom();
+        const [oldEvent, oldEventId] = createEvent();
+        const [liveEvent] = createEvent();
+        room.addLiveEvents([liveEvent]);
+        createOldTimeline(room, [oldEvent]);
+
+        // When we receive a receipt for the live event
+        room.addReceipt(createReceipt(readerId, liveEvent));
+
+        // Then the earlier one is read
+        expect(room.hasUserReadEvent(readerId, oldEventId)).toBe(true);
+    });
+
+    it("compares by timestamp if two events are in separate old timelines", () => {
+        // Given we have 2 events, both in old timelines, with event2 after
+        // event1 in terms of timestamps
+        const room = createRoom();
+        const [event1, event1Id] = createEvent();
+        const [event2, event2Id] = createEvent();
+        event1.event.origin_server_ts = 1;
+        event2.event.origin_server_ts = 2;
+        createOldTimeline(room, [event1]);
+        createOldTimeline(room, [event2]);
+
+        // When we receive a receipt for the older event
+        room.addReceipt(createReceipt(readerId, event1));
+
+        // Then the earlier one is read and the later one is not
+        expect(room.hasUserReadEvent(readerId, event1Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(false);
+    });
+
+    it("reports unread if we receive an unthreaded receipt for an earlier event", () => {
+        // Given we have 2 events
+        const room = createRoom();
+        const [event1] = createEvent();
+        const [event2, event2Id] = createEvent();
+        room.addLiveEvents([event1, event2]);
+
+        // When we receive a receipt for the earlier event
+        room.addReceipt(createReceipt(readerId, event1));
+
+        // Then the later one is unread
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(false);
+    });
+
+    it("reports unread if we receive an unthreaded receipt for a different user", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [event, eventId] = createEvent();
+        room.addLiveEvents([event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for another user
+        room.addReceipt(createReceipt(otherUserId, event));
+
+        // Then the event is still unread since the receipt was not for us
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // But it's read for the other person
+        expect(room.hasUserReadEvent(otherUserId, eventId)).toBe(true);
+    });
+
+    it("reports events we sent as read even if an earlier receipt arrives", () => {
+        // Given we sent an event after some other event
+        const room = createRoom();
+        const [previousEvent] = createEvent();
+        const [myEvent] = createEventSentBy(readerId);
+        room.addLiveEvents([previousEvent, myEvent]);
+
+        // And I just received a receipt for the previous event
+        room.addReceipt(createReceipt(readerId, previousEvent));
+
+        // When I ask about the event I sent, it is read (because of synthetic receipts)
+        expect(room.hasUserReadEvent(readerId, myEvent.getId()!)).toBe(true);
+    });
+
+    it("considers events after ones we sent to be unread", () => {
+        // Given we sent an event, then another event came in
+        const room = createRoom();
+        const [myEvent] = createEventSentBy(readerId);
+        const [laterEvent] = createEvent();
+        room.addLiveEvents([myEvent, laterEvent]);
+
+        // When I ask about the later event, it is unread (because it's after the synthetic receipt)
+        expect(room.hasUserReadEvent(readerId, laterEvent.getId()!)).toBe(false);
+    });
+
+    it("correctly reports readness even when receipts arrive out of order", () => {
+        // Given we have 3 events
+        const room = createRoom();
+        const [event1] = createEvent();
+        const [event2, event2Id] = createEvent();
+        const [event3, event3Id] = createEvent();
+        room.addLiveEvents([event1, event2, event3]);
+
+        // When we receive receipts for the older events out of order
+        room.addReceipt(createReceipt(readerId, event2));
+        room.addReceipt(createReceipt(readerId, event1));
+
+        // Then we correctly ignore the older receipt
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, event3Id)).toBe(false);
+    });
+
+    it("reports read if we receive a threaded receipt for this event (main)", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [event, eventId] = createEvent();
+        room.addLiveEvents([event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for this event+user
+        room.addReceipt(createThreadedReceipt(readerId, event, "main"));
+
+        // Then that event is read
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+    });
+
+    it("reports read if we receive a threaded receipt for this event (non-main)", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event, eventId] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for this event on this thread
+        room.addReceipt(createThreadedReceipt(readerId, event, rootId));
+
+        // Then that event is read
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+    });
+
+    it("reports read if we receive an threaded receipt for a later event", () => {
+        // Given we have 2 events in a thread
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event1, event1Id] = createThreadedEvent(root);
+        const [event2] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event1, event2]);
+
+        // When we receive a receipt for the later event
+        room.addReceipt(createThreadedReceipt(readerId, event2, rootId));
+
+        // Then the earlier one is read
+        expect(room.hasUserReadEvent(readerId, event1Id)).toBe(true);
+    });
+
+    it("reports unread if we receive an threaded receipt for an earlier event", () => {
+        // Given we have 2 events in a thread
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event1] = createThreadedEvent(root);
+        const [event2, event2Id] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event1, event2]);
+
+        // When we receive a receipt for the earlier event
+        room.addReceipt(createThreadedReceipt(readerId, event1, rootId));
+
+        // Then the later one is unread
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(false);
+    });
+
+    it("reports unread if we receive an threaded receipt for a different user", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event, eventId] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for another user
+        room.addReceipt(createThreadedReceipt(otherUserId, event, rootId));
+
+        // Then the event is still unread since the receipt was not for us
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // But it's read for the other person
+        expect(room.hasUserReadEvent(otherUserId, eventId)).toBe(true);
+    });
+
+    it("reports unread if we receive a receipt for a later event in a different thread", () => {
+        // Given 2 events exist in different threads
+        const room = createRoom();
+        const [root1] = createEvent();
+        const [root2] = createEvent();
+        const [thread1, thread1Id] = createThreadedEvent(root1);
+        const [thread2] = createThreadedEvent(root2);
+        setupThread(room, root1);
+        setupThread(room, root2);
+        room.addLiveEvents([root1, root2, thread1, thread2]);
+
+        // When we receive a receipt for the later event
+        room.addReceipt(createThreadedReceipt(readerId, thread2, root2.getId()!));
+
+        // Then the old one is still unread since the receipt was not for this thread
+        expect(room.hasUserReadEvent(readerId, thread1Id)).toBe(false);
+    });
+
+    it("correctly reports readness even when threaded receipts arrive out of order", () => {
+        // Given we have 3 events
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event1] = createThreadedEvent(root);
+        const [event2, event2Id] = createThreadedEvent(root);
+        const [event3, event3Id] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event1, event2, event3]);
+
+        // When we receive receipts for the older events out of order
+        room.addReceipt(createThreadedReceipt(readerId, event2, rootId));
+        room.addReceipt(createThreadedReceipt(readerId, event1, rootId));
+
+        // Then we correctly ignore the older receipt
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, event3Id)).toBe(false);
+    });
+
+    it("correctly reports readness when mixing threaded and unthreaded receipts", () => {
+        // Given we have a setup from this presentation:
+        // https://docs.google.com/presentation/d/1H1gxRmRFAm8d71hCILWmpOYezsvdlb7cB6ANl-20Gns/edit?usp=sharing
+        //
+        //                     Main1----\
+        //                       |       ---Thread1a <- threaded receipt
+        //                       |             |
+        //                       |          Thread1b
+        // threaded receipt -> Main2--\
+        //                       |     ----------------Thread2a <- unthreaded receipt
+        //                     Main3                      |
+        //                                             Thread2b <- threaded receipt
+        //
+        const room = createRoom();
+        const [main1, main1Id] = createEvent();
+        const [main2, main2Id] = createEvent();
+        const [main3, main3Id] = createEvent();
+        const [thread1a, thread1aId] = createThreadedEvent(main1);
+        const [thread1b, thread1bId] = createThreadedEvent(main1);
+        const [thread2a, thread2aId] = createThreadedEvent(main2);
+        const [thread2b, thread2bId] = createThreadedEvent(main2);
+        setupThread(room, main1);
+        setupThread(room, main2);
+        room.addLiveEvents([main1, thread1a, thread1b, main2, thread2a, main3, thread2b]);
+
+        // And the timestamps on the events are consistent with the order above
+        main1.event.origin_server_ts = 1;
+        thread1a.event.origin_server_ts = 2;
+        thread1b.event.origin_server_ts = 3;
+        main2.event.origin_server_ts = 4;
+        thread2a.event.origin_server_ts = 5;
+        main3.event.origin_server_ts = 6;
+        thread2b.event.origin_server_ts = 7;
+        // (Note: in principle, we have the information needed to order these
+        // events without using their timestamps, since they all came in via
+        // addLiveEvents. In reality, some of them would have come in via the
+        // /relations API, making it impossible to get the correct ordering
+        // without MSC4033, which is why we fall back to timestamps. I.e. we
+        // definitely could fix the code to make the above
+        // timestamp-manipulation unnecessary, but it would only make this test
+        // neater, not actually help in the real world.)
+
+        // When the receipts arrive
+        room.addReceipt(createThreadedReceipt(readerId, main2, "main"));
+        room.addReceipt(createThreadedReceipt(readerId, thread1a, main1Id));
+        room.addReceipt(createReceipt(readerId, thread2a));
+        room.addReceipt(createThreadedReceipt(readerId, thread2b, main2Id));
+
+        // Then we correctly identify that only main3 is unread
+        expect(room.hasUserReadEvent(readerId, main1Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, main2Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, main3Id)).toBe(false);
+        expect(room.hasUserReadEvent(readerId, thread1aId)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, thread1bId)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, thread2aId)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, thread2bId)).toBe(true);
+    });
+
+    describe("dangling receipts", () => {
+        it("reports unread if the unthreaded receipt is in a dangling state", () => {
+            const room = createRoom();
+            const [event, eventId] = createEvent();
+            // When we receive a receipt for this event+user
+            room.addReceipt(createReceipt(readerId, event));
+
+            // The event is not added in the room
+            // So the receipt is in a dangling state
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+            // Add the event to the room
+            // The receipt is removed from the dangling state
+            room.addLiveEvents([event]);
+
+            // Then the event is read
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+        });
+
+        it("reports unread if the threaded receipt is in a dangling state", () => {
+            const room = createRoom();
+            const [root, rootId] = createEvent();
+            const [event, eventId] = createThreadedEvent(root);
+            setupThread(room, root);
+
+            // When we receive a receipt for this event+user
+            room.addReceipt(createThreadedReceipt(readerId, event, rootId));
+
+            // The event is not added in the room
+            // So the receipt is in a dangling state
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+            // Add the events to the room
+            // The receipt is removed from the dangling state
+            room.addLiveEvents([root, event]);
+
+            // Then the event is read
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+        });
+
+        it("should handle multiple dangling receipts for the same event", () => {
+            const room = createRoom();
+            const [event, eventId] = createEvent();
+            // When we receive a receipt for this event+user
+            room.addReceipt(createReceipt(readerId, event));
+            // We receive another receipt in the same event for another user
+            room.addReceipt(createReceipt(otherUserId, event));
+
+            // The event is not added in the room
+            // So the receipt is in a dangling state
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+            // Add the event to the room
+            // The two receipts should be processed
+            room.addLiveEvents([event]);
+
+            // Then the event is read
+            // We expect that the receipt of `otherUserId` didn't replace/erase the receipt of `readerId`
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+        });
+    });
+});
+
+function createFakeClient(): MatrixClient {
+    return {
+        getUserId: jest.fn(),
+        getEventMapper: jest.fn().mockReturnValue(jest.fn()),
+        isInitialSyncComplete: jest.fn().mockReturnValue(true),
+        supportsThreads: jest.fn().mockReturnValue(true),
+        fetchRoomEvent: jest.fn().mockResolvedValue({}),
+        paginateEventTimeline: jest.fn(),
+        canSupport: { get: jest.fn() },
+    } as unknown as MatrixClient;
+}
+
+const senderId = "sender:s.ss";
+const readerId = "reader:r.rr";
+const otherUserId = "other:o.oo";
+
+function createRoom(): Room {
+    return new Room("!rid", createFakeClient(), "@u:s.nz", { timelineSupport: true });
+}
+
+let idCounter = 0;
+function nextId(): string {
+    return "$" + (idCounter++).toString(10);
+}
+
+/**
+ * Create an event and return it and its ID.
+ */
+function createEvent(): [MatrixEvent, string] {
+    return createEventSentBy(senderId);
+}
+
+/**
+ * Create an event with the supplied sender and return it and its ID.
+ */
+function createEventSentBy(customSenderId: string): [MatrixEvent, string] {
+    const event = new MatrixEvent({ sender: customSenderId, event_id: nextId() });
+    return [event, event.getId()!];
+}
+
+/**
+ * Create an event in the thread of the supplied root and return it and its ID.
+ */
+function createThreadedEvent(root: MatrixEvent): [MatrixEvent, string] {
+    const rootEventId = root.getId()!;
+    const event = new MatrixEvent({
+        sender: senderId,
+        event_id: nextId(),
+        content: {
+            "m.relates_to": {
+                event_id: rootEventId,
+                rel_type: THREAD_RELATION_TYPE.name,
+                ["m.in_reply_to"]: {
+                    event_id: rootEventId,
+                },
+            },
+        },
+    });
+    return [event, event.getId()!];
+}
+
+function createReceipt(userId: string, referencedEvent: MatrixEvent): MatrixEvent {
+    const content: ReceiptContent = {
+        [referencedEvent.getId()!]: {
+            "m.read": {
+                [userId]: {
+                    ts: 123,
+                },
+            },
+        },
+    };
+
+    return new MatrixEvent({
+        type: "m.receipt",
+        content,
+    });
+}
+
+function createThreadedReceipt(userId: string, referencedEvent: MatrixEvent, threadId: string): MatrixEvent {
+    const content: ReceiptContent = {
+        [referencedEvent.getId()!]: {
+            "m.read": {
+                [userId]: {
+                    ts: 123,
+                    thread_id: threadId,
+                },
+            },
+        },
+    };
+
+    return new MatrixEvent({
+        type: "m.receipt",
+        content,
+    });
+}
+
+/**
+ * Create a timeline in the timeline set that is not the live timeline.
+ */
+function createOldTimeline(room: Room, events: MatrixEvent[]) {
+    const oldTimeline = room.getUnfilteredTimelineSet().addTimeline();
+    room.getUnfilteredTimelineSet().addEventsToTimeline(events, true, oldTimeline);
+}
+
+/**
+ * Perform the hacks required for this room to create a thread based on the root
+ * event supplied.
+ */
+function setupThread(room: Room, root: MatrixEvent) {
+    const thread = room.createThread(root.getId()!, root, [root], false);
+    thread.initialEventsFetched = true;
+}

spec/unit/models/thread.spec.ts

@@ -19,7 +19,7 @@ import { mocked } from "jest-mock";
 import { MatrixClient, PendingEventOrdering } from "../../../src/client";
 import { Room, RoomEvent } from "../../../src/models/room";
 import { FeatureSupport, Thread, THREAD_RELATION_TYPE, ThreadEvent } from "../../../src/models/thread";
-import { makeThreadEvent, mkThread } from "../../test-utils/thread";
+import { makeThreadEvent, mkThread, populateThread } from "../../test-utils/thread";
 import { TestClient } from "../../TestClient";
 import { emitPromise, mkEdit, mkMessage, mkReaction, mock } from "../../test-utils/test-utils";
 import { Direction, EventStatus, EventType, MatrixEvent } from "../../../src";
@@ -149,20 +149,38 @@ describe("Thread", () => {
         });
 
         it("considers other events with no RR as unread", () => {
-            const { thread, events } = mkThread({
+            // Given a long thread exists
+            const { thread, events } = populateThread({
                 room,
                 client,
-                authorId: myUserId,
-                participantUserIds: [myUserId],
+                authorId: "@other:foo.com",
+                participantUserIds: ["@other:foo.com"],
                 length: 25,
                 ts: 190,
             });
 
-            // Before alice's last unthreaded receipt
-            expect(thread.hasUserReadEvent("@alice:example.org", events.at(1)!.getId() ?? "")).toBeTruthy();
+            const event1 = events.at(1)!;
+            const event2 = events.at(2)!;
+            const event24 = events.at(24)!;
 
-            // After alice's last unthreaded receipt
-            expect(thread.hasUserReadEvent("@alice:example.org", events.at(-1)!.getId() ?? "")).toBeFalsy();
+            // And we have read the second message in it with an unthreaded receipt
+            const receipt = new MatrixEvent({
+                type: "m.receipt",
+                room_id: room.roomId,
+                content: {
+                    // unthreaded receipt for the second message in the thread
+                    [event2.getId()!]: {
+                        [ReceiptType.Read]: {
+                            [myUserId]: { ts: 200 },
+                        },
+                    },
+                },
+            });
+            room.addReceipt(receipt);
+
+            // Then we have read the first message in the thread, and not the last
+            expect(thread.hasUserReadEvent(myUserId, event1.getId()!)).toBe(true);
+            expect(thread.hasUserReadEvent(myUserId, event24.getId()!)).toBe(false);
         });
 
         it("considers event as read if there's a more recent unthreaded receipt", () => {
@@ -481,13 +499,13 @@ describe("Thread", () => {
 
                 // And a thread with an added event (with later timestamp)
                 const userId = "user1";
-                const { thread, message } = await createThreadAndEvent(client, 1, 100, userId);
+                const { thread, message2 } = await createThreadAnd2Events(client, 1, 100, 200, userId);
 
                 // Then a receipt was added to the thread
                 const receipt = thread.getReadReceiptForUserId(userId);
                 expect(receipt).toBeTruthy();
-                expect(receipt?.eventId).toEqual(message.getId());
-                expect(receipt?.data.ts).toEqual(100);
+                expect(receipt?.eventId).toEqual(message2.getId());
+                expect(receipt?.data.ts).toEqual(200);
                 expect(receipt?.data.thread_id).toEqual(thread.id);
 
                 // (And the receipt was synthetic)
@@ -505,14 +523,14 @@ describe("Thread", () => {
 
                 // And a thread with an added event with a lower timestamp than its other events
                 const userId = "user1";
-                const { thread } = await createThreadAndEvent(client, 200, 100, userId);
+                const { thread, message1 } = await createThreadAnd2Events(client, 300, 200, 100, userId);
 
-                // Then no receipt was added to the thread (the receipt is still
-                // for the thread root). This happens because since we have no
+                // Then the receipt is for the first message, because its
+                // timestamp is later. This happens because since we have no
                 // recursive relations support, we know that sometimes events
                 // appear out of order, so we have to check their timestamps as
                 // a guess of the correct order.
-                expect(thread.getReadReceiptForUserId(userId)?.eventId).toEqual(thread.rootEvent?.getId());
+                expect(thread.getReadReceiptForUserId(userId)?.eventId).toEqual(message1.getId());
             });
         });
 
@@ -530,11 +548,11 @@ describe("Thread", () => {
 
                 // And a thread with an added event (with later timestamp)
                 const userId = "user1";
-                const { thread, message } = await createThreadAndEvent(client, 1, 100, userId);
+                const { thread, message2 } = await createThreadAnd2Events(client, 1, 100, 200, userId);
 
                 // Then a receipt was added to the thread
                 const receipt = thread.getReadReceiptForUserId(userId);
-                expect(receipt?.eventId).toEqual(message.getId());
+                expect(receipt?.eventId).toEqual(message2.getId());
             });
 
             it("Creates a local echo receipt even for events BEFORE an existing receipt", async () => {
@@ -550,22 +568,24 @@ describe("Thread", () => {
 
                 // And a thread with an added event with a lower timestamp than its other events
                 const userId = "user1";
-                const { thread, message } = await createThreadAndEvent(client, 200, 100, userId);
+                const { thread, message2 } = await createThreadAnd2Events(client, 300, 200, 100, userId);
 
-                // Then a receipt was added to the thread, because relations
-                // recursion is available, so we trust the server to have
-                // provided us with events in the right order.
+                // Then a receipt was added for the last message, even though it
+                // has lower ts, because relations recursion is available, so we
+                // trust the server to have provided us with events in the right
+                // order.
                 const receipt = thread.getReadReceiptForUserId(userId);
-                expect(receipt?.eventId).toEqual(message.getId());
+                expect(receipt?.eventId).toEqual(message2.getId());
             });
         });
 
-        async function createThreadAndEvent(
+        async function createThreadAnd2Events(
             client: MatrixClient,
             rootTs: number,
-            eventTs: number,
+            message1Ts: number,
+            message2Ts: number,
             userId: string,
-        ): Promise<{ thread: Thread; message: MatrixEvent }> {
+        ): Promise<{ thread: Thread; message1: MatrixEvent; message2: MatrixEvent }> {
             const room = new Room("room1", client, userId);
 
             // Given a thread
@@ -576,24 +596,41 @@ describe("Thread", () => {
                 participantUserIds: [],
                 ts: rootTs,
             });
-            // Sanity: the current receipt is for the thread root
-            expect(thread.getReadReceiptForUserId(userId)?.eventId).toEqual(thread.rootEvent?.getId());
+            // Sanity: there is no read receipt on the thread yet because the
+            // thread events don't get properly added to the room by mkThread.
+            expect(thread.getReadReceiptForUserId(userId)).toBeNull();
 
             const awaitTimelineEvent = new Promise<void>((res) => thread.on(RoomEvent.Timeline, () => res()));
 
-            // When we add a message that is before the latest receipt
-            const message = makeThreadEvent({
+            // Add a message with ts message1Ts
+            const message1 = makeThreadEvent({
                 event: true,
                 rootEventId: thread.id,
                 replyToEventId: thread.id,
                 user: userId,
                 room: room.roomId,
-                ts: eventTs,
+                ts: message1Ts,
             });
-            await thread.addEvent(message, false, true);
+            await thread.addEvent(message1, false, true);
             await awaitTimelineEvent;
 
-            return { thread, message };
+            // Sanity: the thread now has a properly-added event, so this event
+            // has a synthetic receipt.
+            expect(thread.getReadReceiptForUserId(userId)?.eventId).toEqual(message1.getId());
+
+            // Add a message with ts message2Ts
+            const message2 = makeThreadEvent({
+                event: true,
+                rootEventId: thread.id,
+                replyToEventId: thread.id,
+                user: userId,
+                room: room.roomId,
+                ts: message2Ts,
+            });
+            await thread.addEvent(message2, false, true);
+            await awaitTimelineEvent;
+
+            return { thread, message1, message2 };
         }
 
         function createClientWithEventMapper(canSupport: Map<Feature, ServerSupport> = new Map()): MatrixClient {

spec/unit/notifications.spec.ts

@@ -106,6 +106,8 @@ describe("fixNotificationCountOnDecryption", () => {
             mockClient,
         );
 
+        room.addLiveEvents([event]);
+
         THREAD_ID = event.getId()!;
         threadEvent = mkEvent({
             type: EventType.RoomMessage,

spec/unit/queueToDevice.spec.ts

@@ -265,6 +265,7 @@ describe.each([[StoreType.Memory], [StoreType.IndexedDB]])("queueToDevice (%s st
         });
         const mockRoom = {
             updatePendingEvent: jest.fn(),
+            hasEncryptionStateEvent: jest.fn().mockReturnValue(false),
         } as unknown as Room;
         client.resendEvent(dummyEvent, mockRoom);
 

spec/unit/randomstring.spec.ts

@@ -0,0 +1,66 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { decodeBase64 } from "../../src/base64";
+import {
+    randomLowercaseString,
+    randomString,
+    randomUppercaseString,
+    secureRandomBase64Url,
+} from "../../src/randomstring";
+
+describe("Random strings", () => {
+    it.each([8, 16, 32])("secureRandomBase64 generates %i valid base64 bytes", (n: number) => {
+        const randb641 = secureRandomBase64Url(n);
+        const randb642 = secureRandomBase64Url(n);
+
+        expect(randb641).not.toEqual(randb642);
+
+        const decoded = decodeBase64(randb641);
+        expect(decoded).toHaveLength(n);
+    });
+
+    it.each([8, 16, 32])("randomString generates string of %i characters", (n: number) => {
+        const rand1 = randomString(n);
+        const rand2 = randomString(n);
+
+        expect(rand1).not.toEqual(rand2);
+
+        expect(rand1).toHaveLength(n);
+    });
+
+    it.each([8, 16, 32])("randomLowercaseString generates lowercase string of %i characters", (n: number) => {
+        const rand1 = randomLowercaseString(n);
+        const rand2 = randomLowercaseString(n);
+
+        expect(rand1).not.toEqual(rand2);
+
+        expect(rand1).toHaveLength(n);
+
+        expect(rand1.toLowerCase()).toEqual(rand1);
+    });
+
+    it.each([8, 16, 32])("randomUppercaseString generates lowercase string of %i characters", (n: number) => {
+        const rand1 = randomUppercaseString(n);
+        const rand2 = randomUppercaseString(n);
+
+        expect(rand1).not.toEqual(rand2);
+
+        expect(rand1).toHaveLength(n);
+
+        expect(rand1.toUpperCase()).toEqual(rand1);
+    });
+});

spec/unit/read-receipt.spec.ts

@@ -43,8 +43,10 @@ const THREAD_ID = "$thread_event_id";
 const ROOM_ID = "!123:matrix.org";
 
 describe("Read receipt", () => {
+    let threadRoot: MatrixEvent;
     let threadEvent: MatrixEvent;
     let roomEvent: MatrixEvent;
+    let editOfThreadRoot: MatrixEvent;
 
     beforeEach(() => {
         httpBackend = new MockHttpBackend();
@@ -57,6 +59,15 @@ describe("Read receipt", () => {
         client.isGuest = () => false;
         client.supportsThreads = () => true;
 
+        threadRoot = utils.mkEvent({
+            event: true,
+            type: EventType.RoomMessage,
+            user: "@bob:matrix.org",
+            room: ROOM_ID,
+            content: { body: "This is the thread root" },
+        });
+        threadRoot.event.event_id = THREAD_ID;
+
         threadEvent = utils.mkEvent({
             event: true,
             type: EventType.RoomMessage,
@@ -82,6 +93,9 @@ describe("Read receipt", () => {
                 body: "Hello from a room",
             },
         });
+
+        editOfThreadRoot = utils.mkEdit(threadRoot, client, "@bob:matrix.org", ROOM_ID);
+        editOfThreadRoot.setThreadId(THREAD_ID);
     });
 
     describe("sendReceipt", () => {
@@ -208,6 +222,7 @@ describe("Read receipt", () => {
         it.each([
             { getEvent: () => roomEvent, destinationId: MAIN_ROOM_TIMELINE },
             { getEvent: () => threadEvent, destinationId: THREAD_ID },
+            { getEvent: () => editOfThreadRoot, destinationId: MAIN_ROOM_TIMELINE },
         ])("adds the receipt to $destinationId", ({ getEvent, destinationId }) => {
             const event = getEvent();
             const userId = "@bob:example.org";
@@ -225,6 +240,7 @@ describe("Read receipt", () => {
         it("should not allow an older unthreaded receipt to clobber a `main` threaded one", () => {
             const userId = client.getSafeUserId();
             const room = new Room(ROOM_ID, client, userId);
+            room.findEventById = jest.fn().mockReturnValue({} as MatrixEvent);
 
             const unthreadedReceipt: WrappedReceipt = {
                 eventId: "$olderEvent",

spec/unit/rendezvous/DummyTransport.ts

@@ -32,7 +32,10 @@ export class DummyTransport<D extends RendezvousTransportDetails, T> implements
     ready = false;
     cancelled = false;
 
-    constructor(private name: string, private mockDetails: D) {}
+    constructor(
+        private name: string,
+        private mockDetails: D,
+    ) {}
     onCancelled?: RendezvousFailureListener;
 
     details(): Promise<RendezvousTransportDetails> {

spec/unit/rendezvous/rendezvous.spec.ts

@@ -23,7 +23,7 @@ import {
     MSC3903ECDHPayload,
     MSC3903ECDHv2RendezvousChannel as MSC3903ECDHRendezvousChannel,
 } from "../../../src/rendezvous/channels";
-import { MatrixClient } from "../../../src";
+import { Device, MatrixClient } from "../../../src";
 import {
     MSC3886SimpleHttpRendezvousTransport,
     MSC3886SimpleHttpRendezvousTransportDetails,
@@ -31,16 +31,57 @@ import {
 import { DummyTransport } from "./DummyTransport";
 import { decodeBase64 } from "../../../src/base64";
 import { logger } from "../../../src/logger";
-import { DeviceInfo } from "../../../src/crypto/deviceinfo";
+import { CrossSigningKey, OwnDeviceKeys } from "../../../src/crypto-api";
+
+type UserID = string;
+type DeviceID = string;
+type Fingerprint = string;
+type SimpleDeviceMap = Record<UserID, Record<DeviceID, Fingerprint>>;
+
+function mockDevice(userId: UserID, deviceId: DeviceID, fingerprint: Fingerprint): Device {
+    return {
+        deviceId,
+        userId,
+        getFingerprint: () => fingerprint,
+    } as unknown as Device;
+}
+
+function mockDeviceMap(
+    userId: UserID,
+    deviceId: DeviceID,
+    deviceKey?: Fingerprint,
+    otherDevices: SimpleDeviceMap = {},
+): Map<string, Map<string, Device>> {
+    const deviceMap: Map<string, Map<string, Device>> = new Map();
+
+    const myDevices: Map<string, Device> = new Map();
+    if (deviceKey) {
+        myDevices.set(deviceId, mockDevice(userId, deviceId, deviceKey));
+    }
+    deviceMap.set(userId, myDevices);
+
+    for (const u in otherDevices) {
+        let userDevices = deviceMap.get(u);
+        if (!userDevices) {
+            userDevices = new Map();
+            deviceMap.set(u, userDevices);
+        }
+        for (const d in otherDevices[u]) {
+            userDevices.set(d, mockDevice(u, d, otherDevices[u][d]));
+        }
+    }
+
+    return deviceMap;
+}
 
 function makeMockClient(opts: {
-    userId: string;
-    deviceId: string;
-    deviceKey?: string;
+    userId: UserID;
+    deviceId: DeviceID;
+    deviceKey?: Fingerprint;
     getLoginTokenEnabled: boolean;
     msc3882r0Only: boolean;
     msc3886Enabled: boolean;
-    devices?: Record<string, Partial<DeviceInfo>>;
+    devices?: SimpleDeviceMap;
     verificationFunction?: (
         userId: string,
         deviceId: string,
@@ -48,50 +89,77 @@ function makeMockClient(opts: {
         blocked: boolean,
         known: boolean,
     ) => void;
-    crossSigningIds?: Record<string, string>;
-}): MatrixClient {
-    return {
-        getVersions() {
-            return {
-                unstable_features: {
-                    "org.matrix.msc3882": opts.getLoginTokenEnabled,
-                    "org.matrix.msc3886": opts.msc3886Enabled,
-                },
-            };
-        },
-        getCapabilities() {
-            return opts.msc3882r0Only
-                ? {}
-                : {
-                      capabilities: {
-                          "m.get_login_token": {
-                              enabled: opts.getLoginTokenEnabled,
+    crossSigningIds?: Partial<Record<CrossSigningKey, string>>;
+}): [MatrixClient, Map<string, Map<string, Device>>] {
+    const deviceMap = mockDeviceMap(opts.userId, opts.deviceId, opts.deviceKey, opts.devices);
+    return [
+        {
+            doesServerSupportUnstableFeature: jest.fn().mockImplementation((feature) => {
+                if (feature === "org.matrix.msc3886") {
+                    return opts.msc3886Enabled;
+                } else if (feature === "org.matrix.msc3882") {
+                    return opts.getLoginTokenEnabled;
+                } else {
+                    return false;
+                }
+            }),
+            getVersions() {
+                return {
+                    unstable_features: {
+                        "org.matrix.msc3882": opts.getLoginTokenEnabled,
+                        "org.matrix.msc3886": opts.msc3886Enabled,
+                    },
+                };
+            },
+            getCapabilities() {
+                return opts.msc3882r0Only
+                    ? {}
+                    : {
+                          capabilities: {
+                              "m.get_login_token": {
+                                  enabled: opts.getLoginTokenEnabled,
+                              },
                           },
-                      },
-                  };
-        },
-        getUserId() {
-            return opts.userId;
-        },
-        getDeviceId() {
-            return opts.deviceId;
-        },
-        getDeviceEd25519Key() {
-            return opts.deviceKey;
-        },
-        baseUrl: "https://example.com",
-        crypto: {
-            getStoredDevice(userId: string, deviceId: string) {
-                return opts.devices?.[deviceId] ?? null;
+                      };
             },
-            setDeviceVerification: opts.verificationFunction,
-            crossSigningInfo: {
-                getId(key: string) {
-                    return opts.crossSigningIds?.[key];
-                },
+            getUserId() {
+                return opts.userId;
             },
-        },
-    } as unknown as MatrixClient;
+            getSafeUserId() {
+                return opts.userId;
+            },
+            getDeviceId() {
+                return opts.deviceId;
+            },
+            baseUrl: "https://example.com",
+            getCrypto() {
+                return {
+                    getUserDeviceInfo(
+                        [userId]: string[],
+                        downloadUncached?: boolean,
+                    ): Promise<Map<string, Map<string, Device>>> {
+                        return Promise.resolve(deviceMap);
+                    },
+                    getCrossSigningKeyId(key: CrossSigningKey): string | null {
+                        return opts.crossSigningIds?.[key] ?? null;
+                    },
+                    setDeviceVerified(userId: string, deviceId: string, verified: boolean): Promise<void> {
+                        return Promise.resolve();
+                    },
+                    crossSignDevice(deviceId: string): Promise<void> {
+                        return Promise.resolve();
+                    },
+                    getOwnDeviceKeys(): Promise<OwnDeviceKeys> {
+                        return Promise.resolve({
+                            ed25519: opts.deviceKey!,
+                            curve25519: "aaaa",
+                        });
+                    },
+                };
+            },
+        } as unknown as MatrixClient,
+        deviceMap,
+    ];
 }
 
 function makeTransport(name: string, uri = "https://test.rz/123456") {
@@ -106,6 +174,7 @@ describe("Rendezvous", function () {
     let httpBackend: MockHttpBackend;
     let fetchFn: typeof global.fetch;
     let transports: DummyTransport<any, MSC3903ECDHPayload>[];
+    const userId: UserID = "@user:example.com";
 
     beforeEach(function () {
         httpBackend = new MockHttpBackend();
@@ -118,9 +187,9 @@ describe("Rendezvous", function () {
     });
 
     it("generate and cancel", async function () {
-        const alice = makeMockClient({
-            userId: "@alice:example.com",
-            deviceId: "DEVICEID",
+        const [alice] = makeMockClient({
+            userId,
+            deviceId: "ALICE",
             msc3886Enabled: false,
             getLoginTokenEnabled: true,
             msc3882r0Only: true,
@@ -194,8 +263,8 @@ describe("Rendezvous", function () {
 
         // alice is already signs in and generates a code
         const aliceOnFailure = jest.fn();
-        const alice = makeMockClient({
-            userId: "alice",
+        const [alice] = makeMockClient({
+            userId,
             deviceId: "ALICE",
             msc3886Enabled: false,
             getLoginTokenEnabled,
@@ -257,8 +326,8 @@ describe("Rendezvous", function () {
 
         // alice is already signs in and generates a code
         const aliceOnFailure = jest.fn();
-        const alice = makeMockClient({
-            userId: "alice",
+        const [alice] = makeMockClient({
+            userId,
             deviceId: "ALICE",
             getLoginTokenEnabled: true,
             msc3882r0Only: false,
@@ -316,8 +385,8 @@ describe("Rendezvous", function () {
 
         // alice is already signs in and generates a code
         const aliceOnFailure = jest.fn();
-        const alice = makeMockClient({
-            userId: "alice",
+        const [alice] = makeMockClient({
+            userId,
             deviceId: "ALICE",
             getLoginTokenEnabled: true,
             msc3882r0Only: false,
@@ -375,7 +444,7 @@ describe("Rendezvous", function () {
 
         // alice is already signs in and generates a code
         const aliceOnFailure = jest.fn();
-        const alice = makeMockClient({
+        const [alice] = makeMockClient({
             userId: "alice",
             deviceId: "ALICE",
             getLoginTokenEnabled: true,
@@ -436,7 +505,7 @@ describe("Rendezvous", function () {
 
         // alice is already signs in and generates a code
         const aliceOnFailure = jest.fn();
-        const alice = makeMockClient({
+        const [alice] = makeMockClient({
             userId: "alice",
             deviceId: "ALICE",
             getLoginTokenEnabled: true,
@@ -495,7 +564,7 @@ describe("Rendezvous", function () {
         await bobCompleteProm;
     });
 
-    async function completeLogin(devices: Record<string, Partial<DeviceInfo>>) {
+    async function completeLogin(devices: SimpleDeviceMap) {
         const aliceTransport = makeTransport("Alice", "https://test.rz/123456");
         const bobTransport = makeTransport("Bob", "https://test.rz/999999");
         transports.push(aliceTransport, bobTransport);
@@ -505,8 +574,8 @@ describe("Rendezvous", function () {
         // alice is already signs in and generates a code
         const aliceOnFailure = jest.fn();
         const aliceVerification = jest.fn();
-        const alice = makeMockClient({
-            userId: "alice",
+        const [alice, deviceMap] = makeMockClient({
+            userId,
             deviceId: "ALICE",
             getLoginTokenEnabled: true,
             msc3882r0Only: false,
@@ -575,13 +644,14 @@ describe("Rendezvous", function () {
             aliceRz,
             bobTransport,
             bobEcdh,
+            deviceMap,
         };
     }
 
     it("approve on existing device + verification", async function () {
         const { bobEcdh, aliceRz } = await completeLogin({
-            BOB: {
-                getFingerprint: () => "bbbb",
+            [userId]: {
+                BOB: "bbbb",
             },
         });
         const verifyProm = aliceRz.verifyNewDeviceOnExistingDevice();
@@ -607,33 +677,29 @@ describe("Rendezvous", function () {
     });
 
     it("device appears online within timeout", async function () {
-        const devices: Record<string, Partial<DeviceInfo>> = {};
-        const { aliceRz } = await completeLogin(devices);
-        // device appears after 1 second
+        const devices: SimpleDeviceMap = {};
+        const { aliceRz, deviceMap } = await completeLogin(devices);
+        // device appears before the timeout
         setTimeout(() => {
-            devices.BOB = {
-                getFingerprint: () => "bbbb",
-            };
+            deviceMap.get(userId)!.set("BOB", mockDevice(userId, "BOB", "bbbb"));
         }, 1000);
         await aliceRz.verifyNewDeviceOnExistingDevice(2000);
     });
 
     it("device appears online after timeout", async function () {
-        const devices: Record<string, Partial<DeviceInfo>> = {};
-        const { aliceRz } = await completeLogin(devices);
-        // device appears after 1 second
+        const devices: SimpleDeviceMap = {};
+        const { aliceRz, deviceMap } = await completeLogin(devices);
+        // device appears after the timeout
         setTimeout(() => {
-            devices.BOB = {
-                getFingerprint: () => "bbbb",
-            };
+            deviceMap.get(userId)!.set("BOB", mockDevice(userId, "BOB", "bbbb"));
         }, 1500);
         await expect(aliceRz.verifyNewDeviceOnExistingDevice(1000)).rejects.toThrow();
     });
 
     it("mismatched device key", async function () {
         const { aliceRz } = await completeLogin({
-            BOB: {
-                getFingerprint: () => "XXXX",
+            [userId]: {
+                BOB: "XXXX",
             },
         });
         await expect(aliceRz.verifyNewDeviceOnExistingDevice(1000)).rejects.toThrow(/different key/);

spec/unit/room-state.spec.ts

@@ -27,6 +27,7 @@ import { M_BEACON } from "../../src/@types/beacon";
 import { MatrixClient } from "../../src/client";
 import { DecryptionError } from "../../src/crypto/algorithms";
 import { defer } from "../../src/utils";
+import { Room } from "../../src/models/room";
 
 describe("RoomState", function () {
     const roomId = "!foo:bar";
@@ -362,9 +363,11 @@ describe("RoomState", function () {
         });
 
         it("does not add redacted beacon info events to state", () => {
+            const mockClient = {} as unknown as MockedObject<MatrixClient>;
             const redactedBeaconEvent = makeBeaconInfoEvent(userA, roomId);
             const redactionEvent = new MatrixEvent({ type: "m.room.redaction" });
-            redactedBeaconEvent.makeRedacted(redactionEvent);
+            const room = new Room(roomId, mockClient, userA);
+            redactedBeaconEvent.makeRedacted(redactionEvent, room);
             const emitSpy = jest.spyOn(state, "emit");
 
             state.setStateEvents([redactedBeaconEvent]);
@@ -394,11 +397,13 @@ describe("RoomState", function () {
         });
 
         it("destroys and removes redacted beacon events", () => {
+            const mockClient = {} as unknown as MockedObject<MatrixClient>;
             const beaconId = "$beacon1";
             const beaconEvent = makeBeaconInfoEvent(userA, roomId, { isLive: true }, beaconId);
             const redactedBeaconEvent = makeBeaconInfoEvent(userA, roomId, { isLive: true }, beaconId);
             const redactionEvent = new MatrixEvent({ type: "m.room.redaction", redacts: beaconEvent.getId() });
-            redactedBeaconEvent.makeRedacted(redactionEvent);
+            const room = new Room(roomId, mockClient, userA);
+            redactedBeaconEvent.makeRedacted(redactionEvent, room);
 
             state.setStateEvents([beaconEvent]);
             const beaconInstance = state.beacons.get(getBeaconInfoIdentifier(beaconEvent));

spec/unit/room.spec.ts

@@ -1460,7 +1460,7 @@ describe("Room", function () {
                 it("should reset the unread count when our non-synthetic receipt points to the latest event", () => {
                     // Given a room with 2 events, and an unread count set.
                     room.client.isInitialSyncComplete = jest.fn().mockReturnValue(true);
-                    room.timeline = [event1, event2];
+                    jest.spyOn(room, "timeline", "get").mockReturnValue([event1, event2]);
                     room.setUnread(NotificationCountType.Total, 45);
                     room.setUnread(NotificationCountType.Highlight, 57);
                     // Sanity check:
@@ -1479,7 +1479,7 @@ describe("Room", function () {
                 it("should not reset the unread count when someone else's receipt points to the latest event", () => {
                     // Given a room with 2 events, and an unread count set.
                     room.client.isInitialSyncComplete = jest.fn().mockReturnValue(true);
-                    room.timeline = [event1, event2];
+                    jest.spyOn(room, "timeline", "get").mockReturnValue([event1, event2]);
                     room.setUnread(NotificationCountType.Total, 45);
                     room.setUnread(NotificationCountType.Highlight, 57);
                     // Sanity check:
@@ -1498,7 +1498,7 @@ describe("Room", function () {
                 it("should not reset the unread count when our non-synthetic receipt points to an earlier event", () => {
                     // Given a room with 2 events, and an unread count set.
                     room.client.isInitialSyncComplete = jest.fn().mockReturnValue(true);
-                    room.timeline = [event1, event2];
+                    jest.spyOn(room, "timeline", "get").mockReturnValue([event1, event2]);
                     room.setUnread(NotificationCountType.Total, 45);
                     room.setUnread(NotificationCountType.Highlight, 57);
                     // Sanity check:
@@ -1517,7 +1517,7 @@ describe("Room", function () {
                 it("should not reset the unread count when our a synthetic receipt points to the latest event", () => {
                     // Given a room with 2 events, and an unread count set.
                     room.client.isInitialSyncComplete = jest.fn().mockReturnValue(true);
-                    room.timeline = [event1, event2];
+                    jest.spyOn(room, "timeline", "get").mockReturnValue([event1, event2]);
                     room.setUnread(NotificationCountType.Total, 45);
                     room.setUnread(NotificationCountType.Highlight, 57);
                     // Sanity check:
@@ -1743,12 +1743,70 @@ describe("Room", function () {
         });
 
         describe("hasUserReadUpTo", function () {
-            it("should acknowledge if an event has been read", function () {
+            it("returns true if there is a receipt for this event (main timeline)", function () {
                 const ts = 13787898424;
+                room.addLiveEvents([eventToAck]);
                 room.addReceipt(mkReceipt(roomId, [mkRecord(eventToAck.getId()!, "m.read", userB, ts)]));
+                room.findEventById = jest.fn().mockReturnValue({ getThread: jest.fn() } as unknown as MatrixEvent);
                 expect(room.hasUserReadEvent(userB, eventToAck.getId()!)).toEqual(true);
             });
-            it("return false for an unknown event", function () {
+
+            it("returns true if there is a receipt for a later event (main timeline)", async function () {
+                // Given some events exist in the room
+                const events: MatrixEvent[] = [
+                    utils.mkMessage({
+                        room: roomId,
+                        user: userA,
+                        msg: "1111",
+                        event: true,
+                    }),
+                    utils.mkMessage({
+                        room: roomId,
+                        user: userA,
+                        msg: "2222",
+                        event: true,
+                    }),
+                    utils.mkMessage({
+                        room: roomId,
+                        user: userA,
+                        msg: "3333",
+                        event: true,
+                    }),
+                ];
+                await room.addLiveEvents(events);
+
+                // When I add a receipt for the latest one
+                room.addReceipt(mkReceipt(roomId, [mkRecord(events[2].getId()!, "m.read", userB, 102)]));
+
+                // Then the older ones are read too
+                expect(room.hasUserReadEvent(userB, events[0].getId()!)).toEqual(true);
+                expect(room.hasUserReadEvent(userB, events[1].getId()!)).toEqual(true);
+            });
+
+            describe("threads enabled", () => {
+                beforeEach(() => {
+                    jest.spyOn(room.client, "supportsThreads").mockReturnValue(true);
+                });
+
+                afterEach(() => {
+                    jest.restoreAllMocks();
+                });
+
+                it("returns true if there is an unthreaded receipt for a later event in a thread", async () => {
+                    // Given a thread exists in the room
+                    const { thread, events } = mkThread({ room, length: 3 });
+                    thread.initialEventsFetched = true;
+                    await room.addLiveEvents(events);
+
+                    // When I add an unthreaded receipt for the latest thread message
+                    room.addReceipt(mkReceipt(roomId, [mkRecord(events[2].getId()!, "m.read", userB, 102)]));
+
+                    // Then the main timeline message is read
+                    expect(room.hasUserReadEvent(userB, events[0].getId()!)).toEqual(true);
+                });
+            });
+
+            it("returns false for an unknown event", function () {
                 expect(room.hasUserReadEvent(userB, "unknown_event")).toEqual(false);
             });
         });
@@ -3147,106 +3205,194 @@ describe("Room", function () {
         const client = new TestClient(userA).client;
         const room = new Room(roomId, client, userA);
 
-        it("handles missing receipt type", () => {
-            room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
-                return receiptType === ReceiptType.ReadPrivate ? ({ eventId: "eventId" } as WrappedReceipt) : null;
-            };
-
-            expect(room.getEventReadUpTo(userA)).toEqual("eventId");
-        });
-
-        describe("prefers newer receipt", () => {
-            it("should compare correctly using timelines", () => {
-                room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
-                    if (receiptType === ReceiptType.ReadPrivate) {
-                        return { eventId: "eventId1" } as WrappedReceipt;
-                    }
-                    if (receiptType === ReceiptType.Read) {
-                        return { eventId: "eventId2" } as WrappedReceipt;
-                    }
-                    return null;
-                };
-
-                for (let i = 1; i <= 2; i++) {
-                    room.getUnfilteredTimelineSet = () =>
-                        ({
-                            compareEventOrdering: (event1, event2) => {
-                                return event1 === `eventId${i}` ? 1 : -1;
-                            },
-                        } as EventTimelineSet);
-
-                    expect(room.getEventReadUpTo(userA)).toEqual(`eventId${i}`);
-                }
+        describe("invalid receipts", () => {
+            beforeEach(() => {
+                // Clear the spies on logger.warn
+                jest.clearAllMocks();
             });
 
-            describe("correctly compares by timestamp", () => {
-                it("should correctly compare, if we have all receipts", () => {
+            it("ignores receipts pointing at missing events", () => {
+                // Given a receipt exists
+                room.getReadReceiptForUserId = (): WrappedReceipt | null => {
+                    return { eventId: "missingEventId" } as WrappedReceipt;
+                };
+                // But the event ID it contains does not refer to an event we have
+                room.findEventById = jest.fn().mockReturnValue(null);
+
+                // When we ask what they have read
+                // Then we say "nothing"
+                expect(room.getEventReadUpTo(userA)).toBeNull();
+            });
+
+            it("ignores receipts pointing at the wrong thread", () => {
+                // Given a threaded receipt exists
+                room.getReadReceiptForUserId = (): WrappedReceipt | null => {
+                    return { eventId: "wrongThreadEventId", data: { ts: 0, thread_id: "thread1" } } as WrappedReceipt;
+                };
+                // But the event it refers to is in a thread
+                room.findEventById = jest.fn().mockReturnValue({ threadRootId: "thread2" } as MatrixEvent);
+
+                // When we ask what they have read
+                // Then we say "nothing"
+                expect(room.getEventReadUpTo(userA)).toBeNull();
+                expect(logger.warn).toHaveBeenCalledWith(
+                    "Ignoring receipt because its thread_id (thread1) disagrees with the thread root (thread2) " +
+                        "of the referenced event (event ID = wrongThreadEventId)",
+                );
+            });
+
+            it("accepts unthreaded receipts pointing at an event in a thread", () => {
+                // Given an unthreaded receipt exists
+                room.getReadReceiptForUserId = (): WrappedReceipt | null => {
+                    return { eventId: "inThreadEventId" } as WrappedReceipt;
+                };
+                // And the event it refers to is in a thread
+                room.findEventById = jest.fn().mockReturnValue({ threadRootId: "thread2" } as MatrixEvent);
+
+                // When we ask what they have read
+                // Then we say the event
+                expect(room.getEventReadUpTo(userA)).toEqual("inThreadEventId");
+            });
+
+            it("accepts main thread receipts pointing at an event in main timeline", () => {
+                // Given a threaded receipt exists, in main thread
+                room.getReadReceiptForUserId = (): WrappedReceipt | null => {
+                    return { eventId: "mainThreadEventId", data: { ts: 12, thread_id: "main" } } as WrappedReceipt;
+                };
+                // And the event it refers to is in a thread
+                room.findEventById = jest.fn().mockReturnValue({ threadRootId: undefined } as MatrixEvent);
+
+                // When we ask what they have read
+                // Then we say the event
+                expect(room.getEventReadUpTo(userA)).toEqual("mainThreadEventId");
+            });
+
+            it("accepts main thread receipts pointing at a thread root", () => {
+                // Given a threaded receipt exists, in main thread
+                room.getReadReceiptForUserId = (): WrappedReceipt | null => {
+                    return { eventId: "rootId", data: { ts: 12, thread_id: "main" } } as WrappedReceipt;
+                };
+                // And the event it refers to is in a thread, because it is a thread root
+                room.findEventById = jest
+                    .fn()
+                    .mockReturnValue({ isThreadRoot: true, threadRootId: "thread1" } as MatrixEvent);
+
+                // When we ask what they have read
+                // Then we say the event
+                expect(room.getEventReadUpTo(userA)).toEqual("rootId");
+            });
+        });
+
+        describe("valid receipts", () => {
+            beforeEach(() => {
+                // When we look up the event referred to by the receipt, it exists
+                room.findEventById = jest.fn().mockReturnValue({} as MatrixEvent);
+            });
+
+            it("handles missing receipt type", () => {
+                room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
+                    return receiptType === ReceiptType.ReadPrivate ? ({ eventId: "eventId" } as WrappedReceipt) : null;
+                };
+                expect(room.getEventReadUpTo(userA)).toEqual("eventId");
+            });
+
+            describe("prefers newer receipt", () => {
+                it("should compare correctly using timelines", () => {
+                    room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
+                        if (receiptType === ReceiptType.ReadPrivate) {
+                            return { eventId: "eventId1" } as WrappedReceipt;
+                        }
+                        if (receiptType === ReceiptType.Read) {
+                            return { eventId: "eventId2" } as WrappedReceipt;
+                        }
+                        return null;
+                    };
+
                     for (let i = 1; i <= 2; i++) {
                         room.getUnfilteredTimelineSet = () =>
                             ({
-                                compareEventOrdering: (_1, _2) => null,
-                            } as EventTimelineSet);
-                        room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
-                            if (receiptType === ReceiptType.ReadPrivate) {
-                                return { eventId: "eventId1", data: { ts: i === 1 ? 2 : 1 } } as WrappedReceipt;
-                            }
-                            if (receiptType === ReceiptType.Read) {
-                                return { eventId: "eventId2", data: { ts: i === 2 ? 2 : 1 } } as WrappedReceipt;
-                            }
-                            return null;
-                        };
+                                compareEventOrdering: (event1: string, _event2: string) => {
+                                    return event1 === `eventId${i}` ? 1 : -1;
+                                },
+                                findEventById: jest.fn().mockReturnValue({} as MatrixEvent),
+                            }) as unknown as EventTimelineSet;
 
                         expect(room.getEventReadUpTo(userA)).toEqual(`eventId${i}`);
                     }
                 });
 
-                it("should correctly compare, if private read receipt is missing", () => {
-                    room.getUnfilteredTimelineSet = () =>
-                        ({
-                            compareEventOrdering: (_1, _2) => null,
-                        } as EventTimelineSet);
-                    room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
-                        if (receiptType === ReceiptType.Read) {
-                            return { eventId: "eventId2", data: { ts: 1 } } as WrappedReceipt;
+                describe("correctly compares by timestamp", () => {
+                    it("should correctly compare, if we have all receipts", () => {
+                        for (let i = 1; i <= 2; i++) {
+                            room.getUnfilteredTimelineSet = () =>
+                                ({
+                                    compareEventOrdering: () => null,
+                                    findEventById: jest.fn().mockReturnValue({} as MatrixEvent),
+                                }) as unknown as EventTimelineSet;
+                            room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
+                                if (receiptType === ReceiptType.ReadPrivate) {
+                                    return { eventId: "eventId1", data: { ts: i === 1 ? 2 : 1 } } as WrappedReceipt;
+                                }
+                                if (receiptType === ReceiptType.Read) {
+                                    return { eventId: "eventId2", data: { ts: i === 2 ? 2 : 1 } } as WrappedReceipt;
+                                }
+                                return null;
+                            };
+
+                            expect(room.getEventReadUpTo(userA)).toEqual(`eventId${i}`);
                         }
-                        return null;
-                    };
+                    });
 
-                    expect(room.getEventReadUpTo(userA)).toEqual(`eventId2`);
-                });
-            });
+                    it("should correctly compare, if private read receipt is missing", () => {
+                        room.getUnfilteredTimelineSet = () =>
+                            ({
+                                compareEventOrdering: () => null,
+                                findEventById: jest.fn().mockReturnValue({} as MatrixEvent),
+                            }) as unknown as EventTimelineSet;
+                        room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
+                            if (receiptType === ReceiptType.Read) {
+                                return { eventId: "eventId2", data: { ts: 1 } } as WrappedReceipt;
+                            }
+                            return null;
+                        };
 
-            describe("fallback precedence", () => {
-                beforeAll(() => {
-                    room.getUnfilteredTimelineSet = () =>
-                        ({
-                            compareEventOrdering: (_1, _2) => null,
-                        } as EventTimelineSet);
+                        expect(room.getEventReadUpTo(userA)).toEqual(`eventId2`);
+                    });
                 });
 
-                it("should give precedence to m.read.private", () => {
-                    room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
-                        if (receiptType === ReceiptType.ReadPrivate) {
-                            return { eventId: "eventId1", data: { ts: 123 } };
-                        }
-                        if (receiptType === ReceiptType.Read) {
-                            return { eventId: "eventId2", data: { ts: 123 } };
-                        }
-                        return null;
-                    };
+                describe("fallback precedence", () => {
+                    beforeAll(() => {
+                        room.getUnfilteredTimelineSet = () =>
+                            ({
+                                compareEventOrdering: () => null,
+                                findEventById: jest.fn().mockReturnValue({} as MatrixEvent),
+                            }) as unknown as EventTimelineSet;
+                    });
 
-                    expect(room.getEventReadUpTo(userA)).toEqual(`eventId1`);
-                });
+                    it("should give precedence to m.read.private", () => {
+                        room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
+                            if (receiptType === ReceiptType.ReadPrivate) {
+                                return { eventId: "eventId1", data: { ts: 123 } };
+                            }
+                            if (receiptType === ReceiptType.Read) {
+                                return { eventId: "eventId2", data: { ts: 123 } };
+                            }
+                            return null;
+                        };
 
-                it("should give precedence to m.read", () => {
-                    room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
-                        if (receiptType === ReceiptType.Read) {
-                            return { eventId: "eventId3" } as WrappedReceipt;
-                        }
-                        return null;
-                    };
+                        expect(room.getEventReadUpTo(userA)).toEqual(`eventId1`);
+                    });
 
-                    expect(room.getEventReadUpTo(userA)).toEqual(`eventId3`);
+                    it("should give precedence to m.read", () => {
+                        room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
+                            if (receiptType === ReceiptType.Read) {
+                                return { eventId: "eventId3" } as WrappedReceipt;
+                            }
+                            return null;
+                        };
+
+                        expect(room.getEventReadUpTo(userA)).toEqual(`eventId3`);
+                    });
                 });
             });
         });
@@ -3564,7 +3710,7 @@ describe("Room", function () {
             expect(room.polls.get(pollStartEvent.getId()!)).toBeTruthy();
 
             const redactedEvent = new MatrixEvent({ type: "m.room.redaction" });
-            pollStartEvent.makeRedacted(redactedEvent);
+            pollStartEvent.makeRedacted(redactedEvent, room);
 
             await flushPromises();
 

spec/unit/rust-crypto/KeyClaimManager.spec.ts

@@ -23,6 +23,7 @@ import { OutgoingRequestProcessor } from "../../../src/rust-crypto/OutgoingReque
 import { KeyClaimManager } from "../../../src/rust-crypto/KeyClaimManager";
 import { TypedEventEmitter } from "../../../src/models/typed-event-emitter";
 import { HttpApiEvent, HttpApiEventHandlerMap, MatrixHttpApi } from "../../../src";
+import { logger, LogSpan } from "../../../src/logger";
 
 afterEach(() => {
     fetchMock.mockReset();
@@ -93,10 +94,13 @@ describe("KeyClaimManager", () => {
         olmMachine.markRequestAsSent.mockResolvedValueOnce(undefined);
 
         // fire off the request
-        await keyClaimManager.ensureSessionsForUsers([u1, u2]);
+        await keyClaimManager.ensureSessionsForUsers(new LogSpan(logger, "test"), [u1, u2]);
 
         // check that all the calls were made
-        expect(olmMachine.getMissingSessions).toHaveBeenCalledWith([u1, u2]);
+        // We can't use directly toHaveBeenCalledWith because the UserId are cloned in the process.
+        const calledWith = olmMachine.getMissingSessions.mock.calls[0][0].map((u) => u.toString());
+        expect(calledWith).toEqual([u1.toString(), u2.toString()]);
+
         expect(fetchMock).toHaveFetched("https://example.com/_matrix/client/v3/keys/claim", {
             method: "POST",
             body: { k1: "v1" },
@@ -119,12 +123,13 @@ describe("KeyClaimManager", () => {
         let markRequestAsSentPromise = awaitCallToMarkRequestAsSent();
 
         // fire off two requests, and keep track of whether their promises resolve
+        const span = new LogSpan(logger, "test");
         let req1Resolved = false;
-        keyClaimManager.ensureSessionsForUsers([u1]).then(() => {
+        keyClaimManager.ensureSessionsForUsers(span, [u1]).then(() => {
             req1Resolved = true;
         });
         let req2Resolved = false;
-        const req2 = keyClaimManager.ensureSessionsForUsers([u2]).then(() => {
+        const req2 = keyClaimManager.ensureSessionsForUsers(span, [u2]).then(() => {
             req2Resolved = true;
         });
 
@@ -133,7 +138,10 @@ describe("KeyClaimManager", () => {
 
         // at this point, there should have been a single call to getMissingSessions, and a single fetch; and neither
         // call to ensureSessionsAsUsers should have completed
-        expect(olmMachine.getMissingSessions).toHaveBeenCalledWith([u1]);
+        // check that all the calls were made
+        // We can't use directly toHaveBeenCalledWith because the UserId are cloned in the process.
+        const calledWith = olmMachine.getMissingSessions.mock.calls[0][0].map((u) => u.toString());
+        expect(calledWith).toEqual([u1.toString()]);
         expect(olmMachine.getMissingSessions).toHaveBeenCalledTimes(1);
         expect(fetchMock).toHaveBeenCalledTimes(1);
         expect(req1Resolved).toBe(false);
@@ -145,7 +153,9 @@ describe("KeyClaimManager", () => {
         resolveMarkRequestAsSentCallback = await markRequestAsSentPromise;
 
         // the first request should now have completed, and we should have more calls and fetches
-        expect(olmMachine.getMissingSessions).toHaveBeenCalledWith([u2]);
+        // We can't use directly toHaveBeenCalledWith because the UserId are cloned in the process.
+        const calledWith2 = olmMachine.getMissingSessions.mock.calls[1][0].map((u) => u.toString());
+        expect(calledWith2).toEqual([u2.toString()]);
         expect(olmMachine.getMissingSessions).toHaveBeenCalledTimes(2);
         expect(fetchMock).toHaveBeenCalledTimes(2);
         expect(req1Resolved).toBe(true);

spec/unit/rust-crypto/OutgoingRequestProcessor.spec.ts

@@ -24,7 +24,7 @@ import {
     KeysUploadRequest,
     RoomMessageRequest,
     SignatureUploadRequest,
-    SigningKeysUploadRequest,
+    UploadSigningKeysRequest,
     ToDeviceRequest,
 } from "@matrix-org/matrix-sdk-crypto-wasm";
 
@@ -173,10 +173,10 @@ describe("OutgoingRequestProcessor", () => {
         httpBackend.verifyNoOutstandingRequests();
     });
 
-    it("should handle SigningKeysUploadRequests without UIA", async () => {
+    it("should handle UploadSigningKeysRequest without UIA", async () => {
         // first, mock up a request as we might expect to receive it from the Rust layer ...
         const testReq = { foo: "bar" };
-        const outgoingRequest = new SigningKeysUploadRequest(JSON.stringify(testReq));
+        const outgoingRequest = new UploadSigningKeysRequest(JSON.stringify(testReq));
 
         // ... then poke the request into the OutgoingRequestProcessor under test
         const reqProm = processor.makeOutgoingRequest(outgoingRequest);
@@ -200,10 +200,10 @@ describe("OutgoingRequestProcessor", () => {
         httpBackend.verifyNoOutstandingRequests();
     });
 
-    it("should handle SigningKeysUploadRequests with UIA", async () => {
+    it("should handle UploadSigningKeysRequest with UIA", async () => {
         // first, mock up a request as we might expect to receive it from the Rust layer ...
         const testReq = { foo: "bar" };
-        const outgoingRequest = new SigningKeysUploadRequest(JSON.stringify(testReq));
+        const outgoingRequest = new UploadSigningKeysRequest(JSON.stringify(testReq));
 
         // also create a UIA callback
         const authCallback: UIAuthCallback<Object> = async (makeRequest) => {

spec/unit/rust-crypto/OutgoingRequestsManager.spec.ts

@@ -0,0 +1,237 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { Mocked } from "jest-mock";
+import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
+
+import { OutgoingRequest, OutgoingRequestProcessor } from "../../../src/rust-crypto/OutgoingRequestProcessor";
+import { OutgoingRequestsManager } from "../../../src/rust-crypto/OutgoingRequestsManager";
+import { defer, IDeferred } from "../../../src/utils";
+import { logger } from "../../../src/logger";
+
+describe("OutgoingRequestsManager", () => {
+    /** the OutgoingRequestsManager implementation under test */
+    let manager: OutgoingRequestsManager;
+
+    /** a mock OutgoingRequestProcessor */
+    let processor: Mocked<OutgoingRequestProcessor>;
+
+    /** a mocked-up OlmMachine which manager is connected to */
+    let olmMachine: Mocked<RustSdkCryptoJs.OlmMachine>;
+
+    beforeEach(async () => {
+        olmMachine = {
+            outgoingRequests: jest.fn(),
+        } as unknown as Mocked<RustSdkCryptoJs.OlmMachine>;
+
+        processor = {
+            makeOutgoingRequest: jest.fn(),
+        } as unknown as Mocked<OutgoingRequestProcessor>;
+
+        manager = new OutgoingRequestsManager(logger, olmMachine, processor);
+    });
+
+    describe("Call doProcessOutgoingRequests", () => {
+        it("The call triggers handling of the machine outgoing requests", async () => {
+            const request1 = new RustSdkCryptoJs.KeysQueryRequest("foo", "{}");
+            const request2 = new RustSdkCryptoJs.KeysUploadRequest("foo2", "{}");
+            olmMachine.outgoingRequests.mockImplementationOnce(async () => {
+                return [request1, request2];
+            });
+
+            processor.makeOutgoingRequest.mockImplementationOnce(async () => {
+                return;
+            });
+
+            await manager.doProcessOutgoingRequests();
+
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(1);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(2);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request1);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request2);
+        });
+
+        it("Stack and batch calls to doProcessOutgoingRequests while one is already running", async () => {
+            const request1 = new RustSdkCryptoJs.KeysQueryRequest("foo", "{}");
+            const request2 = new RustSdkCryptoJs.KeysUploadRequest("foo2", "{}");
+            const request3 = new RustSdkCryptoJs.KeysBackupRequest("foo3", "{}", "1");
+
+            const firstOutgoingRequestDefer = defer<OutgoingRequest[]>();
+
+            olmMachine.outgoingRequests
+                .mockImplementationOnce(async (): Promise<OutgoingRequest[]> => {
+                    return firstOutgoingRequestDefer.promise;
+                })
+                .mockImplementationOnce(async () => {
+                    return [request3];
+                });
+
+            const firstRequest = manager.doProcessOutgoingRequests();
+
+            // stack 2 additional requests while the first one is still running
+            const secondRequest = manager.doProcessOutgoingRequests();
+            const thirdRequest = manager.doProcessOutgoingRequests();
+
+            // let the first request complete
+            firstOutgoingRequestDefer.resolve([request1, request2]);
+
+            await firstRequest;
+            await secondRequest;
+            await thirdRequest;
+
+            // outgoingRequests should be called twice in total, as the second and third requests are
+            // processed in the same loop.
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(2);
+
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(3);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request1);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request2);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request3);
+        });
+
+        it("Process 3 consecutive calls to doProcessOutgoingRequests while not blocking previous ones", async () => {
+            const request1 = new RustSdkCryptoJs.KeysQueryRequest("foo", "{}");
+            const request2 = new RustSdkCryptoJs.KeysUploadRequest("foo2", "{}");
+            const request3 = new RustSdkCryptoJs.KeysBackupRequest("foo3", "{}", "1");
+
+            // promises which will resolve when OlmMachine.outgoingRequests is called
+            const outgoingRequestCalledPromises: Promise<void>[] = [];
+
+            // deferreds which will provide the results of OlmMachine.outgoingRequests
+            const outgoingRequestResultDeferreds: IDeferred<OutgoingRequest[]>[] = [];
+
+            for (let i = 0; i < 3; i++) {
+                const resultDeferred = defer<OutgoingRequest[]>();
+                const calledPromise = new Promise<void>((resolve) => {
+                    olmMachine.outgoingRequests.mockImplementationOnce(() => {
+                        resolve();
+                        return resultDeferred.promise;
+                    });
+                });
+                outgoingRequestCalledPromises.push(calledPromise);
+                outgoingRequestResultDeferreds.push(resultDeferred);
+            }
+
+            const call1 = manager.doProcessOutgoingRequests();
+
+            // First call will start an iteration and for now is awaiting on outgoingRequests
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(1);
+
+            // Make a new call now: this will request a new iteration
+            const call2 = manager.doProcessOutgoingRequests();
+
+            // let the first iteration complete
+            outgoingRequestResultDeferreds[0].resolve([request1]);
+
+            // The first call should now complete
+            await call1;
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(1);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request1);
+
+            // Wait for the second iteration to fire and be waiting on `outgoingRequests`
+            await outgoingRequestCalledPromises[1];
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(2);
+
+            // Stack a new call that should be processed in an additional iteration.
+            const call3 = manager.doProcessOutgoingRequests();
+
+            outgoingRequestResultDeferreds[1].resolve([request2]);
+            await call2;
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(2);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request2);
+
+            // Wait for the third iteration to fire and be waiting on `outgoingRequests`
+            await outgoingRequestCalledPromises[2];
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(3);
+            outgoingRequestResultDeferreds[2].resolve([request3]);
+            await call3;
+
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(3);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request3);
+
+            // ensure that no other iteration is going on
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(3);
+        });
+
+        it("Should not bubble exceptions if server request is rejected", async () => {
+            const request = new RustSdkCryptoJs.KeysQueryRequest("foo", "{}");
+            olmMachine.outgoingRequests.mockImplementationOnce(async () => {
+                return [request];
+            });
+
+            processor.makeOutgoingRequest.mockImplementationOnce(async () => {
+                throw new Error("Some network error");
+            });
+
+            await manager.doProcessOutgoingRequests();
+
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(1);
+        });
+    });
+
+    describe("Calling stop on the manager should stop ongoing work", () => {
+        it("When the manager is stopped after outgoingRequests() call, do not make sever requests", async () => {
+            const request1 = new RustSdkCryptoJs.KeysQueryRequest("foo", "{}");
+
+            const firstOutgoingRequestDefer = defer<OutgoingRequest[]>();
+
+            olmMachine.outgoingRequests.mockImplementationOnce(async (): Promise<OutgoingRequest[]> => {
+                return firstOutgoingRequestDefer.promise;
+            });
+
+            const firstRequest = manager.doProcessOutgoingRequests();
+
+            // stop
+            manager.stop();
+
+            // let the first request complete
+            firstOutgoingRequestDefer.resolve([request1]);
+
+            await firstRequest;
+
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(0);
+        });
+
+        it("When the manager is stopped while doing server calls, it should stop before the next sever call", async () => {
+            const request1 = new RustSdkCryptoJs.KeysQueryRequest("11", "{}");
+            const request2 = new RustSdkCryptoJs.KeysUploadRequest("12", "{}");
+
+            const firstRequestDefer = defer<void>();
+
+            olmMachine.outgoingRequests.mockImplementationOnce(async (): Promise<OutgoingRequest[]> => {
+                return [request1, request2];
+            });
+
+            processor.makeOutgoingRequest
+                .mockImplementationOnce(async () => {
+                    manager.stop();
+                    return firstRequestDefer.promise;
+                })
+                .mockImplementationOnce(async () => {
+                    return;
+                });
+
+            const firstRequest = manager.doProcessOutgoingRequests();
+
+            firstRequestDefer.resolve();
+
+            await firstRequest;
+
+            // should have been called once but not twice
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(1);
+        });
+    });
+});

spec/unit/rust-crypto/PerSessionKeyBackupDownloader.spec.ts

@@ -0,0 +1,598 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { Mocked, SpyInstance } from "jest-mock";
+import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
+import { OlmMachine } from "@matrix-org/matrix-sdk-crypto-wasm";
+import fetchMock from "fetch-mock-jest";
+
+import { PerSessionKeyBackupDownloader } from "../../../src/rust-crypto/PerSessionKeyBackupDownloader";
+import { logger } from "../../../src/logger";
+import { defer, IDeferred } from "../../../src/utils";
+import { RustBackupCryptoEventMap, RustBackupCryptoEvents, RustBackupManager } from "../../../src/rust-crypto/backup";
+import * as TestData from "../../test-utils/test-data";
+import {
+    ConnectionError,
+    CryptoEvent,
+    HttpApiEvent,
+    HttpApiEventHandlerMap,
+    IHttpOpts,
+    IMegolmSessionData,
+    MatrixHttpApi,
+    TypedEventEmitter,
+} from "../../../src";
+import * as testData from "../../test-utils/test-data";
+import { BackupDecryptor } from "../../../src/common-crypto/CryptoBackend";
+import { KeyBackupSession } from "../../../src/crypto-api/keybackup";
+
+describe("PerSessionKeyBackupDownloader", () => {
+    /** The downloader under test */
+    let downloader: PerSessionKeyBackupDownloader;
+
+    const mockCipherKey: Mocked<KeyBackupSession> = {} as unknown as Mocked<KeyBackupSession>;
+
+    // matches the const in PerSessionKeyBackupDownloader
+    const BACKOFF_TIME = 5000;
+
+    let mockEmitter: TypedEventEmitter<RustBackupCryptoEvents, RustBackupCryptoEventMap>;
+    let mockHttp: MatrixHttpApi<IHttpOpts & { onlyData: true }>;
+    let mockRustBackupManager: Mocked<RustBackupManager>;
+    let mockOlmMachine: Mocked<OlmMachine>;
+    let mockBackupDecryptor: Mocked<BackupDecryptor>;
+
+    let expectedSession: { [roomId: string]: { [sessionId: string]: IDeferred<void> } };
+
+    function expectSessionImported(roomId: string, sessionId: string) {
+        const deferred = defer<void>();
+        if (!expectedSession[roomId]) {
+            expectedSession[roomId] = {};
+        }
+        expectedSession[roomId][sessionId] = deferred;
+        return deferred.promise;
+    }
+
+    function mockClearSession(sessionId: string): Mocked<IMegolmSessionData> {
+        return {
+            session_id: sessionId,
+        } as unknown as Mocked<IMegolmSessionData>;
+    }
+
+    beforeEach(async () => {
+        mockEmitter = new TypedEventEmitter() as TypedEventEmitter<RustBackupCryptoEvents, RustBackupCryptoEventMap>;
+
+        mockHttp = new MatrixHttpApi(new TypedEventEmitter<HttpApiEvent, HttpApiEventHandlerMap>(), {
+            baseUrl: "http://server/",
+            prefix: "",
+            onlyData: true,
+        });
+
+        mockBackupDecryptor = {
+            decryptSessions: jest.fn(),
+        } as unknown as Mocked<BackupDecryptor>;
+
+        mockBackupDecryptor.decryptSessions.mockImplementation(async (ciphertexts) => {
+            const sessionId = Object.keys(ciphertexts)[0];
+            return [mockClearSession(sessionId)];
+        });
+
+        mockRustBackupManager = {
+            getActiveBackupVersion: jest.fn(),
+            requestKeyBackupVersion: jest.fn(),
+            importBackedUpRoomKeys: jest.fn(),
+            createBackupDecryptor: jest.fn().mockReturnValue(mockBackupDecryptor),
+            on: jest.fn().mockImplementation((event, listener) => {
+                mockEmitter.on(event, listener);
+            }),
+            off: jest.fn().mockImplementation((event, listener) => {
+                mockEmitter.off(event, listener);
+            }),
+        } as unknown as Mocked<RustBackupManager>;
+
+        mockOlmMachine = {
+            getBackupKeys: jest.fn(),
+        } as unknown as Mocked<OlmMachine>;
+
+        downloader = new PerSessionKeyBackupDownloader(logger, mockOlmMachine, mockHttp, mockRustBackupManager);
+
+        expectedSession = {};
+        mockRustBackupManager.importBackedUpRoomKeys.mockImplementation(async (keys) => {
+            const roomId = keys[0].room_id;
+            const sessionId = keys[0].session_id;
+            const deferred = expectedSession[roomId] && expectedSession[roomId][sessionId];
+            if (deferred) {
+                deferred.resolve();
+            }
+        });
+
+        jest.useFakeTimers();
+    });
+
+    afterEach(() => {
+        expectedSession = {};
+        downloader.stop();
+        fetchMock.mockReset();
+        jest.useRealTimers();
+    });
+
+    describe("Given valid backup available", () => {
+        beforeEach(async () => {
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA.version!);
+            mockOlmMachine.getBackupKeys.mockResolvedValue({
+                backupVersion: TestData.SIGNED_BACKUP_DATA.version!,
+                decryptionKey: RustSdkCryptoJs.BackupDecryptionKey.fromBase64(TestData.BACKUP_DECRYPTION_KEY_BASE64),
+            } as unknown as RustSdkCryptoJs.BackupKeys);
+
+            mockRustBackupManager.requestKeyBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA);
+        });
+
+        it("Should download and import a missing key from backup", async () => {
+            const awaitKeyImported = defer<void>();
+            const roomId = "!roomId";
+            const sessionId = "sessionId";
+            const expectAPICall = new Promise<void>((resolve) => {
+                fetchMock.get(`path:/_matrix/client/v3/room_keys/keys/${roomId}/${sessionId}`, (url, request) => {
+                    resolve();
+                    return TestData.CURVE25519_KEY_BACKUP_DATA;
+                });
+            });
+            mockRustBackupManager.importBackedUpRoomKeys.mockImplementation(async (keys) => {
+                awaitKeyImported.resolve();
+            });
+            mockBackupDecryptor.decryptSessions.mockResolvedValue([TestData.MEGOLM_SESSION_DATA]);
+
+            downloader.onDecryptionKeyMissingError(roomId, sessionId);
+
+            await expectAPICall;
+            await awaitKeyImported.promise;
+            expect(mockRustBackupManager.createBackupDecryptor).toHaveBeenCalledTimes(1);
+        });
+
+        it("Should not hammer the backup if the key is requested repeatedly", async () => {
+            const blockOnServerRequest = defer<void>();
+
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/!roomId/:session_id`, async (url, request) => {
+                await blockOnServerRequest.promise;
+                return [mockCipherKey];
+            });
+
+            const awaitKey2Imported = defer<void>();
+
+            mockRustBackupManager.importBackedUpRoomKeys.mockImplementation(async (keys) => {
+                if (keys[0].session_id === "sessionId2") {
+                    awaitKey2Imported.resolve();
+                }
+            });
+
+            // @ts-ignore access to private function
+            const spy = jest.spyOn(downloader, "queryKeyBackup");
+
+            // Call 3 times for same key
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+
+            // Call again for a different key
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId2");
+
+            // Allow the first server request to complete
+            blockOnServerRequest.resolve();
+
+            await awaitKey2Imported.promise;
+            expect(spy).toHaveBeenCalledTimes(2);
+        });
+
+        it("should continue to next key if current not in backup", async () => {
+            fetchMock.get(`path:/_matrix/client/v3/room_keys/keys/!roomA/sessionA0`, {
+                status: 404,
+                body: {
+                    errcode: "M_NOT_FOUND",
+                    error: "No backup found",
+                },
+            });
+            fetchMock.get(`path:/_matrix/client/v3/room_keys/keys/!roomA/sessionA1`, mockCipherKey);
+
+            // @ts-ignore access to private function
+            const spy: SpyInstance = jest.spyOn(downloader, "queryKeyBackup");
+
+            const expectImported = expectSessionImported("!roomA", "sessionA1");
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            await jest.runAllTimersAsync();
+            expect(spy).toHaveBeenCalledTimes(1);
+            expect(spy).toHaveLastReturnedWith(Promise.resolve({ ok: false, error: "MISSING_DECRYPTION_KEY" }));
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA1");
+            await jest.runAllTimersAsync();
+            expect(spy).toHaveBeenCalledTimes(2);
+
+            await expectImported;
+        });
+
+        it("Should not query repeatedly for a key not in backup", async () => {
+            fetchMock.get(`path:/_matrix/client/v3/room_keys/keys/!roomA/sessionA0`, {
+                status: 404,
+                body: {
+                    errcode: "M_NOT_FOUND",
+                    error: "No backup found",
+                },
+            });
+
+            // @ts-ignore access to private function
+            const spy: SpyInstance = jest.spyOn(downloader, "queryKeyBackup");
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            await jest.runAllTimersAsync();
+
+            expect(spy).toHaveBeenCalledTimes(1);
+            const returnedPromise = spy.mock.results[0].value;
+            await expect(returnedPromise).rejects.toThrow("Failed to get key from backup: MISSING_DECRYPTION_KEY");
+
+            // Should not query again for a key not in backup
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            await jest.runAllTimersAsync();
+
+            expect(spy).toHaveBeenCalledTimes(1);
+
+            // advance time to retry
+            jest.advanceTimersByTime(BACKOFF_TIME + 10);
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            await jest.runAllTimersAsync();
+
+            expect(spy).toHaveBeenCalledTimes(2);
+            await expect(spy.mock.results[1].value).rejects.toThrow(
+                "Failed to get key from backup: MISSING_DECRYPTION_KEY",
+            );
+        });
+
+        it("Should stop properly", async () => {
+            // Simulate a call to stop while request is in flight
+            const blockOnServerRequest = defer<void>();
+            const requestRoomKeyCalled = defer<void>();
+
+            // Mock the request to block
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`, async (url, request) => {
+                requestRoomKeyCalled.resolve();
+                await blockOnServerRequest.promise;
+                return mockCipherKey;
+            });
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA1");
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA2");
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA3");
+
+            await requestRoomKeyCalled.promise;
+            downloader.stop();
+
+            blockOnServerRequest.resolve();
+
+            // let the first request complete
+            await jest.runAllTimersAsync();
+
+            expect(mockRustBackupManager.importBackedUpRoomKeys).not.toHaveBeenCalled();
+            expect(
+                fetchMock.calls(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`).length,
+            ).toStrictEqual(1);
+        });
+    });
+
+    describe("Given no usable backup available", () => {
+        let getConfigSpy: SpyInstance;
+
+        beforeEach(async () => {
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(null);
+            mockOlmMachine.getBackupKeys.mockResolvedValue(null);
+
+            // @ts-ignore access to private function
+            getConfigSpy = jest.spyOn(downloader, "getOrCreateBackupConfiguration");
+        });
+
+        it("Should not query server if no backup", async () => {
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", {
+                status: 404,
+                body: { errcode: "M_NOT_FOUND", error: "No current backup version." },
+            });
+
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+
+            await jest.runAllTimersAsync();
+
+            expect(getConfigSpy).toHaveBeenCalledTimes(1);
+            expect(getConfigSpy).toHaveReturnedWith(Promise.resolve(null));
+        });
+
+        it("Should not query server if backup not active", async () => {
+            // there is a backup
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+
+            // but it's not trusted
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(null);
+
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+
+            await jest.runAllTimersAsync();
+
+            expect(getConfigSpy).toHaveBeenCalledTimes(1);
+            expect(getConfigSpy).toHaveReturnedWith(Promise.resolve(null));
+        });
+
+        it("Should stop if backup key is not cached", async () => {
+            // there is a backup
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+            // it is trusted
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA.version!);
+            // but the key is not cached
+            mockOlmMachine.getBackupKeys.mockResolvedValue(null);
+
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+
+            await jest.runAllTimersAsync();
+
+            expect(getConfigSpy).toHaveBeenCalledTimes(1);
+            expect(getConfigSpy).toHaveReturnedWith(Promise.resolve(null));
+        });
+
+        it("Should stop if backup key cached as wrong version", async () => {
+            // there is a backup
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+            // it is trusted
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA.version!);
+            // but the cached key has the wrong version
+            mockOlmMachine.getBackupKeys.mockResolvedValue({
+                backupVersion: "0",
+                decryptionKey: RustSdkCryptoJs.BackupDecryptionKey.fromBase64(TestData.BACKUP_DECRYPTION_KEY_BASE64),
+            } as unknown as RustSdkCryptoJs.BackupKeys);
+
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+
+            await jest.runAllTimersAsync();
+
+            expect(getConfigSpy).toHaveBeenCalledTimes(1);
+            expect(getConfigSpy).toHaveReturnedWith(Promise.resolve(null));
+        });
+
+        it("Should stop if backup key version does not match the active one", async () => {
+            // there is a backup
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+            // The sdk is out of sync, the trusted version is the old one
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue("0");
+            // key for old backup cached
+            mockOlmMachine.getBackupKeys.mockResolvedValue({
+                backupVersion: "0",
+                decryptionKey: RustSdkCryptoJs.BackupDecryptionKey.fromBase64(TestData.BACKUP_DECRYPTION_KEY_BASE64),
+            } as unknown as RustSdkCryptoJs.BackupKeys);
+
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+
+            await jest.runAllTimersAsync();
+
+            expect(getConfigSpy).toHaveBeenCalledTimes(1);
+            expect(getConfigSpy).toHaveReturnedWith(Promise.resolve(null));
+        });
+    });
+
+    describe("Given Backup state update", () => {
+        it("After initial sync, when backup becomes trusted it should request keys for past requests", async () => {
+            // there is a backup
+            mockRustBackupManager.requestKeyBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA);
+
+            // but at this point it's not trusted and we don't have the key
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(null);
+            mockOlmMachine.getBackupKeys.mockResolvedValue(null);
+
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`, mockCipherKey);
+
+            const a0Imported = expectSessionImported("!roomA", "sessionA0");
+            const a1Imported = expectSessionImported("!roomA", "sessionA1");
+            const b1Imported = expectSessionImported("!roomB", "sessionB1");
+            const c1Imported = expectSessionImported("!roomC", "sessionC1");
+
+            // During initial sync several keys are requested
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA1");
+            downloader.onDecryptionKeyMissingError("!roomB", "sessionB1");
+            downloader.onDecryptionKeyMissingError("!roomC", "sessionC1");
+            await jest.runAllTimersAsync();
+
+            // @ts-ignore access to private property
+            expect(downloader.hasConfigurationProblem).toEqual(true);
+
+            // Now the backup becomes trusted
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA.version!);
+            // And we have the key in cache
+            mockOlmMachine.getBackupKeys.mockResolvedValue({
+                backupVersion: TestData.SIGNED_BACKUP_DATA.version!,
+                decryptionKey: RustSdkCryptoJs.BackupDecryptionKey.fromBase64(TestData.BACKUP_DECRYPTION_KEY_BASE64),
+            } as unknown as RustSdkCryptoJs.BackupKeys);
+
+            // In that case the sdk would fire a backup status update
+            mockEmitter.emit(CryptoEvent.KeyBackupStatus, true);
+
+            await jest.runAllTimersAsync();
+
+            await a0Imported;
+            await a1Imported;
+            await b1Imported;
+            await c1Imported;
+        });
+    });
+
+    describe("Error cases", () => {
+        beforeEach(async () => {
+            // there is a backup
+            mockRustBackupManager.requestKeyBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA);
+            // It's trusted
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA.version!);
+            // And we have the key in cache
+            mockOlmMachine.getBackupKeys.mockResolvedValue({
+                backupVersion: TestData.SIGNED_BACKUP_DATA.version!,
+                decryptionKey: RustSdkCryptoJs.BackupDecryptionKey.fromBase64(TestData.BACKUP_DECRYPTION_KEY_BASE64),
+            } as unknown as RustSdkCryptoJs.BackupKeys);
+        });
+
+        it("Should wait on rate limit error", async () => {
+            // simulate rate limit error
+            fetchMock.get(
+                `express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`,
+                {
+                    status: 429,
+                    body: {
+                        errcode: "M_LIMIT_EXCEEDED",
+                        error: "Too many requests",
+                        retry_after_ms: 5000,
+                    },
+                },
+                { overwriteRoutes: true },
+            );
+
+            const keyImported = expectSessionImported("!roomA", "sessionA0");
+
+            // @ts-ignore
+            const originalImplementation = downloader.queryKeyBackup.bind(downloader);
+
+            // @ts-ignore access to private function
+            const keyQuerySpy: SpyInstance = jest.spyOn(downloader, "queryKeyBackup");
+            const rateDeferred = defer<void>();
+
+            keyQuerySpy.mockImplementation(
+                // @ts-ignore
+                async (targetRoomId: string, targetSessionId: string, configuration: any) => {
+                    try {
+                        return await originalImplementation(targetRoomId, targetSessionId, configuration);
+                    } catch (err: any) {
+                        if (err.name === "KeyDownloadRateLimitError") {
+                            rateDeferred.resolve();
+                        }
+                        throw err;
+                    }
+                },
+            );
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+
+            await rateDeferred.promise;
+            expect(keyQuerySpy).toHaveBeenCalledTimes(1);
+            await expect(keyQuerySpy.mock.results[0].value).rejects.toThrow(
+                "Failed to get key from backup: rate limited",
+            );
+
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`, mockCipherKey, {
+                overwriteRoutes: true,
+            });
+
+            // Advance less than the retry_after_ms
+            jest.advanceTimersByTime(100);
+            // let any pending callbacks in PromiseJobs run
+            await Promise.resolve();
+            // no additional call should have been made
+            expect(keyQuerySpy).toHaveBeenCalledTimes(1);
+
+            // The loop should resume after the retry_after_ms
+            jest.advanceTimersByTime(5000);
+            // let any pending callbacks in PromiseJobs run
+            await Promise.resolve();
+
+            await keyImported;
+            expect(keyQuerySpy).toHaveBeenCalledTimes(2);
+        });
+
+        it("After a network error the same key is retried", async () => {
+            // simulate connectivity error
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`, () => {
+                throw new ConnectionError("fetch failed", new Error("fetch failed"));
+            });
+
+            // @ts-ignore
+            const originalImplementation = downloader.queryKeyBackup.bind(downloader);
+
+            // @ts-ignore
+            const keyQuerySpy: SpyInstance = jest.spyOn(downloader, "queryKeyBackup");
+            const errorDeferred = defer<void>();
+
+            keyQuerySpy.mockImplementation(
+                // @ts-ignore
+                async (targetRoomId: string, targetSessionId: string, configuration: any) => {
+                    try {
+                        return await originalImplementation(targetRoomId, targetSessionId, configuration);
+                    } catch (err: any) {
+                        if (err.name === "KeyDownloadError") {
+                            errorDeferred.resolve();
+                        }
+                        throw err;
+                    }
+                },
+            );
+            const keyImported = expectSessionImported("!roomA", "sessionA0");
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            await errorDeferred.promise;
+            await Promise.resolve();
+
+            await expect(keyQuerySpy.mock.results[0].value).rejects.toThrow(
+                "Failed to get key from backup: NETWORK_ERROR",
+            );
+
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`, mockCipherKey, {
+                overwriteRoutes: true,
+            });
+
+            // Advance less than the retry_after_ms
+            jest.advanceTimersByTime(100);
+            // let any pending callbacks in PromiseJobs run
+            await Promise.resolve();
+            // no additional call should have been made
+            expect(keyQuerySpy).toHaveBeenCalledTimes(1);
+
+            // The loop should resume after the retry_after_ms
+            jest.advanceTimersByTime(BACKOFF_TIME + 100);
+            await Promise.resolve();
+
+            await keyImported;
+        });
+
+        it("On Unknown error on import skip the key and continue", async () => {
+            const keyImported = defer<void>();
+            mockRustBackupManager.importBackedUpRoomKeys
+                .mockImplementationOnce(async () => {
+                    throw new Error("Didn't work");
+                })
+                .mockImplementationOnce(async (sessions) => {
+                    const roomId = sessions[0].room_id;
+                    const sessionId = sessions[0].session_id;
+                    if (roomId === "!roomA" && sessionId === "sessionA1") {
+                        keyImported.resolve();
+                    }
+                    return;
+                });
+
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`, mockCipherKey, {
+                overwriteRoutes: true,
+            });
+
+            // @ts-ignore access to private function
+            const keyQuerySpy: SpyInstance = jest.spyOn(downloader, "queryKeyBackup");
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA1");
+            await jest.runAllTimersAsync();
+
+            await keyImported.promise;
+
+            expect(keyQuerySpy).toHaveBeenCalledTimes(2);
+            expect(mockRustBackupManager.importBackedUpRoomKeys).toHaveBeenCalledTimes(2);
+        });
+    });
+});

spec/unit/rust-crypto/RoomEncryptor.spec.ts

@@ -16,16 +16,169 @@
  * /
  */
 
-import { HistoryVisibility as RustHistoryVisibility } from "@matrix-org/matrix-sdk-crypto-wasm";
+import {
+    Curve25519PublicKey,
+    Ed25519PublicKey,
+    HistoryVisibility as RustHistoryVisibility,
+    IdentityKeys,
+    OlmMachine,
+} from "@matrix-org/matrix-sdk-crypto-wasm";
+import { Mocked } from "jest-mock";
 
-import { HistoryVisibility } from "../../../src";
-import { toRustHistoryVisibility } from "../../../src/rust-crypto/RoomEncryptor";
+import { HistoryVisibility, MatrixEvent, Room, RoomMember } from "../../../src";
+import { RoomEncryptor, toRustHistoryVisibility } from "../../../src/rust-crypto/RoomEncryptor";
+import { KeyClaimManager } from "../../../src/rust-crypto/KeyClaimManager";
+import { defer } from "../../../src/utils";
+import { OutgoingRequestsManager } from "../../../src/rust-crypto/OutgoingRequestsManager";
 
-it.each([
-    [HistoryVisibility.Invited, RustHistoryVisibility.Invited],
-    [HistoryVisibility.Joined, RustHistoryVisibility.Joined],
-    [HistoryVisibility.Shared, RustHistoryVisibility.Shared],
-    [HistoryVisibility.WorldReadable, RustHistoryVisibility.WorldReadable],
-])("JS HistoryVisibility to Rust HistoryVisibility: converts %s to %s", (historyVisibility, expected) => {
-    expect(toRustHistoryVisibility(historyVisibility)).toBe(expected);
+describe("RoomEncryptor", () => {
+    describe("History Visibility", () => {
+        it.each([
+            [HistoryVisibility.Invited, RustHistoryVisibility.Invited],
+            [HistoryVisibility.Joined, RustHistoryVisibility.Joined],
+            [HistoryVisibility.Shared, RustHistoryVisibility.Shared],
+            [HistoryVisibility.WorldReadable, RustHistoryVisibility.WorldReadable],
+        ])("JS HistoryVisibility to Rust HistoryVisibility: converts %s to %s", (historyVisibility, expected) => {
+            expect(toRustHistoryVisibility(historyVisibility)).toBe(expected);
+        });
+    });
+
+    describe("RoomEncryptor", () => {
+        /** The room encryptor under test */
+        let roomEncryptor: RoomEncryptor;
+
+        let mockOlmMachine: Mocked<OlmMachine>;
+        let mockKeyClaimManager: Mocked<KeyClaimManager>;
+        let mockOutgoingRequestManager: Mocked<OutgoingRequestsManager>;
+        let mockRoom: Mocked<Room>;
+
+        const mockRoomMember = {
+            userId: "@alice:example.org",
+            membership: "join",
+        } as unknown as Mocked<RoomMember>;
+
+        function createMockEvent(text: string): Mocked<MatrixEvent> {
+            return {
+                getTxnId: jest.fn().mockReturnValue(""),
+                getType: jest.fn().mockReturnValue("m.room.message"),
+                getContent: jest.fn().mockReturnValue({
+                    body: text,
+                    msgtype: "m.text",
+                }),
+                makeEncrypted: jest.fn().mockReturnValue(undefined),
+            } as unknown as Mocked<MatrixEvent>;
+        }
+
+        beforeEach(() => {
+            mockOlmMachine = {
+                identityKeys: {
+                    curve25519: {
+                        toBase64: jest.fn().mockReturnValue("curve25519"),
+                    } as unknown as Curve25519PublicKey,
+                    ed25519: {
+                        toBase64: jest.fn().mockReturnValue("ed25519"),
+                    } as unknown as Ed25519PublicKey,
+                } as unknown as Mocked<IdentityKeys>,
+                shareRoomKey: jest.fn(),
+                updateTrackedUsers: jest.fn().mockResolvedValue(undefined),
+                encryptRoomEvent: jest.fn().mockResolvedValue("{}"),
+            } as unknown as Mocked<OlmMachine>;
+
+            mockKeyClaimManager = {
+                ensureSessionsForUsers: jest.fn(),
+            } as unknown as Mocked<KeyClaimManager>;
+
+            mockOutgoingRequestManager = {
+                doProcessOutgoingRequests: jest.fn().mockResolvedValue(undefined),
+            } as unknown as Mocked<OutgoingRequestsManager>;
+
+            mockRoom = {
+                roomId: "!foo:example.org",
+                getJoinedMembers: jest.fn().mockReturnValue([mockRoomMember]),
+                getEncryptionTargetMembers: jest.fn().mockReturnValue([mockRoomMember]),
+                shouldEncryptForInvitedMembers: jest.fn().mockReturnValue(true),
+                getHistoryVisibility: jest.fn().mockReturnValue(HistoryVisibility.Invited),
+                getBlacklistUnverifiedDevices: jest.fn().mockReturnValue(false),
+            } as unknown as Mocked<Room>;
+
+            roomEncryptor = new RoomEncryptor(
+                mockOlmMachine,
+                mockKeyClaimManager,
+                mockOutgoingRequestManager,
+                mockRoom,
+                { algorithm: "m.megolm.v1.aes-sha2" },
+            );
+        });
+
+        it("should ensure that there is only one shareRoomKey at a time", async () => {
+            const deferredShare = defer<void>();
+            const insideOlmShareRoom = defer<void>();
+
+            mockOlmMachine.shareRoomKey.mockImplementationOnce(async () => {
+                insideOlmShareRoom.resolve();
+                await deferredShare.promise;
+            });
+
+            roomEncryptor.prepareForEncryption(false);
+            await insideOlmShareRoom.promise;
+
+            // call several times more
+            roomEncryptor.prepareForEncryption(false);
+            roomEncryptor.encryptEvent(createMockEvent("Hello"), false);
+            roomEncryptor.prepareForEncryption(false);
+            roomEncryptor.encryptEvent(createMockEvent("World"), false);
+
+            expect(mockOlmMachine.shareRoomKey).toHaveBeenCalledTimes(1);
+
+            deferredShare.resolve();
+            await roomEncryptor.prepareForEncryption(false);
+
+            // should have been called again
+            expect(mockOlmMachine.shareRoomKey).toHaveBeenCalledTimes(6);
+        });
+
+        // Regression test for https://github.com/element-hq/element-web/issues/26684
+        it("Should maintain order of encryption requests", async () => {
+            const firstTargetMembers = defer<void>();
+            const secondTargetMembers = defer<void>();
+
+            mockOlmMachine.shareRoomKey.mockResolvedValue(undefined);
+
+            // Hook into this method to demonstrate the race condition
+            mockRoom.getEncryptionTargetMembers
+                .mockImplementationOnce(async () => {
+                    await firstTargetMembers.promise;
+                    return [mockRoomMember];
+                })
+                .mockImplementationOnce(async () => {
+                    await secondTargetMembers.promise;
+                    return [mockRoomMember];
+                });
+
+            let firstMessageFinished: string | null = null;
+
+            const firstRequest = roomEncryptor.encryptEvent(createMockEvent("Hello"), false);
+            const secondRequest = roomEncryptor.encryptEvent(createMockEvent("Edit of Hello"), false);
+
+            firstRequest.then(() => {
+                if (firstMessageFinished === null) {
+                    firstMessageFinished = "hello";
+                }
+            });
+
+            secondRequest.then(() => {
+                if (firstMessageFinished === null) {
+                    firstMessageFinished = "edit";
+                }
+            });
+
+            // suppose the second getEncryptionTargetMembers call returns first
+            secondTargetMembers.resolve();
+            firstTargetMembers.resolve();
+
+            await Promise.all([firstRequest, secondRequest]);
+
+            expect(firstMessageFinished).toBe("hello");
+        });
+    });
 });

spec/unit/rust-crypto/backup.spec.ts

@@ -0,0 +1,144 @@
+import { Mocked } from "jest-mock";
+import fetchMock from "fetch-mock-jest";
+import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
+
+import { CryptoEvent, HttpApiEvent, HttpApiEventHandlerMap, MatrixHttpApi, TypedEventEmitter } from "../../../src";
+import { OutgoingRequestProcessor } from "../../../src/rust-crypto/OutgoingRequestProcessor";
+import * as testData from "../../test-utils/test-data";
+import * as TestData from "../../test-utils/test-data";
+import { IKeyBackup } from "../../../src/crypto/backup";
+import { IKeyBackupSession } from "../../../src/crypto/keybackup";
+import { RustBackupManager } from "../../../src/rust-crypto/backup";
+
+describe("Upload keys to backup", () => {
+    /** The backup manager under test */
+    let rustBackupManager: RustBackupManager;
+
+    let mockOlmMachine: Mocked<RustSdkCryptoJs.OlmMachine>;
+
+    let outgoingRequestProcessor: Mocked<OutgoingRequestProcessor>;
+
+    const httpAPi = new MatrixHttpApi(new TypedEventEmitter<HttpApiEvent, HttpApiEventHandlerMap>(), {
+        baseUrl: "http://server/",
+        prefix: "",
+        onlyData: true,
+    });
+
+    let idGenerator = 0;
+    function mockBackupRequest(keyCount: number): RustSdkCryptoJs.KeysBackupRequest {
+        const requestBody: IKeyBackup = {
+            rooms: {
+                "!room1:server": {
+                    sessions: {},
+                },
+            },
+        };
+        for (let i = 0; i < keyCount; i++) {
+            requestBody.rooms["!room1:server"].sessions["session" + i] = {} as IKeyBackupSession;
+        }
+        return {
+            id: "id" + idGenerator++,
+            body: JSON.stringify(requestBody),
+        } as unknown as Mocked<RustSdkCryptoJs.KeysBackupRequest>;
+    }
+
+    beforeEach(async () => {
+        jest.useFakeTimers();
+        idGenerator = 0;
+
+        mockOlmMachine = {
+            getBackupKeys: jest.fn().mockResolvedValue({
+                backupVersion: TestData.SIGNED_BACKUP_DATA.version!,
+                decryptionKey: RustSdkCryptoJs.BackupDecryptionKey.fromBase64(TestData.BACKUP_DECRYPTION_KEY_BASE64),
+            } as unknown as RustSdkCryptoJs.BackupKeys),
+            backupRoomKeys: jest.fn(),
+            isBackupEnabled: jest.fn().mockResolvedValue(true),
+            enableBackupV1: jest.fn(),
+            verifyBackup: jest.fn().mockResolvedValue({
+                trusted: jest.fn().mockResolvedValue(true),
+            } as unknown as RustSdkCryptoJs.SignatureVerification),
+            roomKeyCounts: jest.fn(),
+        } as unknown as Mocked<RustSdkCryptoJs.OlmMachine>;
+
+        outgoingRequestProcessor = {
+            makeOutgoingRequest: jest.fn(),
+        } as unknown as Mocked<OutgoingRequestProcessor>;
+
+        rustBackupManager = new RustBackupManager(mockOlmMachine, httpAPi, outgoingRequestProcessor);
+
+        fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+    });
+
+    afterEach(() => {
+        fetchMock.reset();
+        jest.useRealTimers();
+        jest.resetAllMocks();
+    });
+
+    it("Should call expensive roomKeyCounts only once per loop", async () => {
+        const remainingEmitted: number[] = [];
+
+        const zeroRemainingWasEmitted = new Promise<void>((resolve) => {
+            rustBackupManager.on(CryptoEvent.KeyBackupSessionsRemaining, (count) => {
+                remainingEmitted.push(count);
+                if (count == 0) {
+                    resolve();
+                }
+            });
+        });
+
+        // We want several batch of keys to check that we don't call expensive room key count several times
+        mockOlmMachine.backupRoomKeys
+            .mockResolvedValueOnce(mockBackupRequest(100))
+            .mockResolvedValueOnce(mockBackupRequest(100))
+            .mockResolvedValueOnce(mockBackupRequest(100))
+            .mockResolvedValueOnce(mockBackupRequest(100))
+            .mockResolvedValueOnce(mockBackupRequest(100))
+            .mockResolvedValueOnce(mockBackupRequest(100))
+            .mockResolvedValueOnce(mockBackupRequest(2))
+            .mockResolvedValue(null);
+
+        mockOlmMachine.roomKeyCounts.mockResolvedValue({
+            total: 602,
+            // First iteration won't call roomKeyCounts(); it will be called on the second iteration after 200 keys have been saved.
+            backedUp: 200,
+        });
+
+        await rustBackupManager.checkKeyBackupAndEnable(false);
+        await jest.runAllTimersAsync();
+
+        await zeroRemainingWasEmitted;
+
+        expect(outgoingRequestProcessor.makeOutgoingRequest).toHaveBeenCalledTimes(7);
+        expect(mockOlmMachine.roomKeyCounts).toHaveBeenCalledTimes(1);
+
+        // check event emission
+        expect(remainingEmitted[0]).toEqual(402);
+        expect(remainingEmitted[1]).toEqual(302);
+        expect(remainingEmitted[2]).toEqual(202);
+        expect(remainingEmitted[3]).toEqual(102);
+        expect(remainingEmitted[4]).toEqual(2);
+        expect(remainingEmitted[5]).toEqual(0);
+    });
+
+    it("Should not call expensive roomKeyCounts when only one iteration is needed", async () => {
+        const zeroRemainingWasEmitted = new Promise<void>((resolve) => {
+            rustBackupManager.on(CryptoEvent.KeyBackupSessionsRemaining, (count) => {
+                if (count == 0) {
+                    resolve();
+                }
+            });
+        });
+
+        // Only returns 2 keys on the first call, then none.
+        mockOlmMachine.backupRoomKeys.mockResolvedValueOnce(mockBackupRequest(2)).mockResolvedValue(null);
+
+        await rustBackupManager.checkKeyBackupAndEnable(false);
+        await jest.runAllTimersAsync();
+
+        await zeroRemainingWasEmitted;
+
+        expect(outgoingRequestProcessor.makeOutgoingRequest).toHaveBeenCalledTimes(1);
+        expect(mockOlmMachine.roomKeyCounts).toHaveBeenCalledTimes(0);
+    });
+});

spec/unit/rust-crypto/rust-crypto.spec.ts

@@ -15,7 +15,15 @@ limitations under the License.
 */
 
 import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
-import { KeysQueryRequest, OlmMachine } from "@matrix-org/matrix-sdk-crypto-wasm";
+import {
+    BaseMigrationData,
+    KeysQueryRequest,
+    Migration,
+    OlmMachine,
+    PickledInboundGroupSession,
+    PickledSession,
+    StoreHandle,
+} from "@matrix-org/matrix-sdk-crypto-wasm";
 import { mocked, Mocked } from "jest-mock";
 import fetchMock from "fetch-mock-jest";
 
@@ -25,6 +33,7 @@ import {
     CryptoEvent,
     Device,
     DeviceVerification,
+    encodeBase64,
     HttpApiEvent,
     HttpApiEventHandlerMap,
     IHttpOpts,
@@ -32,13 +41,20 @@ import {
     MatrixClient,
     MatrixEvent,
     MatrixHttpApi,
+    MemoryCryptoStore,
     TypedEventEmitter,
 } from "../../../src";
 import { mkEvent } from "../../test-utils/test-utils";
 import { CryptoBackend } from "../../../src/common-crypto/CryptoBackend";
-import { IEventDecryptionResult } from "../../../src/@types/crypto";
+import { IEventDecryptionResult, IMegolmSessionData } from "../../../src/@types/crypto";
 import { OutgoingRequestProcessor } from "../../../src/rust-crypto/OutgoingRequestProcessor";
-import { ServerSideSecretStorage } from "../../../src/secret-storage";
+import {
+    AccountDataClient,
+    AddSecretStorageKeyOpts,
+    SecretStorageCallbacks,
+    ServerSideSecretStorage,
+    ServerSideSecretStorageImpl,
+} from "../../../src/secret-storage";
 import {
     CryptoCallbacks,
     EventShieldColour,
@@ -50,10 +66,21 @@ import {
 import * as testData from "../../test-utils/test-data";
 import { defer } from "../../../src/utils";
 import { logger } from "../../../src/logger";
+import { OutgoingRequestsManager } from "../../../src/rust-crypto/OutgoingRequestsManager";
+import { ClientEvent, ClientEventHandlerMap } from "../../../src/client";
+import { Curve25519AuthData } from "../../../src/crypto-api/keybackup";
+import { encryptAES } from "../../../src/crypto/aes";
+import { CryptoStore, SecretStorePrivateKeys } from "../../../src/crypto/store/base";
 
 const TEST_USER = "@alice:example.com";
 const TEST_DEVICE_ID = "TEST_DEVICE";
 
+beforeAll(async () => {
+    // Load the WASM upfront, before any of the tests. This can take some time, and doing it here means that it gets
+    // a separate timeout.
+    await RustSdkCryptoJs.initAsync();
+}, 15000);
+
 afterEach(() => {
     fetchMock.reset();
     jest.restoreAllMocks();
@@ -64,71 +91,287 @@ describe("initRustCrypto", () => {
         return {
             registerRoomKeyUpdatedCallback: jest.fn(),
             registerUserIdentityUpdatedCallback: jest.fn(),
-            getSecretsFromInbox: jest.fn().mockResolvedValue(["dGhpc2lzYWZha2VzZWNyZXQ="]),
+            getSecretsFromInbox: jest.fn().mockResolvedValue([]),
             deleteSecretsFromInbox: jest.fn(),
             registerReceiveSecretCallback: jest.fn(),
+            registerDevicesUpdatedCallback: jest.fn(),
             outgoingRequests: jest.fn(),
+            isBackupEnabled: jest.fn().mockResolvedValue(false),
+            verifyBackup: jest.fn().mockResolvedValue({ trusted: jest.fn().mockReturnValue(false) }),
+            getBackupKeys: jest.fn(),
         } as unknown as Mocked<OlmMachine>;
     }
 
     it("passes through the store params", async () => {
+        const mockStore = { free: jest.fn() } as unknown as StoreHandle;
+        jest.spyOn(StoreHandle, "open").mockResolvedValue(mockStore);
+
         const testOlmMachine = makeTestOlmMachine();
-        jest.spyOn(OlmMachine, "initialize").mockResolvedValue(testOlmMachine);
+        jest.spyOn(OlmMachine, "initFromStore").mockResolvedValue(testOlmMachine);
 
-        await initRustCrypto(
+        await initRustCrypto({
             logger,
-            {} as MatrixClient["http"],
-            TEST_USER,
-            TEST_DEVICE_ID,
-            {} as ServerSideSecretStorage,
-            {} as CryptoCallbacks,
-            "storePrefix",
-            "storePassphrase",
-        );
+            http: {} as MatrixClient["http"],
+            userId: TEST_USER,
+            deviceId: TEST_DEVICE_ID,
+            secretStorage: {} as ServerSideSecretStorage,
+            cryptoCallbacks: {} as CryptoCallbacks,
+            storePrefix: "storePrefix",
+            storePassphrase: "storePassphrase",
+        });
 
-        expect(OlmMachine.initialize).toHaveBeenCalledWith(
-            expect.anything(),
-            expect.anything(),
-            "storePrefix",
-            "storePassphrase",
-        );
+        expect(StoreHandle.open).toHaveBeenCalledWith("storePrefix", "storePassphrase");
+        expect(OlmMachine.initFromStore).toHaveBeenCalledWith(expect.anything(), expect.anything(), mockStore);
     });
 
     it("suppresses the storePassphrase if storePrefix is unset", async () => {
+        const mockStore = { free: jest.fn() } as unknown as StoreHandle;
+        jest.spyOn(StoreHandle, "open").mockResolvedValue(mockStore);
+
         const testOlmMachine = makeTestOlmMachine();
-        jest.spyOn(OlmMachine, "initialize").mockResolvedValue(testOlmMachine);
+        jest.spyOn(OlmMachine, "initFromStore").mockResolvedValue(testOlmMachine);
 
-        await initRustCrypto(
+        await initRustCrypto({
             logger,
-            {} as MatrixClient["http"],
-            TEST_USER,
-            TEST_DEVICE_ID,
-            {} as ServerSideSecretStorage,
-            {} as CryptoCallbacks,
-            null,
-            "storePassphrase",
-        );
+            http: {} as MatrixClient["http"],
+            userId: TEST_USER,
+            deviceId: TEST_DEVICE_ID,
+            secretStorage: {} as ServerSideSecretStorage,
+            cryptoCallbacks: {} as CryptoCallbacks,
+            storePrefix: null,
+            storePassphrase: "storePassphrase",
+        });
 
-        expect(OlmMachine.initialize).toHaveBeenCalledWith(expect.anything(), expect.anything(), undefined, undefined);
+        expect(StoreHandle.open).toHaveBeenCalledWith(undefined, undefined);
+        expect(OlmMachine.initFromStore).toHaveBeenCalledWith(expect.anything(), expect.anything(), mockStore);
     });
 
     it("Should get secrets from inbox on start", async () => {
-        const testOlmMachine = makeTestOlmMachine() as OlmMachine;
-        jest.spyOn(OlmMachine, "initialize").mockResolvedValue(testOlmMachine);
+        const mockStore = { free: jest.fn() } as unknown as StoreHandle;
+        jest.spyOn(StoreHandle, "open").mockResolvedValue(mockStore);
 
-        await initRustCrypto(
+        const testOlmMachine = makeTestOlmMachine();
+        jest.spyOn(OlmMachine, "initFromStore").mockResolvedValue(testOlmMachine);
+
+        await initRustCrypto({
             logger,
-            {} as MatrixClient["http"],
-            TEST_USER,
-            TEST_DEVICE_ID,
-            {} as ServerSideSecretStorage,
-            {} as CryptoCallbacks,
-            "storePrefix",
-            "storePassphrase",
-        );
+            http: {} as MatrixClient["http"],
+            userId: TEST_USER,
+            deviceId: TEST_DEVICE_ID,
+            secretStorage: {} as ServerSideSecretStorage,
+            cryptoCallbacks: {} as CryptoCallbacks,
+            storePrefix: "storePrefix",
+            storePassphrase: "storePassphrase",
+        });
 
         expect(testOlmMachine.getSecretsFromInbox).toHaveBeenCalledWith("m.megolm_backup.v1");
     });
+
+    describe("libolm migration", () => {
+        let mockStore: RustSdkCryptoJs.StoreHandle;
+
+        beforeEach(() => {
+            // Stub out a bunch of stuff in the Rust library
+            mockStore = { free: jest.fn() } as unknown as StoreHandle;
+            jest.spyOn(StoreHandle, "open").mockResolvedValue(mockStore);
+
+            jest.spyOn(Migration, "migrateBaseData").mockResolvedValue(undefined);
+            jest.spyOn(Migration, "migrateOlmSessions").mockResolvedValue(undefined);
+            jest.spyOn(Migration, "migrateMegolmSessions").mockResolvedValue(undefined);
+
+            const testOlmMachine = makeTestOlmMachine();
+            jest.spyOn(OlmMachine, "initFromStore").mockResolvedValue(testOlmMachine);
+        });
+
+        it("migrates data from a legacy crypto store", async () => {
+            const PICKLE_KEY = "pickle1234";
+            const legacyStore = new MemoryCryptoStore();
+
+            // Populate the legacy store with some test data
+            const storeSecretKey = (type: string, key: string) =>
+                encryptAndStoreSecretKey(type, new TextEncoder().encode(key), PICKLE_KEY, legacyStore);
+
+            await legacyStore.storeAccount({}, "not a real account");
+            await storeSecretKey("m.megolm_backup.v1", "backup key");
+            await storeSecretKey("master", "master key");
+            await storeSecretKey("self_signing", "ssk");
+            await storeSecretKey("user_signing", "usk");
+            const nDevices = 6;
+            const nSessionsPerDevice = 10;
+            createSessions(legacyStore, nDevices, nSessionsPerDevice);
+            createMegolmSessions(legacyStore, nDevices, nSessionsPerDevice);
+            await legacyStore.markSessionsNeedingBackup([{ senderKey: pad43("device5"), sessionId: "session5" }]);
+
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", { version: "45" });
+
+            function legacyMigrationProgressListener(progress: number, total: number): void {
+                logger.log(`migrated ${progress} of ${total}`);
+            }
+
+            await initRustCrypto({
+                logger,
+                http: makeMatrixHttpApi(),
+                userId: TEST_USER,
+                deviceId: TEST_DEVICE_ID,
+                secretStorage: {} as ServerSideSecretStorage,
+                cryptoCallbacks: {} as CryptoCallbacks,
+                storePrefix: "storePrefix",
+                storePassphrase: "storePassphrase",
+                legacyCryptoStore: legacyStore,
+                legacyPickleKey: PICKLE_KEY,
+                legacyMigrationProgressListener,
+            });
+
+            // Check that the migration functions were correctly called
+            expect(Migration.migrateBaseData).toHaveBeenCalledWith(
+                expect.any(BaseMigrationData),
+                new Uint8Array(Buffer.from(PICKLE_KEY)),
+                mockStore,
+            );
+            const data = mocked(Migration.migrateBaseData).mock.calls[0][0];
+            expect(data.pickledAccount).toEqual("not a real account");
+            expect(data.userId!.toString()).toEqual(TEST_USER);
+            expect(data.deviceId!.toString()).toEqual(TEST_DEVICE_ID);
+            expect(atob(data.backupRecoveryKey!)).toEqual("backup key");
+            expect(data.backupVersion).toEqual("45");
+            expect(atob(data.privateCrossSigningMasterKey!)).toEqual("master key");
+            expect(atob(data.privateCrossSigningUserSigningKey!)).toEqual("usk");
+            expect(atob(data.privateCrossSigningSelfSigningKey!)).toEqual("ssk");
+
+            expect(Migration.migrateOlmSessions).toHaveBeenCalledTimes(2);
+            expect(Migration.migrateOlmSessions).toHaveBeenCalledWith(
+                expect.any(Array),
+                new Uint8Array(Buffer.from(PICKLE_KEY)),
+                mockStore,
+            );
+            // First call should have 50 entries; second should have 10
+            const sessions1: PickledSession[] = mocked(Migration.migrateOlmSessions).mock.calls[0][0];
+            expect(sessions1.length).toEqual(50);
+            const sessions2: PickledSession[] = mocked(Migration.migrateOlmSessions).mock.calls[1][0];
+            expect(sessions2.length).toEqual(10);
+            const sessions = [...sessions1, ...sessions2];
+            for (let i = 0; i < nDevices; i++) {
+                for (let j = 0; j < nSessionsPerDevice; j++) {
+                    const session = sessions[i * nSessionsPerDevice + j];
+                    expect(session.senderKey).toEqual(`device${i}`);
+                    expect(session.pickle).toEqual(`session${i}.${j}`);
+                    expect(session.creationTime).toEqual(new Date(1000));
+                    expect(session.lastUseTime).toEqual(new Date(1000));
+                }
+            }
+
+            expect(Migration.migrateMegolmSessions).toHaveBeenCalledTimes(2);
+            expect(Migration.migrateMegolmSessions).toHaveBeenCalledWith(
+                expect.any(Array),
+                new Uint8Array(Buffer.from(PICKLE_KEY)),
+                mockStore,
+            );
+            // First call should have 50 entries; second should have 10
+            const megolmSessions1: PickledInboundGroupSession[] = mocked(Migration.migrateMegolmSessions).mock
+                .calls[0][0];
+            expect(megolmSessions1.length).toEqual(50);
+            const megolmSessions2: PickledInboundGroupSession[] = mocked(Migration.migrateMegolmSessions).mock
+                .calls[1][0];
+            expect(megolmSessions2.length).toEqual(10);
+            const megolmSessions = [...megolmSessions1, ...megolmSessions2];
+            for (let i = 0; i < nDevices; i++) {
+                for (let j = 0; j < nSessionsPerDevice; j++) {
+                    const session = megolmSessions[i * nSessionsPerDevice + j];
+                    expect(session.senderKey).toEqual(pad43(`device${i}`));
+                    expect(session.pickle).toEqual("sessionPickle");
+                    expect(session.roomId!.toString()).toEqual("!room:id");
+                    expect(session.senderSigningKey).toEqual("sender_signing_key");
+
+                    // only one of the sessions needs backing up
+                    expect(session.backedUp).toEqual(i !== 5 || j !== 5);
+                }
+            }
+        }, 10000);
+
+        it("handles megolm sessions with no `keysClaimed`", async () => {
+            const legacyStore = new MemoryCryptoStore();
+            legacyStore.storeAccount({}, "not a real account");
+
+            legacyStore.storeEndToEndInboundGroupSession(
+                pad43(`device1`),
+                `session1`,
+                {
+                    forwardingCurve25519KeyChain: [],
+                    room_id: "!room:id",
+                    session: "sessionPickle",
+                },
+                undefined,
+            );
+
+            const PICKLE_KEY = "pickle1234";
+            await initRustCrypto({
+                logger,
+                http: makeMatrixHttpApi(),
+                userId: TEST_USER,
+                deviceId: TEST_DEVICE_ID,
+                secretStorage: {} as ServerSideSecretStorage,
+                cryptoCallbacks: {} as CryptoCallbacks,
+                storePrefix: "storePrefix",
+                storePassphrase: "storePassphrase",
+                legacyCryptoStore: legacyStore,
+                legacyPickleKey: PICKLE_KEY,
+            });
+
+            expect(Migration.migrateMegolmSessions).toHaveBeenCalledTimes(1);
+            expect(Migration.migrateMegolmSessions).toHaveBeenCalledWith(
+                expect.any(Array),
+                new Uint8Array(Buffer.from(PICKLE_KEY)),
+                mockStore,
+            );
+            const megolmSessions: PickledInboundGroupSession[] = mocked(Migration.migrateMegolmSessions).mock
+                .calls[0][0];
+            expect(megolmSessions.length).toEqual(1);
+            const session = megolmSessions[0];
+            expect(session.senderKey).toEqual(pad43(`device1`));
+            expect(session.pickle).toEqual("sessionPickle");
+            expect(session.roomId!.toString()).toEqual("!room:id");
+            expect(session.senderSigningKey).toBe(undefined);
+        }, 10000);
+
+        async function encryptAndStoreSecretKey(type: string, key: Uint8Array, pickleKey: string, store: CryptoStore) {
+            const encryptedKey = await encryptAES(encodeBase64(key), Buffer.from(pickleKey), type);
+            store.storeSecretStorePrivateKey(undefined, type as keyof SecretStorePrivateKeys, encryptedKey);
+        }
+
+        /** Create a bunch of fake Olm sessions and stash them in the DB. */
+        function createSessions(store: CryptoStore, nDevices: number, nSessionsPerDevice: number) {
+            for (let i = 0; i < nDevices; i++) {
+                for (let j = 0; j < nSessionsPerDevice; j++) {
+                    const sessionData = {
+                        deviceKey: `device${i}`,
+                        sessionId: `session${j}`,
+                        session: `session${i}.${j}`,
+                        lastReceivedMessageTs: 1000,
+                    };
+                    store.storeEndToEndSession(`device${i}`, `session${j}`, sessionData, undefined);
+                }
+            }
+        }
+
+        /** Create a bunch of fake Megolm sessions and stash them in the DB. */
+        function createMegolmSessions(store: CryptoStore, nDevices: number, nSessionsPerDevice: number) {
+            for (let i = 0; i < nDevices; i++) {
+                for (let j = 0; j < nSessionsPerDevice; j++) {
+                    store.storeEndToEndInboundGroupSession(
+                        pad43(`device${i}`),
+                        `session${j}`,
+                        {
+                            forwardingCurve25519KeyChain: [],
+                            keysClaimed: { ed25519: "sender_signing_key" },
+                            room_id: "!room:id",
+                            session: "sessionPickle",
+                        },
+                        undefined,
+                    );
+                }
+            }
+        }
+    });
 });
 
 describe("RustCrypto", () => {
@@ -156,7 +399,7 @@ describe("RustCrypto", () => {
             let importTotal = 0;
             const opt: ImportRoomKeysOpts = {
                 progressCallback: (stage) => {
-                    importTotal = stage.total;
+                    importTotal = stage.total ?? 0;
                 },
             };
             await rustCrypto.importRoomKeys(someRoomKeys, opt);
@@ -292,6 +535,62 @@ describe("RustCrypto", () => {
         expect(mockCrossSigningIdentity.bootstrapCrossSigning).toHaveBeenCalledWith({});
     });
 
+    it("bootstrapSecretStorage creates new backup when requested", async () => {
+        const secretStorageCallbacks = {
+            getSecretStorageKey: async (keys: any, name: string) => {
+                return [[...Object.keys(keys.keys)][0], new Uint8Array(32)];
+            },
+        } as SecretStorageCallbacks;
+        const secretStorage = new ServerSideSecretStorageImpl(new DummyAccountDataClient(), secretStorageCallbacks);
+
+        const outgoingRequestProcessor = {
+            makeOutgoingRequest: jest.fn(),
+        } as unknown as Mocked<OutgoingRequestProcessor>;
+
+        const rustCrypto = await makeTestRustCrypto(
+            new MatrixHttpApi(new TypedEventEmitter<HttpApiEvent, HttpApiEventHandlerMap>(), {
+                baseUrl: "http://server/",
+                prefix: "",
+                onlyData: true,
+            }),
+            testData.TEST_USER_ID,
+            undefined,
+            secretStorage,
+        );
+
+        rustCrypto["checkKeyBackupAndEnable"] = async () => {
+            return null;
+        };
+        (rustCrypto["crossSigningIdentity"] as any)["outgoingRequestProcessor"] = outgoingRequestProcessor;
+        const resetKeyBackup = (rustCrypto["resetKeyBackup"] = jest.fn());
+
+        async function createSecretStorageKey() {
+            return {
+                keyInfo: {} as AddSecretStorageKeyOpts,
+                privateKey: new Uint8Array(32),
+            };
+        }
+
+        // create initial secret storage
+        await rustCrypto.bootstrapCrossSigning({ setupNewCrossSigning: true });
+        await rustCrypto.bootstrapSecretStorage({
+            createSecretStorageKey,
+            setupNewSecretStorage: true,
+            setupNewKeyBackup: true,
+        });
+        // check that rustCrypto.resetKeyBackup was called
+        expect(resetKeyBackup.mock.calls).toHaveLength(1);
+
+        // reset secret storage
+        await rustCrypto.bootstrapSecretStorage({
+            createSecretStorageKey,
+            setupNewSecretStorage: true,
+            setupNewKeyBackup: true,
+        });
+        // check that rustCrypto.resetKeyBackup was called again
+        expect(resetKeyBackup.mock.calls).toHaveLength(2);
+    });
+
     it("isSecretStorageReady", async () => {
         const mockSecretStorage = {
             getDefaultKeyId: jest.fn().mockResolvedValue(null),
@@ -347,6 +646,8 @@ describe("RustCrypto", () => {
                 makeOutgoingRequest: jest.fn(),
             } as unknown as Mocked<OutgoingRequestProcessor>;
 
+            const outgoingRequestsManager = new OutgoingRequestsManager(logger, olmMachine, outgoingRequestProcessor);
+
             rustCrypto = new RustCrypto(
                 logger,
                 olmMachine,
@@ -357,6 +658,7 @@ describe("RustCrypto", () => {
                 {} as CryptoCallbacks,
             );
             rustCrypto["outgoingRequestProcessor"] = outgoingRequestProcessor;
+            rustCrypto["outgoingRequestsManager"] = outgoingRequestsManager;
         });
 
         it("should poll for outgoing messages and send them", async () => {
@@ -396,48 +698,56 @@ describe("RustCrypto", () => {
             expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(2);
         });
 
-        it("stops looping when stop() is called", async () => {
-            for (let i = 0; i < 5; i++) {
-                outgoingRequestQueue.push([new KeysQueryRequest("1234", "{}")]);
-            }
+        it("should encode outgoing requests properly", async () => {
+            // we need a real OlmMachine, so replace the one created by beforeEach
+            rustCrypto = await makeTestRustCrypto();
+            const olmMachine: OlmMachine = rustCrypto["olmMachine"];
 
-            let makeRequestPromise = awaitCallToMakeOutgoingRequest();
+            const outgoingRequestProcessor = {} as unknown as OutgoingRequestProcessor;
+            rustCrypto["outgoingRequestProcessor"] = outgoingRequestProcessor;
+            const outgoingRequestsManager = new OutgoingRequestsManager(logger, olmMachine, outgoingRequestProcessor);
+            rustCrypto["outgoingRequestsManager"] = outgoingRequestsManager;
 
-            rustCrypto.onSyncCompleted({});
-
-            expect(rustCrypto["outgoingRequestLoopRunning"]).toBeTruthy();
-
-            // go a couple of times round the loop
-            let resolveMakeRequest = await makeRequestPromise;
-            makeRequestPromise = awaitCallToMakeOutgoingRequest();
-            resolveMakeRequest();
-
-            resolveMakeRequest = await makeRequestPromise;
-            makeRequestPromise = awaitCallToMakeOutgoingRequest();
-            resolveMakeRequest();
-
-            // a second sync while this is going on shouldn't make any difference
-            rustCrypto.onSyncCompleted({});
-
-            resolveMakeRequest = await makeRequestPromise;
-            outgoingRequestProcessor.makeOutgoingRequest.mockReset();
-            resolveMakeRequest();
-
-            // now stop...
-            rustCrypto.stop();
-
-            // which should (eventually) cause the loop to stop with no further calls to outgoingRequests
-            olmMachine.outgoingRequests.mockReset();
-
-            await new Promise((resolve) => {
-                setTimeout(resolve, 100);
+            // The second time we do a /keys/upload, the `device_keys` property
+            // should be absent from the request body
+            // cf. https://github.com/matrix-org/matrix-rust-sdk-crypto-wasm/issues/57
+            //
+            // On the first upload, we pretend that there are no OTKs, so it will
+            // try to upload more keys
+            let keysUploadCount = 0;
+            let deviceKeys: object;
+            let deviceKeysAbsent = false;
+            outgoingRequestProcessor.makeOutgoingRequest = jest.fn(async (request, uiaCallback?) => {
+                let resp: any = {};
+                if (request instanceof RustSdkCryptoJs.KeysUploadRequest) {
+                    if (keysUploadCount == 0) {
+                        deviceKeys = JSON.parse(request.body).device_keys;
+                        resp = { one_time_key_counts: { signed_curve25519: 0 } };
+                    } else {
+                        deviceKeysAbsent = !("device_keys" in JSON.parse(request.body));
+                        resp = { one_time_key_counts: { signed_curve25519: 50 } };
+                    }
+                    keysUploadCount++;
+                } else if (request instanceof RustSdkCryptoJs.KeysQueryRequest) {
+                    resp = {
+                        device_keys: {
+                            [TEST_USER]: {
+                                [TEST_DEVICE_ID]: deviceKeys,
+                            },
+                        },
+                    };
+                } else if (request instanceof RustSdkCryptoJs.UploadSigningKeysRequest) {
+                    // SigningKeysUploadRequest does not implement OutgoingRequest and does not need to be marked as sent.
+                    return;
+                }
+                if (request.id) {
+                    olmMachine.markRequestAsSent(request.id, request.type, JSON.stringify(resp));
+                }
             });
-            expect(rustCrypto["outgoingRequestLoopRunning"]).toBeFalsy();
-            expect(outgoingRequestProcessor.makeOutgoingRequest).not.toHaveBeenCalled();
-            expect(olmMachine.outgoingRequests).not.toHaveBeenCalled();
+            await outgoingRequestsManager.doProcessOutgoingRequests();
+            await outgoingRequestsManager.doProcessOutgoingRequests();
 
-            // we sent three, so there should be 2 left
-            expect(outgoingRequestQueue.length).toEqual(2);
+            expect(deviceKeysAbsent).toBe(true);
         });
     });
 
@@ -460,7 +770,7 @@ describe("RustCrypto", () => {
                 decryptEvent: () =>
                     ({
                         senderCurve25519Key: "1234",
-                    } as IEventDecryptionResult),
+                    }) as IEventDecryptionResult,
             } as unknown as CryptoBackend;
             await event.attemptDecryption(mockCryptoBackend);
 
@@ -500,7 +810,7 @@ describe("RustCrypto", () => {
                 decryptEvent: () =>
                     ({
                         clearEvent: { content: { body: "1234" } },
-                    } as unknown as IEventDecryptionResult),
+                    }) as unknown as IEventDecryptionResult,
             } as unknown as CryptoBackend;
             await encryptedEvent.attemptDecryption(mockCryptoBackend);
             return encryptedEvent;
@@ -685,6 +995,7 @@ describe("RustCrypto", () => {
 
         it("should call getDevice", async () => {
             olmMachine.getDevice.mockResolvedValue({
+                free: jest.fn(),
                 isCrossSigningTrusted: jest.fn().mockReturnValue(false),
                 isLocallyTrusted: jest.fn().mockReturnValue(false),
                 isCrossSignedByOwner: jest.fn().mockReturnValue(false),
@@ -777,8 +1088,8 @@ describe("RustCrypto", () => {
             // Expect the private key to be an Uint8Array with a length of 32
             expect(recoveryKey.privateKey).toBeInstanceOf(Uint8Array);
             expect(recoveryKey.privateKey.length).toBe(32);
-            // Expect keyInfo to be empty
-            expect(Object.keys(recoveryKey.keyInfo!).length).toBe(0);
+            // Expect passphrase info to be absent
+            expect(recoveryKey.keyInfo?.passphrase).toBeUndefined();
         });
 
         it("should create a recovery key with password", async () => {
@@ -796,13 +1107,9 @@ describe("RustCrypto", () => {
     });
 
     it("should wait for a keys/query before returning devices", async () => {
-        jest.useFakeTimers();
+        // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+        jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
-        const mockHttpApi = new MatrixHttpApi(new TypedEventEmitter<HttpApiEvent, HttpApiEventHandlerMap>(), {
-            baseUrl: "http://server/",
-            prefix: "",
-            onlyData: true,
-        });
         fetchMock.post("path:/_matrix/client/v3/keys/upload", { one_time_key_counts: {} });
         fetchMock.post("path:/_matrix/client/v3/keys/query", {
             device_keys: {
@@ -812,7 +1119,7 @@ describe("RustCrypto", () => {
             },
         });
 
-        const rustCrypto = await makeTestRustCrypto(mockHttpApi, testData.TEST_USER_ID);
+        const rustCrypto = await makeTestRustCrypto(makeMatrixHttpApi(), testData.TEST_USER_ID);
 
         // an attempt to fetch the device list should block
         const devicesPromise = rustCrypto.getUserDeviceInfo([testData.TEST_USER_ID]);
@@ -826,6 +1133,33 @@ describe("RustCrypto", () => {
         rustCrypto.stop();
     });
 
+    it("should emit events on device changes", async () => {
+        jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
+
+        fetchMock.post("path:/_matrix/client/v3/keys/upload", { one_time_key_counts: {} });
+        fetchMock.post("path:/_matrix/client/v3/keys/query", {
+            device_keys: {
+                [testData.TEST_USER_ID]: {
+                    [testData.TEST_DEVICE_ID]: testData.SIGNED_TEST_DEVICE_DATA,
+                },
+            },
+        });
+
+        const rustCrypto = await makeTestRustCrypto(makeMatrixHttpApi(), testData.TEST_USER_ID);
+        const willUpdateCallback = jest.fn();
+        rustCrypto.on(CryptoEvent.WillUpdateDevices, willUpdateCallback);
+        const devicesUpdatedCallback = jest.fn();
+        rustCrypto.on(CryptoEvent.DevicesUpdated, devicesUpdatedCallback);
+
+        rustCrypto.onSyncCompleted({});
+
+        // wait for the devices to be updated
+        await rustCrypto.getUserDeviceInfo([testData.TEST_USER_ID]);
+        expect(willUpdateCallback).toHaveBeenCalledWith([testData.TEST_USER_ID], false);
+        expect(devicesUpdatedCallback).toHaveBeenCalledWith([testData.TEST_USER_ID], false);
+        rustCrypto.stop();
+    });
+
     describe("requestDeviceVerification", () => {
         it("throws an error if the device is unknown", async () => {
             const rustCrypto = await makeTestRustCrypto();
@@ -911,7 +1245,7 @@ describe("RustCrypto", () => {
         });
 
         it("returns a verified UserVerificationStatus when the UserIdentity is verified", async () => {
-            olmMachine.getIdentity.mockResolvedValue({ isVerified: jest.fn().mockReturnValue(true) });
+            olmMachine.getIdentity.mockResolvedValue({ free: jest.fn(), isVerified: jest.fn().mockReturnValue(true) });
 
             const userVerificationStatus = await rustCrypto.getUserVerificationStatus(testData.TEST_USER_ID);
             expect(userVerificationStatus.isVerified()).toBeTruthy();
@@ -938,12 +1272,6 @@ describe("RustCrypto", () => {
             // Return the key backup
             fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
 
-            const mockHttpApi = new MatrixHttpApi(new TypedEventEmitter<HttpApiEvent, HttpApiEventHandlerMap>(), {
-                baseUrl: "http://server/",
-                prefix: "",
-                onlyData: true,
-            });
-
             const olmMachine = {
                 getIdentity: jest.fn(),
                 // Force the backup to be trusted by the olmMachine
@@ -956,7 +1284,7 @@ describe("RustCrypto", () => {
             const rustCrypto = new RustCrypto(
                 logger,
                 olmMachine,
-                mockHttpApi,
+                makeMatrixHttpApi(),
                 testData.TEST_USER_ID,
                 testData.TEST_DEVICE_ID,
                 {} as ServerSideSecretStorage,
@@ -970,9 +1298,88 @@ describe("RustCrypto", () => {
             await rustCrypto.onUserIdentityUpdated(new RustSdkCryptoJs.UserId(testData.TEST_USER_ID));
             expect(await keyBackupStatusPromise).toBe(true);
         });
+
+        it("does not back up keys that came from backup", async () => {
+            const rustCrypto = await makeTestRustCrypto();
+            const olmMachine: OlmMachine = rustCrypto["olmMachine"];
+
+            await olmMachine.enableBackupV1(
+                (testData.SIGNED_BACKUP_DATA.auth_data as Curve25519AuthData).public_key,
+                testData.SIGNED_BACKUP_DATA.version!,
+            );
+
+            // we import two keys: one "from backup", and one "from export"
+            const [backedUpRoomKey, exportedRoomKey] = testData.MEGOLM_SESSION_DATA_ARRAY;
+            await rustCrypto.importBackedUpRoomKeys([backedUpRoomKey]);
+            await rustCrypto.importRoomKeys([exportedRoomKey]);
+
+            // we ask for the keys that should be backed up
+            const roomKeysRequest = await olmMachine.backupRoomKeys();
+            expect(roomKeysRequest).toBeTruthy();
+            const roomKeys = JSON.parse(roomKeysRequest!.body);
+
+            // we expect that the key "from export" is present
+            expect(roomKeys).toMatchObject({
+                rooms: {
+                    [exportedRoomKey.room_id]: {
+                        sessions: {
+                            [exportedRoomKey.session_id]: {},
+                        },
+                    },
+                },
+            });
+
+            // we expect that the key "from backup" is not present
+            expect(roomKeys).not.toMatchObject({
+                rooms: {
+                    [backedUpRoomKey.room_id]: {
+                        sessions: {
+                            [backedUpRoomKey.session_id]: {},
+                        },
+                    },
+                },
+            });
+        });
+
+        it("ignores invalid keys when restoring from backup", async () => {
+            const rustCrypto = await makeTestRustCrypto();
+            const olmMachine: OlmMachine = rustCrypto["olmMachine"];
+
+            await olmMachine.enableBackupV1(
+                (testData.SIGNED_BACKUP_DATA.auth_data as Curve25519AuthData).public_key,
+                testData.SIGNED_BACKUP_DATA.version!,
+            );
+
+            const backup = Array.from(testData.MEGOLM_SESSION_DATA_ARRAY);
+            // in addition to correct keys, we restore an invalid key
+            backup.push({ room_id: "!roomid", session_id: "sessionid" } as IMegolmSessionData);
+            const progressCallback = jest.fn();
+            await rustCrypto.importBackedUpRoomKeys(backup, { progressCallback });
+            expect(progressCallback).toHaveBeenCalledWith({
+                total: 3,
+                successes: 0,
+                stage: "load_keys",
+                failures: 1,
+            });
+            expect(progressCallback).toHaveBeenCalledWith({
+                total: 3,
+                successes: 1,
+                stage: "load_keys",
+                failures: 1,
+            });
+        });
     });
 });
 
+/** Build a MatrixHttpApi instance */
+function makeMatrixHttpApi(): MatrixHttpApi<IHttpOpts & { onlyData: true }> {
+    return new MatrixHttpApi(new TypedEventEmitter<HttpApiEvent, HttpApiEventHandlerMap>(), {
+        baseUrl: "http://server/",
+        prefix: "",
+        onlyData: true,
+    });
+}
+
 /** build a basic RustCrypto instance for testing
  *
  * just provides default arguments for initRustCrypto()
@@ -984,5 +1391,54 @@ async function makeTestRustCrypto(
     secretStorage: ServerSideSecretStorage = {} as ServerSideSecretStorage,
     cryptoCallbacks: CryptoCallbacks = {} as CryptoCallbacks,
 ): Promise<RustCrypto> {
-    return await initRustCrypto(logger, http, userId, deviceId, secretStorage, cryptoCallbacks, null, undefined);
+    return await initRustCrypto({
+        logger,
+        http,
+        userId,
+        deviceId,
+        secretStorage,
+        cryptoCallbacks,
+        storePrefix: null,
+        storePassphrase: undefined,
+    });
+}
+
+/** emulate account data, storing in memory
+ */
+class DummyAccountDataClient
+    extends TypedEventEmitter<ClientEvent.AccountData, ClientEventHandlerMap>
+    implements AccountDataClient
+{
+    private storage: Map<string, any> = new Map();
+
+    public constructor() {
+        super();
+    }
+
+    public async getAccountDataFromServer<T extends Record<string, any>>(eventType: string): Promise<T | null> {
+        const ret = this.storage.get(eventType);
+
+        if (eventType) {
+            return ret as T;
+        } else {
+            return null;
+        }
+    }
+
+    public async setAccountData(eventType: string, content: any): Promise<{}> {
+        this.storage.set(eventType, content);
+        this.emit(
+            ClientEvent.AccountData,
+            new MatrixEvent({
+                content,
+                type: eventType,
+            }),
+        );
+        return {};
+    }
+}
+
+/** Pad a string to 43 characters long */
+function pad43(x: string): string {
+    return x + ".".repeat(43 - x.length);
 }

spec/unit/rust-crypto/verification.spec.ts

@@ -17,8 +17,19 @@ limitations under the License.
 import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
 import { Mocked } from "jest-mock";
 
-import { isVerificationEvent, RustVerificationRequest } from "../../../src/rust-crypto/verification";
-import { OutgoingRequestProcessor } from "../../../src/rust-crypto/OutgoingRequestProcessor";
+import {
+    isVerificationEvent,
+    RustVerificationRequest,
+    verificationMethodIdentifierToMethod,
+} from "../../../src/rust-crypto/verification";
+import {
+    ShowSasCallbacks,
+    VerificationRequestEvent,
+    Verifier,
+    VerifierEvent,
+} from "../../../src/crypto-api/verification";
+import { OutgoingRequest, OutgoingRequestProcessor } from "../../../src/rust-crypto/OutgoingRequestProcessor";
+import { IDeviceKeys } from "../../../src/@types/crypto";
 import { EventType, MatrixEvent, MsgType } from "../../../src";
 
 describe("VerificationRequest", () => {
@@ -62,9 +73,13 @@ describe("VerificationRequest", () => {
 
     describe("startVerification", () => {
         let request: RustVerificationRequest;
+        let machine: Mocked<RustSdkCryptoJs.OlmMachine>;
+        let inner: Mocked<RustSdkCryptoJs.VerificationRequest>;
 
         beforeEach(() => {
-            request = makeTestRequest();
+            inner = makeMockedInner();
+            machine = { getDevice: jest.fn() } as unknown as Mocked<RustSdkCryptoJs.OlmMachine>;
+            request = makeTestRequest(inner, machine);
         });
 
         it("does not permit methods other than SAS", async () => {
@@ -73,12 +88,368 @@ describe("VerificationRequest", () => {
             );
         });
 
+        it("raises an error if the other device is unknown", async () => {
+            await expect(request.startVerification("m.sas.v1")).rejects.toThrow(
+                "startVerification(): other device is unknown",
+            );
+        });
+
         it("raises an error if starting verification does not produce a verifier", async () => {
+            jest.spyOn(inner, "otherDeviceId", "get").mockReturnValue(new RustSdkCryptoJs.DeviceId("other_device"));
+            machine.getDevice.mockResolvedValue({} as RustSdkCryptoJs.Device);
             await expect(request.startVerification("m.sas.v1")).rejects.toThrow(
                 "Still no verifier after startSas() call",
             );
         });
     });
+
+    it("can verify with SAS", async () => {
+        const aliceUserId = "@alice:example.org";
+        const aliceDeviceId = "ABCDEFG";
+        const bobUserId = "@bob:example.org";
+        const bobDeviceId = "HIJKLMN";
+        const [aliceOlmMachine, aliceDeviceKeys, aliceCrossSigningKeys] = await initOlmMachineAndKeys(
+            aliceUserId,
+            aliceDeviceId,
+        );
+        const [bobOlmMachine, bobDeviceKeys, bobCrossSigningKeys] = await initOlmMachineAndKeys(bobUserId, bobDeviceId);
+
+        const aliceRequestLoop = makeRequestLoop(
+            aliceOlmMachine,
+            aliceDeviceKeys,
+            aliceCrossSigningKeys,
+            bobOlmMachine,
+            bobDeviceKeys,
+            bobCrossSigningKeys,
+        );
+        const bobRequestLoop = makeRequestLoop(
+            bobOlmMachine,
+            bobDeviceKeys,
+            bobCrossSigningKeys,
+            aliceOlmMachine,
+            aliceDeviceKeys,
+            aliceCrossSigningKeys,
+        );
+
+        try {
+            await aliceOlmMachine.updateTrackedUsers([new RustSdkCryptoJs.UserId(bobUserId)]);
+            await bobOlmMachine.updateTrackedUsers([new RustSdkCryptoJs.UserId(aliceUserId)]);
+
+            // Alice requests verification
+            const bobUserIdentity = await aliceOlmMachine.getIdentity(new RustSdkCryptoJs.UserId(bobUserId));
+
+            const roomId = new RustSdkCryptoJs.RoomId("!roomId:example.org");
+            const methods = [verificationMethodIdentifierToMethod("m.sas.v1")];
+            const innerVerificationRequest = await bobUserIdentity.requestVerification(
+                roomId,
+                new RustSdkCryptoJs.EventId("$m.key.verification.request"),
+                methods,
+            );
+            const aliceVerificationRequest = new RustVerificationRequest(
+                aliceOlmMachine,
+                innerVerificationRequest,
+                aliceRequestLoop as unknown as OutgoingRequestProcessor,
+                ["m.sas.v1"],
+            );
+
+            const verificationRequestContent = JSON.parse(await bobUserIdentity.verificationRequestContent(methods));
+            await bobOlmMachine.receiveVerificationEvent(
+                JSON.stringify({
+                    type: "m.room.message",
+                    sender: aliceUserId,
+                    event_id: "$m.key.verification.request",
+                    content: verificationRequestContent,
+                    origin_server_ts: Date.now(),
+                    unsigned: {
+                        age: 0,
+                    },
+                }),
+                roomId,
+            );
+
+            // Bob accepts
+            const bobInnerVerificationRequest = bobOlmMachine.getVerificationRequest(
+                new RustSdkCryptoJs.UserId(aliceUserId),
+                "$m.key.verification.request",
+            )!;
+            const bobVerificationRequest = new RustVerificationRequest(
+                bobOlmMachine,
+                bobInnerVerificationRequest,
+                bobRequestLoop as unknown as OutgoingRequestProcessor,
+                ["m.sas.v1"],
+            );
+
+            await bobVerificationRequest.accept();
+
+            // Alice starts the verification
+            const bobVerifierPromise: Promise<Verifier> = new Promise((resolve, reject) => {
+                bobVerificationRequest.on(VerificationRequestEvent.Change, () => {
+                    const verifier = bobVerificationRequest.verifier;
+                    if (verifier) {
+                        resolve(verifier);
+                    }
+                });
+            });
+            const aliceVerifier = await aliceVerificationRequest.startVerification("m.sas.v1");
+            const bobVerifier = await bobVerifierPromise;
+
+            // create a function to compare the SAS, and then let the verification run
+            let otherCallbacks: ShowSasCallbacks | undefined;
+            const compareSas = (callbacks: ShowSasCallbacks): void => {
+                if (otherCallbacks) {
+                    const ourDecimal = callbacks.sas.decimal!;
+                    const theirDecimal = otherCallbacks.sas.decimal!;
+                    if (ourDecimal.every((el, idx) => el == theirDecimal[idx])) {
+                        otherCallbacks.confirm();
+                        callbacks.confirm();
+                    } else {
+                        otherCallbacks.mismatch();
+                        callbacks.mismatch();
+                    }
+                } else {
+                    otherCallbacks = callbacks;
+                }
+            };
+            aliceVerifier.on(VerifierEvent.ShowSas, compareSas);
+            bobVerifier.on(VerifierEvent.ShowSas, compareSas);
+
+            await Promise.all([aliceVerifier.verify(), await bobVerifier.verify()]);
+        } finally {
+            await aliceRequestLoop.stop();
+            await bobRequestLoop.stop();
+        }
+    });
+
+    it("can handle simultaneous starts in SAS", async () => {
+        const aliceUserId = "@alice:example.org";
+        const aliceDeviceId = "ABCDEFG";
+        const bobUserId = "@bob:example.org";
+        const bobDeviceId = "HIJKLMN";
+        const [aliceOlmMachine, aliceDeviceKeys, aliceCrossSigningKeys] = await initOlmMachineAndKeys(
+            aliceUserId,
+            aliceDeviceId,
+        );
+        const [bobOlmMachine, bobDeviceKeys, bobCrossSigningKeys] = await initOlmMachineAndKeys(bobUserId, bobDeviceId);
+
+        let aliceStartRequest: RustSdkCryptoJs.RoomMessageRequest | undefined;
+        const aliceRequestLoop = makeRequestLoop(
+            aliceOlmMachine,
+            aliceDeviceKeys,
+            aliceCrossSigningKeys,
+            bobOlmMachine,
+            bobDeviceKeys,
+            bobCrossSigningKeys,
+            async (request): Promise<any> => {
+                // If the request is sending the m.key.verification.start
+                // event, we delay sending it until after Bob has also started
+                // a verification
+                if (
+                    !aliceStartRequest &&
+                    request instanceof RustSdkCryptoJs.RoomMessageRequest &&
+                    request.event_type == "m.key.verification.start"
+                ) {
+                    aliceStartRequest = request;
+                    return { event_id: "$m.key.verification.start" };
+                }
+            },
+        );
+        const bobRequestLoop = makeRequestLoop(
+            bobOlmMachine,
+            bobDeviceKeys,
+            bobCrossSigningKeys,
+            aliceOlmMachine,
+            aliceDeviceKeys,
+            aliceCrossSigningKeys,
+        );
+
+        try {
+            await aliceOlmMachine.updateTrackedUsers([new RustSdkCryptoJs.UserId(bobUserId)]);
+            await bobOlmMachine.updateTrackedUsers([new RustSdkCryptoJs.UserId(aliceUserId)]);
+
+            // Alice requests verification
+            const bobUserIdentity = await aliceOlmMachine.getIdentity(new RustSdkCryptoJs.UserId(bobUserId));
+
+            const roomId = new RustSdkCryptoJs.RoomId("!roomId:example.org");
+            const methods = [verificationMethodIdentifierToMethod("m.sas.v1")];
+            const innerVerificationRequest = await bobUserIdentity.requestVerification(
+                roomId,
+                new RustSdkCryptoJs.EventId("$m.key.verification.request"),
+                methods,
+            );
+            const aliceVerificationRequest = new RustVerificationRequest(
+                aliceOlmMachine,
+                innerVerificationRequest,
+                aliceRequestLoop as unknown as OutgoingRequestProcessor,
+                ["m.sas.v1"],
+            );
+
+            const verificationRequestContent = JSON.parse(await bobUserIdentity.verificationRequestContent(methods));
+            await bobOlmMachine.receiveVerificationEvent(
+                JSON.stringify({
+                    type: "m.room.message",
+                    sender: aliceUserId,
+                    event_id: "$m.key.verification.request",
+                    content: verificationRequestContent,
+                    origin_server_ts: Date.now(),
+                    unsigned: {
+                        age: 0,
+                    },
+                }),
+                roomId,
+            );
+
+            // Bob accepts
+            const bobInnerVerificationRequest = bobOlmMachine.getVerificationRequest(
+                new RustSdkCryptoJs.UserId(aliceUserId),
+                "$m.key.verification.request",
+            )!;
+            const bobVerificationRequest = new RustVerificationRequest(
+                bobOlmMachine,
+                bobInnerVerificationRequest,
+                bobRequestLoop as unknown as OutgoingRequestProcessor,
+                ["m.sas.v1"],
+            );
+
+            await bobVerificationRequest.accept();
+
+            // Alice and Bob both start the verification
+            const aliceVerifier = await aliceVerificationRequest.startVerification("m.sas.v1");
+            const bobVerifier = await bobVerificationRequest.startVerification("m.sas.v1");
+            // We can now send Alice's start message to Bob
+            await aliceRequestLoop.makeOutgoingRequest(aliceStartRequest!);
+
+            // create a function to compare the SAS, and then let the verification run
+            let otherCallbacks: ShowSasCallbacks | undefined;
+            const compareSas = (callbacks: ShowSasCallbacks) => {
+                if (otherCallbacks) {
+                    const ourDecimal = callbacks.sas.decimal!;
+                    const theirDecimal = otherCallbacks.sas.decimal!;
+                    if (ourDecimal.every((el, idx) => el == theirDecimal[idx])) {
+                        otherCallbacks.confirm();
+                        callbacks.confirm();
+                    } else {
+                        otherCallbacks.mismatch();
+                        callbacks.mismatch();
+                    }
+                } else {
+                    otherCallbacks = callbacks;
+                }
+            };
+            aliceVerifier.on(VerifierEvent.ShowSas, compareSas);
+            bobVerifier.on(VerifierEvent.ShowSas, compareSas);
+
+            await Promise.all([aliceVerifier.verify(), await bobVerifier.verify()]);
+        } finally {
+            await aliceRequestLoop.stop();
+            await bobRequestLoop.stop();
+        }
+    });
+
+    it("can verify by QR code", async () => {
+        const aliceUserId = "@alice:example.org";
+        const aliceDeviceId = "ABCDEFG";
+        const bobUserId = "@bob:example.org";
+        const bobDeviceId = "HIJKLMN";
+        const [aliceOlmMachine, aliceDeviceKeys, aliceCrossSigningKeys] = await initOlmMachineAndKeys(
+            aliceUserId,
+            aliceDeviceId,
+        );
+        const [bobOlmMachine, bobDeviceKeys, bobCrossSigningKeys] = await initOlmMachineAndKeys(bobUserId, bobDeviceId);
+
+        const aliceRequestLoop = makeRequestLoop(
+            aliceOlmMachine,
+            aliceDeviceKeys,
+            aliceCrossSigningKeys,
+            bobOlmMachine,
+            bobDeviceKeys,
+            bobCrossSigningKeys,
+        );
+        const bobRequestLoop = makeRequestLoop(
+            bobOlmMachine,
+            bobDeviceKeys,
+            bobCrossSigningKeys,
+            aliceOlmMachine,
+            aliceDeviceKeys,
+            aliceCrossSigningKeys,
+        );
+
+        try {
+            await aliceOlmMachine.updateTrackedUsers([new RustSdkCryptoJs.UserId(bobUserId)]);
+            await bobOlmMachine.updateTrackedUsers([new RustSdkCryptoJs.UserId(aliceUserId)]);
+
+            // Alice requests verification
+            const bobUserIdentity = await aliceOlmMachine.getIdentity(new RustSdkCryptoJs.UserId(bobUserId));
+
+            const roomId = new RustSdkCryptoJs.RoomId("!roomId:example.org");
+            const methods = [
+                verificationMethodIdentifierToMethod("m.reciprocate.v1"),
+                verificationMethodIdentifierToMethod("m.qr_code.show.v1"),
+            ];
+            const innerVerificationRequest = await bobUserIdentity.requestVerification(
+                roomId,
+                new RustSdkCryptoJs.EventId("$m.key.verification.request"),
+                methods,
+            );
+            const aliceVerificationRequest = new RustVerificationRequest(
+                aliceOlmMachine,
+                innerVerificationRequest,
+                aliceRequestLoop as unknown as OutgoingRequestProcessor,
+                ["m.reciprocate.v1", "m.qr_code.show.v1"],
+            );
+
+            const verificationRequestContent = JSON.parse(await bobUserIdentity.verificationRequestContent(methods));
+            await bobOlmMachine.receiveVerificationEvent(
+                JSON.stringify({
+                    type: "m.room.message",
+                    sender: aliceUserId,
+                    event_id: "$m.key.verification.request",
+                    content: verificationRequestContent,
+                    origin_server_ts: Date.now(),
+                    unsigned: {
+                        age: 0,
+                    },
+                }),
+                roomId,
+            );
+
+            // Bob accepts
+            const bobInnerVerificationRequest = bobOlmMachine.getVerificationRequest(
+                new RustSdkCryptoJs.UserId(aliceUserId),
+                "$m.key.verification.request",
+            )!;
+            const bobVerificationRequest = new RustVerificationRequest(
+                bobOlmMachine,
+                bobInnerVerificationRequest,
+                bobRequestLoop as unknown as OutgoingRequestProcessor,
+                ["m.reciprocate.v1", "m.qr_code.show.v1", "m.qr_code.scan.v1"],
+            );
+
+            await bobVerificationRequest.accept();
+
+            // Bob scans
+            const qrCode = await aliceVerificationRequest.generateQRCode();
+
+            const aliceVerifierPromise: Promise<Verifier> = new Promise((resolve, reject) => {
+                aliceVerificationRequest.on(VerificationRequestEvent.Change, () => {
+                    const verifier = aliceVerificationRequest.verifier;
+                    if (verifier) {
+                        resolve(verifier);
+                    }
+                });
+            });
+            const bobVerifier = await bobVerificationRequest.scanQRCode(qrCode!);
+
+            const aliceVerifier = await aliceVerifierPromise;
+            aliceVerifier.on(VerifierEvent.ShowReciprocateQr, (showQrCodeCallbacks) => {
+                showQrCodeCallbacks.confirm();
+            });
+
+            await Promise.all([aliceVerifier.verify(), await bobVerifier.verify()]);
+        } finally {
+            await aliceRequestLoop.stop();
+            await bobRequestLoop.stop();
+        }
+    });
 });
 
 describe("isVerificationEvent", () => {
@@ -118,11 +489,13 @@ describe("isVerificationEvent", () => {
 /** build a RustVerificationRequest with default parameters */
 function makeTestRequest(
     inner?: RustSdkCryptoJs.VerificationRequest,
+    olmMachine?: RustSdkCryptoJs.OlmMachine,
     outgoingRequestProcessor?: OutgoingRequestProcessor,
 ): RustVerificationRequest {
     inner ??= makeMockedInner();
+    olmMachine ??= {} as RustSdkCryptoJs.OlmMachine;
     outgoingRequestProcessor ??= {} as OutgoingRequestProcessor;
-    return new RustVerificationRequest(inner, outgoingRequestProcessor, []);
+    return new RustVerificationRequest(olmMachine, inner, outgoingRequestProcessor, []);
 }
 
 /** Mock up a rust-side VerificationRequest */
@@ -133,5 +506,153 @@ function makeMockedInner(): Mocked<RustSdkCryptoJs.VerificationRequest> {
         phase: jest.fn().mockReturnValue(RustSdkCryptoJs.VerificationRequestPhase.Created),
         isPassive: jest.fn().mockReturnValue(false),
         timeRemainingMillis: jest.fn(),
+        get otherDeviceId() {
+            return undefined;
+        },
     } as unknown as Mocked<RustSdkCryptoJs.VerificationRequest>;
 }
+
+interface CrossSigningKeys {
+    master_key: any;
+    self_signing_key: any;
+    user_signing_key: any;
+}
+
+/** create an Olm machine and device/cross-signing keys for a user */
+async function initOlmMachineAndKeys(
+    userId: string,
+    deviceId: string,
+): Promise<[RustSdkCryptoJs.OlmMachine, IDeviceKeys, CrossSigningKeys]> {
+    const olmMachine = await RustSdkCryptoJs.OlmMachine.initialize(
+        new RustSdkCryptoJs.UserId(userId),
+        new RustSdkCryptoJs.DeviceId(deviceId),
+        undefined,
+        undefined,
+    );
+    const { uploadKeysRequest, uploadSignaturesRequest, uploadSigningKeysRequest } =
+        await olmMachine.bootstrapCrossSigning(true);
+    const deviceKeys = JSON.parse(uploadKeysRequest.body).device_keys;
+    await olmMachine.markRequestAsSent(
+        uploadKeysRequest.id,
+        uploadKeysRequest.type,
+        '{"one_time_key_counts":{"signed_curve25519":100}}',
+    );
+    const crossSigningSignatures = JSON.parse(uploadSignaturesRequest.body);
+    for (const [keyId, signature] of Object.entries(crossSigningSignatures[userId][deviceId]["signatures"][userId])) {
+        deviceKeys["signatures"][userId][keyId] = signature;
+    }
+    const crossSigningKeys = JSON.parse(uploadSigningKeysRequest.body);
+    // note: the upload signatures request and upload signing keys requests
+    // don't need to be marked as sent in the Olm machine
+
+    return [olmMachine, deviceKeys, crossSigningKeys];
+}
+
+type CustomRequestHandler = (request: OutgoingRequest | RustSdkCryptoJs.UploadSigningKeysRequest) => Promise<any>;
+
+/** Loop for handling outgoing requests from an Olm machine.
+ *
+ * Simulates a server with two users: "us" and "them".  Handles key query
+ * requests, querying either our keys or the other user's keys.  Room messages
+ * are sent as incoming verification events to the other user.  A custom
+ * handler can be added to override default request processing (the handler
+ * should return a response body to inhibit default processing).
+ *
+ * Can also be used as an OutgoingRequestProcessor. */
+function makeRequestLoop(
+    ourOlmMachine: RustSdkCryptoJs.OlmMachine,
+    ourDeviceKeys: IDeviceKeys,
+    ourCrossSigningKeys: CrossSigningKeys,
+    theirOlmMachine: RustSdkCryptoJs.OlmMachine,
+    theirDeviceKeys: IDeviceKeys,
+    theirCrossSigningKeys: CrossSigningKeys,
+    customHandler?: CustomRequestHandler,
+) {
+    let stopRequestLoop = false;
+    const ourUserId = ourOlmMachine.userId.toString();
+    const ourDeviceId = ourOlmMachine.deviceId.toString();
+    const theirUserId = theirOlmMachine.userId.toString();
+    const theirDeviceId = theirOlmMachine.deviceId.toString();
+
+    function defaultHandler(request: OutgoingRequest | RustSdkCryptoJs.UploadSigningKeysRequest): any {
+        if (request instanceof RustSdkCryptoJs.KeysQueryRequest) {
+            const resp: Record<string, any> = {
+                device_keys: {},
+            };
+            const body = JSON.parse(request.body);
+            const query = body.device_keys;
+            const masterKeys: Record<string, any> = {};
+            const selfSigningKeys: Record<string, any> = {};
+            if (ourUserId in query) {
+                resp.device_keys[ourUserId] = { [ourDeviceId]: ourDeviceKeys };
+                masterKeys[ourUserId] = ourCrossSigningKeys.master_key;
+                selfSigningKeys[ourUserId] = ourCrossSigningKeys.self_signing_key;
+                resp.user_signing_keys = {
+                    [ourUserId]: ourCrossSigningKeys.user_signing_key,
+                };
+            }
+            if (theirUserId in query) {
+                resp.device_keys[theirUserId] = {
+                    [theirDeviceId]: theirDeviceKeys,
+                };
+                masterKeys[theirUserId] = theirCrossSigningKeys.master_key;
+                selfSigningKeys[theirUserId] = theirCrossSigningKeys.self_signing_key;
+            }
+            if (Object.keys(masterKeys).length) {
+                resp.master_keys = masterKeys;
+            }
+            if (Object.keys(selfSigningKeys).length) {
+                resp.self_signing_keys = selfSigningKeys;
+            }
+            return resp;
+        } else if (request instanceof RustSdkCryptoJs.RoomMessageRequest) {
+            theirOlmMachine.receiveVerificationEvent(
+                JSON.stringify({
+                    type: request.event_type,
+                    sender: ourUserId,
+                    event_id: "$" + request.event_type,
+                    content: JSON.parse(request.body),
+                    origin_server_ts: Date.now(),
+                    unsigned: {
+                        age: 0,
+                    },
+                }),
+                new RustSdkCryptoJs.RoomId(request.room_id),
+            );
+            return { event_id: "$" + request.event_type };
+        } else if (request instanceof RustSdkCryptoJs.SignatureUploadRequest) {
+            // this only gets called at the end after the verification
+            // succeeds, so we don't actually have to do anything.
+            return { failures: {} };
+        }
+        return {};
+    }
+
+    async function makeOutgoingRequest(
+        request: OutgoingRequest | RustSdkCryptoJs.UploadSigningKeysRequest,
+    ): Promise<any> {
+        const resp = (await customHandler?.(request)) ?? defaultHandler(request);
+        if (!(request instanceof RustSdkCryptoJs.UploadSigningKeysRequest) && request.id) {
+            await ourOlmMachine.markRequestAsSent(request.id!, request.type, JSON.stringify(resp));
+        }
+    }
+
+    async function runLoop() {
+        while (!stopRequestLoop) {
+            const requests = await ourOlmMachine.outgoingRequests();
+            for (const request of requests) {
+                await makeOutgoingRequest(request);
+            }
+        }
+    }
+
+    const loopCompletedPromise = runLoop();
+
+    return {
+        makeOutgoingRequest,
+        stop: async () => {
+            stopRequestLoop = true;
+            await loopCompletedPromise;
+        },
+    };
+}

spec/unit/secret-storage.spec.ts

@@ -33,7 +33,9 @@ describe("ServerSideSecretStorageImpl", function () {
         it("should allow storing a default key", async function () {
             const accountDataAdapter = mockAccountDataClient();
             const secretStorage = new ServerSideSecretStorageImpl(accountDataAdapter, {});
-            const result = await secretStorage.addKey("m.secret_storage.v1.aes-hmac-sha2");
+            const result = await secretStorage.addKey("m.secret_storage.v1.aes-hmac-sha2", {
+                key: new Uint8Array(32),
+            });
 
             // it should have made up a 32-character key id
             expect(result.keyId.length).toEqual(32);
@@ -46,7 +48,13 @@ describe("ServerSideSecretStorageImpl", function () {
         it("should allow storing a key with an explicit id", async function () {
             const accountDataAdapter = mockAccountDataClient();
             const secretStorage = new ServerSideSecretStorageImpl(accountDataAdapter, {});
-            const result = await secretStorage.addKey("m.secret_storage.v1.aes-hmac-sha2", {}, "myKeyId");
+            const result = await secretStorage.addKey(
+                "m.secret_storage.v1.aes-hmac-sha2",
+                {
+                    key: new Uint8Array(32),
+                },
+                "myKeyId",
+            );
 
             // it should have made up a 32-character key id
             expect(result.keyId).toEqual("myKeyId");
@@ -59,7 +67,10 @@ describe("ServerSideSecretStorageImpl", function () {
         it("should allow storing a key with a name", async function () {
             const accountDataAdapter = mockAccountDataClient();
             const secretStorage = new ServerSideSecretStorageImpl(accountDataAdapter, {});
-            const result = await secretStorage.addKey("m.secret_storage.v1.aes-hmac-sha2", { name: "mykey" });
+            const result = await secretStorage.addKey("m.secret_storage.v1.aes-hmac-sha2", {
+                name: "mykey",
+                key: new Uint8Array(32),
+            });
 
             expect(result.keyInfo.name).toEqual("mykey");
 
@@ -80,6 +91,7 @@ describe("ServerSideSecretStorageImpl", function () {
             };
             const result = await secretStorage.addKey("m.secret_storage.v1.aes-hmac-sha2", {
                 passphrase,
+                key: new Uint8Array(32),
             });
 
             expect(result.keyInfo.passphrase).toEqual(passphrase);
@@ -93,7 +105,9 @@ describe("ServerSideSecretStorageImpl", function () {
         it("should complain about invalid algorithm", async function () {
             const accountDataAdapter = mockAccountDataClient();
             const secretStorage = new ServerSideSecretStorageImpl(accountDataAdapter, {});
-            await expect(() => secretStorage.addKey("bad_alg")).rejects.toThrow("Unknown key algorithm");
+            await expect(() => secretStorage.addKey("bad_alg", { key: new Uint8Array(32) })).rejects.toThrow(
+                "Unknown key algorithm",
+            );
         });
     });
 

spec/unit/thread-utils.spec.ts

@@ -0,0 +1,51 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { IEvent } from "../../src";
+import { randomString } from "../../src/randomstring";
+import { getRelationsThreadFilter } from "../../src/thread-utils";
+
+function makeEvent(relatesToEvent: string, relType: string): Partial<IEvent> {
+    return {
+        event_id: randomString(10),
+        type: "m.room.message",
+        content: {
+            "msgtype": "m.text",
+            "body": "foo",
+            "m.relates_to": {
+                rel_type: relType,
+                event_id: relatesToEvent,
+            },
+        },
+    };
+}
+
+describe("getRelationsThreadFilter", () => {
+    it("should filter out relations directly to the thread root event", () => {
+        const threadId = "thisIsMyThreadRoot";
+
+        const reactionToRoot = makeEvent(threadId, "m.annotation");
+        const editToRoot = makeEvent(threadId, "m.replace");
+        const firstThreadedReply = makeEvent(threadId, "m.thread");
+        const reactionToThreadedEvent = makeEvent(firstThreadedReply.event_id!, "m.annotation");
+
+        const filteredEvents = [reactionToRoot, editToRoot, firstThreadedReply, reactionToThreadedEvent].filter(
+            getRelationsThreadFilter(threadId),
+        );
+
+        expect(filteredEvents).toEqual([firstThreadedReply, reactionToThreadedEvent]);
+    });
+});

spec/unit/timeline-window.spec.ts

@@ -22,12 +22,15 @@ import { Room } from "../../src/models/room";
 import { EventTimeline } from "../../src/models/event-timeline";
 import { TimelineIndex, TimelineWindow } from "../../src/timeline-window";
 import { mkMessage } from "../test-utils/test-utils";
+import { MatrixEvent } from "../../src/models/event";
 
 const ROOM_ID = "roomId";
 const USER_ID = "userId";
 const mockClient = {
     getEventTimeline: jest.fn(),
     paginateEventTimeline: jest.fn(),
+    supportsThreads: jest.fn(),
+    getUserId: jest.fn().mockReturnValue(USER_ID),
 } as unknown as MockedObject<MatrixClient>;
 
 /*
@@ -64,6 +67,23 @@ function addEventsToTimeline(timeline: EventTimeline, numEvents: number, toStart
     }
 }
 
+function createEvents(numEvents: number): Array<MatrixEvent> {
+    const ret = [];
+
+    for (let i = 0; i < numEvents; i++) {
+        ret.push(
+            mkMessage({
+                room: ROOM_ID,
+                user: USER_ID,
+                event: true,
+                unsigned: { age: 1 },
+            }),
+        );
+    }
+
+    return ret;
+}
+
 /*
  * create a pair of linked timelines
  */
@@ -412,4 +432,46 @@ describe("TimelineWindow", function () {
             expect(timelineWindow.canPaginate(EventTimeline.FORWARDS)).toBe(true);
         });
     });
+
+    function idsOf(events: Array<MatrixEvent>): Array<string> {
+        return events.map((e) => (e ? e.getId() ?? "MISSING_ID" : "MISSING_EVENT"));
+    }
+
+    describe("removing events", () => {
+        it("should shorten if removing an event within the window makes it overflow", function () {
+            // Given a room with events in two timelines
+            const room = new Room(ROOM_ID, mockClient, USER_ID, { timelineSupport: true });
+            const timelineSet = room.getUnfilteredTimelineSet();
+            const liveTimeline = room.getLiveTimeline();
+            const oldTimeline = room.addTimeline();
+            liveTimeline.setNeighbouringTimeline(oldTimeline, EventTimeline.BACKWARDS);
+            oldTimeline.setNeighbouringTimeline(liveTimeline, EventTimeline.FORWARDS);
+
+            const oldEvents = createEvents(5);
+            const liveEvents = createEvents(5);
+            const [, , e3, e4, e5] = oldEvents;
+            const [, e7, e8, e9, e10] = liveEvents;
+            room.addLiveEvents(liveEvents);
+            room.addEventsToTimeline(oldEvents, true, oldTimeline);
+
+            // And 2 windows over the timelines in this room
+            const oldWindow = new TimelineWindow(mockClient, timelineSet);
+            oldWindow.load(e5.getId(), 6);
+            expect(idsOf(oldWindow.getEvents())).toEqual(idsOf([e5, e4, e3]));
+
+            const newWindow = new TimelineWindow(mockClient, timelineSet);
+            newWindow.load(e9.getId(), 4);
+            expect(idsOf(newWindow.getEvents())).toEqual(idsOf([e7, e8, e9, e10]));
+
+            // When I remove an event
+            room.removeEvent(e8.getId()!);
+
+            // Then the affected timeline is shortened (because it would have
+            // been too long with the removed event gone)
+            expect(idsOf(newWindow.getEvents())).toEqual(idsOf([e7, e9, e10]));
+
+            // And the unaffected one is not
+            expect(idsOf(oldWindow.getEvents())).toEqual(idsOf([e5, e4, e3]));
+        });
+    });
 });

spec/unit/webrtc/groupCall.spec.ts

@@ -954,7 +954,7 @@ describe("Group Call", function () {
                             },
                         },
                     }),
-                } as MatrixEvent);
+                }) as MatrixEvent;
 
             it("should mute remote feed's audio after receiving metadata with video audio", async () => {
                 const metadataEvent = getMetadataEvent(true, false);
@@ -965,7 +965,7 @@ describe("Group Call", function () {
 
                 // @ts-ignore
                 const call = groupCall.calls.get(FAKE_USER_ID_2)!.get(FAKE_DEVICE_ID_2)!;
-                call.getOpponentMember = () => ({ userId: call.invitee } as RoomMember);
+                call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember;
                 // @ts-ignore Mock
                 call.pushRemoteFeed(
                     // @ts-ignore Mock
@@ -992,7 +992,7 @@ describe("Group Call", function () {
 
                 // @ts-ignore
                 const call = groupCall.calls.get(FAKE_USER_ID_2).get(FAKE_DEVICE_ID_2)!;
-                call.getOpponentMember = () => ({ userId: call.invitee } as RoomMember);
+                call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember;
                 // @ts-ignore Mock
                 call.pushRemoteFeed(
                     // @ts-ignore Mock
@@ -1310,7 +1310,7 @@ describe("Group Call", function () {
 
             // @ts-ignore
             const call = groupCall.calls.get(FAKE_USER_ID_2)!.get(FAKE_DEVICE_ID_2)!;
-            call.getOpponentMember = () => ({ userId: call.invitee } as RoomMember);
+            call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember;
             call.onNegotiateReceived({
                 getContent: () => ({
                     [SDPStreamMetadataKey]: {

src/@types/auth.ts

@@ -250,12 +250,6 @@ export interface LoginTokenPostResponse {
      * The token to use with `m.login.token` to authenticate.
      */
     login_token: string;
-    /**
-     * Expiration in seconds.
-     *
-     * @deprecated this is only provided for compatibility with original revision of [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882).
-     */
-    expires_in: number;
     /**
      * Expiration in milliseconds.
      */

src/@types/event.ts

@@ -55,6 +55,7 @@ export enum EventType {
     CallReplaces = "m.call.replaces",
     CallAssertedIdentity = "m.call.asserted_identity",
     CallAssertedIdentityPrefix = "org.matrix.call.asserted_identity",
+    CallEncryptionKeysPrefix = "io.element.call.encryption_keys",
     KeyVerificationRequest = "m.key.verification.request",
     KeyVerificationStart = "m.key.verification.start",
     KeyVerificationCancel = "m.key.verification.cancel",
@@ -93,6 +94,9 @@ export enum EventType {
     // Group call events
     GroupCallPrefix = "org.matrix.msc3401.call",
     GroupCallMemberPrefix = "org.matrix.msc3401.call.member",
+
+    // MatrixRTC events
+    CallNotify = "org.matrix.msc4075.call.notify",
 }
 
 export enum RelationType {

src/@types/extensible_events.ts

@@ -23,9 +23,8 @@ import { isProvided } from "../extensible_events_v1/utilities";
 /**
  * Represents the stable and unstable values of a given namespace.
  */
-export type TSNamespace<N> = N extends NamespacedValue<infer S, infer U>
-    ? TSNamespaceValue<S> | TSNamespaceValue<U>
-    : never;
+export type TSNamespace<N> =
+    N extends NamespacedValue<infer S, infer U> ? TSNamespaceValue<S> | TSNamespaceValue<U> : never;
 
 /**
  * Represents a namespaced value, if the value is a string. Used to extract provided types

src/NamespacedValue.ts

@@ -26,7 +26,10 @@ export class NamespacedValue<S extends string, U extends string> {
     public constructor(stable: S, unstable: U);
     public constructor(stable: S, unstable?: U);
     public constructor(stable: null | undefined, unstable: U);
-    public constructor(public readonly stable?: S | null, public readonly unstable?: U) {
+    public constructor(
+        public readonly stable?: S | null,
+        public readonly unstable?: U,
+    ) {
         if (!this.unstable && !this.stable) {
             throw new Error("One of stable or unstable values must be supplied");
         }

src/autodiscovery.ts

@@ -28,7 +28,7 @@ import {
     validateWellKnownAuthentication,
 } from "./oidc/validate";
 import { OidcError } from "./oidc/error";
-import { MINIMUM_MATRIX_VERSION } from "./version-support";
+import { SUPPORTED_MATRIX_VERSIONS } from "./version-support";
 
 // Dev note: Auto discovery is part of the spec.
 // See: https://matrix.org/docs/spec/client_server/r0.4.0.html#server-discovery
@@ -51,7 +51,10 @@ export enum AutoDiscoveryError {
     InvalidIs = "Invalid identity server discovery response",
     MissingWellknown = "No .well-known JSON file found",
     InvalidJson = "Invalid JSON",
-    HomeserverTooOld = "The homeserver does not meet the minimum version requirements",
+    UnsupportedHomeserverSpecVersion = "The homeserver does not meet the version requirements",
+
+    /** @deprecated Replaced by `UnsupportedHomeserverSpecVersion` */
+    HomeserverTooOld = UnsupportedHomeserverSpecVersion,
     // TODO: Implement when Sydent supports the `/versions` endpoint - https://github.com/matrix-org/sydent/issues/424
     //IdentityServerTooOld = "The identity server does not meet the minimum version requirements",
 }
@@ -112,7 +115,11 @@ export class AutoDiscovery {
 
     public static readonly ERROR_INVALID_JSON = AutoDiscoveryError.InvalidJson;
 
-    public static readonly ERROR_HOMESERVER_TOO_OLD = AutoDiscoveryError.HomeserverTooOld;
+    public static readonly ERROR_UNSUPPORTED_HOMESERVER_SPEC_VERSION =
+        AutoDiscoveryError.UnsupportedHomeserverSpecVersion;
+
+    /** @deprecated Replaced by ERROR_UNSUPPORTED_HOMESERVER_SPEC_VERSION */
+    public static readonly ERROR_HOMESERVER_TOO_OLD = AutoDiscovery.ERROR_UNSUPPORTED_HOMESERVER_SPEC_VERSION;
 
     public static readonly ALL_ERRORS = Object.keys(AutoDiscoveryError) as AutoDiscoveryError[];
 
@@ -216,10 +223,19 @@ export class AutoDiscovery {
             return Promise.resolve(clientConfig);
         }
 
-        // Step 3.1: Non-spec check to ensure the server will actually work for us
-        if (!hsVersions.raw!["versions"].includes(MINIMUM_MATRIX_VERSION)) {
-            logger.error("Homeserver does not meet minimum version requirements");
-            clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_HOMESERVER_TOO_OLD;
+        // Step 3.1: Non-spec check to ensure the server will actually work for us. We need to check if
+        // any of the versions in `SUPPORTED_MATRIX_VERSIONS` are listed in the /versions response.
+        const hsVersionSet = new Set(hsVersions.raw!["versions"]);
+        let supportedVersionFound = false;
+        for (const version of SUPPORTED_MATRIX_VERSIONS) {
+            if (hsVersionSet.has(version)) {
+                supportedVersionFound = true;
+                break;
+            }
+        }
+        if (!supportedVersionFound) {
+            logger.error("Homeserver does not meet version requirements");
+            clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_UNSUPPORTED_HOMESERVER_SPEC_VERSION;
 
             // Supply the base_url to the caller because they may be ignoring liveliness
             // errors, like this one.

src/base64.ts

@@ -54,7 +54,16 @@ export function encodeUnpaddedBase64(uint8Array: ArrayBuffer | Uint8Array): stri
 }
 
 /**
- * Decode a base64 string to a typed array of uint8.
+ * Encode a typed array of uint8 as unpadded base64 using the URL-safe encoding.
+ * @param uint8Array - The data to encode.
+ * @returns The unpadded base64.
+ */
+export function encodeUnpaddedBase64Url(uint8Array: ArrayBuffer | Uint8Array): string {
+    return encodeUnpaddedBase64(uint8Array).replace("+", "-").replace("/", "_");
+}
+
+/**
+ * Decode a base64 (or base64url) string to a typed array of uint8.
  * @param base64 - The base64 to decode.
  * @returns The decoded data.
  */

src/client.ts

@@ -51,7 +51,7 @@ import { decodeBase64, encodeBase64 } from "./base64";
 import { IExportedDevice as IExportedOlmDevice } from "./crypto/OlmDevice";
 import { IOlmDevice } from "./crypto/algorithms/megolm";
 import { TypedReEmitter } from "./ReEmitter";
-import { IRoomEncryption, RoomList } from "./crypto/RoomList";
+import { IRoomEncryption } from "./crypto/RoomList";
 import { logger, Logger } from "./logger";
 import { SERVICE_TYPES } from "./service-types";
 import {
@@ -209,7 +209,7 @@ import { IgnoredInvites } from "./models/invites-ignorer";
 import { UIARequest, UIAResponse } from "./@types/uia";
 import { LocalNotificationSettings } from "./@types/local_notifications";
 import { buildFeatureSupportMap, Feature, ServerSupport } from "./feature";
-import { CryptoBackend } from "./common-crypto/CryptoBackend";
+import { BackupDecryptor, CryptoBackend } from "./common-crypto/CryptoBackend";
 import { RUST_SDK_STORE_PREFIX } from "./rust-crypto/constants";
 import { BootstrapCrossSigningOpts, CrossSigningKeyInfo, CryptoApi, ImportRoomKeysOpts } from "./crypto-api";
 import { DeviceInfoMap } from "./crypto/DeviceList";
@@ -221,6 +221,7 @@ import {
 } from "./secret-storage";
 import { RegisterRequest, RegisterResponse } from "./@types/registration";
 import { MatrixRTCSessionManager } from "./matrixrtc/MatrixRTCSessionManager";
+import { getRelationsThreadFilter } from "./thread-utils";
 
 export type Store = IStore;
 
@@ -536,21 +537,11 @@ export interface IThreadsCapability extends ICapability {}
 
 export interface IGetLoginTokenCapability extends ICapability {}
 
-/**
- * @deprecated use {@link IGetLoginTokenCapability} instead
- */
-export type IMSC3882GetLoginTokenCapability = IGetLoginTokenCapability;
-
 export const GET_LOGIN_TOKEN_CAPABILITY = new NamespacedValue(
     "m.get_login_token",
     "org.matrix.msc3882.get_login_token",
 );
 
-/**
- * @deprecated use {@link GET_LOGIN_TOKEN_CAPABILITY} instead
- */
-export const UNSTABLE_MSC3882_CAPABILITY = GET_LOGIN_TOKEN_CAPABILITY;
-
 export const UNSTABLE_MSC2666_SHARED_ROOMS = "uk.half-shot.msc2666";
 export const UNSTABLE_MSC2666_MUTUAL_ROOMS = "uk.half-shot.msc2666.mutual_rooms";
 export const UNSTABLE_MSC2666_QUERY_MUTUAL_ROOMS = "uk.half-shot.msc2666.query_mutual_rooms";
@@ -905,7 +896,7 @@ interface IRoomHierarchy {
 
 export interface TimestampToEventResponse {
     event_id: string;
-    origin_server_ts: string;
+    origin_server_ts: number;
 }
 
 interface IWhoamiResponse {
@@ -961,6 +952,7 @@ type CryptoEvents =
     | CryptoEvent.KeyBackupStatus
     | CryptoEvent.KeyBackupFailed
     | CryptoEvent.KeyBackupSessionsRemaining
+    | CryptoEvent.KeyBackupDecryptionKeyCached
     | CryptoEvent.RoomKeyRequest
     | CryptoEvent.RoomKeyRequestCancellation
     | CryptoEvent.VerificationRequest
@@ -970,7 +962,8 @@ type CryptoEvents =
     | CryptoEvent.KeysChanged
     | CryptoEvent.Warning
     | CryptoEvent.DevicesUpdated
-    | CryptoEvent.WillUpdateDevices;
+    | CryptoEvent.WillUpdateDevices
+    | CryptoEvent.LegacyCryptoStoreMigrationProgress;
 
 type MatrixEventEvents = MatrixEventEvent.Decrypted | MatrixEventEvent.Replaced | MatrixEventEvent.VisibilityChange;
 
@@ -1227,7 +1220,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     public reEmitter = new TypedReEmitter<EmittedEvents, ClientEventHandlerMap>(this);
     public olmVersion: [number, number, number] | null = null; // populated after initCrypto
     public usingExternalCrypto = false;
-    public store: Store;
+    private _store!: Store;
     public deviceId: string | null;
     public credentials: { userId: string | null };
 
@@ -1281,7 +1274,6 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     protected cryptoStore?: CryptoStore;
     protected verificationMethods?: VerificationMethod[];
     protected fallbackICEServerAllowed = false;
-    protected roomList: RoomList;
     protected syncApi?: SlidingSyncSdk | SyncApi;
     public roomNameGenerator?: ICreateClientOpts["roomNameGenerator"];
     public pushRules?: IPushRules;
@@ -1313,7 +1305,13 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     protected txnCtr = 0;
     protected mediaHandler = new MediaHandler(this);
     protected sessionId: string;
-    protected pendingEventEncryption = new Map<string, Promise<void>>();
+
+    /** IDs of events which are currently being encrypted.
+     *
+     * This is part of the cancellation mechanism: if the event is no longer listed here when encryption completes,
+     * that tells us that it has been cancelled, and we should not send it.
+     */
+    private eventsBeingEncrypted = new Set<string>();
 
     private useE2eForGroupCall = true;
     private toDeviceMessageQueue: ToDeviceMessageQueue;
@@ -1342,7 +1340,6 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         this.usingExternalCrypto = opts.usingExternalCrypto ?? false;
         this.store = opts.store || new StubStore();
-        this.store.setUserCreator((userId) => User.createUser(userId, this));
         this.deviceId = opts.deviceId || null;
         this.sessionId = randomString(10);
 
@@ -1438,10 +1435,6 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         this.livekitServiceURL = opts.livekitServiceURL;
 
-        // List of which rooms have encryption enabled: separate from crypto because
-        // we still want to know which rooms are encrypted even if crypto is disabled:
-        // we don't want to start sending unencrypted events to them.
-        this.roomList = new RoomList(this.cryptoStore);
         this.roomNameGenerator = opts.roomNameGenerator;
 
         this.toDeviceMessageQueue = new ToDeviceMessageQueue(this);
@@ -1458,7 +1451,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         // correctly handle notification counts on encrypted rooms.
         // This fixes https://github.com/vector-im/element-web/issues/9421
         this.on(RoomEvent.Receipt, (event, room) => {
-            if (room && this.isRoomEncrypted(room.roomId)) {
+            if (room?.hasEncryptionStateEvent()) {
                 // Figure out if we've read something or if it's just informational
                 const content = event.getContent();
                 const isSelf =
@@ -1505,6 +1498,18 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         this.ignoredInvites = new IgnoredInvites(this);
         this._secretStorage = new ServerSideSecretStorageImpl(this, opts.cryptoCallbacks ?? {});
+
+        // having lots of event listeners is not unusual. 0 means "unlimited".
+        this.setMaxListeners(0);
+    }
+
+    public set store(newStore: Store) {
+        this._store = newStore;
+        this._store.setUserCreator((userId) => User.createUser(userId, this));
+    }
+
+    public get store(): Store {
+        return this._store;
     }
 
     /**
@@ -2231,10 +2236,6 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         this.logger.debug("Crypto: Starting up crypto store...");
         await this.cryptoStore.startup();
 
-        // initialise the list of encrypted rooms (whether or not crypto is enabled)
-        this.logger.debug("Crypto: initialising roomlist...");
-        await this.roomList.init();
-
         const userId = this.getUserId();
         if (userId === null) {
             throw new Error(
@@ -2249,15 +2250,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             );
         }
 
-        const crypto = new Crypto(
-            this,
-            userId,
-            this.deviceId,
-            this.store,
-            this.cryptoStore,
-            this.roomList,
-            this.verificationMethods!,
-        );
+        const crypto = new Crypto(this, userId, this.deviceId, this.store, this.cryptoStore, this.verificationMethods!);
 
         this.reEmitter.reEmit(crypto, [
             CryptoEvent.KeyBackupFailed,
@@ -2330,17 +2323,25 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         // importing rust-crypto will download the webassembly, so we delay it until we know it will be
         // needed.
+        this.logger.debug("Downloading Rust crypto library");
         const RustCrypto = await import("./rust-crypto");
-        const rustCrypto = await RustCrypto.initRustCrypto(
-            this.logger,
-            this.http,
-            userId,
-            deviceId,
-            this.secretStorage,
-            this.cryptoCallbacks,
-            useIndexedDB ? RUST_SDK_STORE_PREFIX : null,
-            this.pickleKey,
-        );
+
+        const rustCrypto = await RustCrypto.initRustCrypto({
+            logger: this.logger,
+            http: this.http,
+            userId: userId,
+            deviceId: deviceId,
+            secretStorage: this.secretStorage,
+            cryptoCallbacks: this.cryptoCallbacks,
+            storePrefix: useIndexedDB ? RUST_SDK_STORE_PREFIX : null,
+            storePassphrase: this.pickleKey,
+            legacyCryptoStore: this.cryptoStore,
+            legacyPickleKey: this.pickleKey ?? "DEFAULT_KEY",
+            legacyMigrationProgressListener: (progress, total) => {
+                this.emit(CryptoEvent.LegacyCryptoStoreMigrationProgress, progress, total);
+            },
+        });
+
         rustCrypto.setSupportedVerificationMethods(this.verificationMethods);
 
         this.cryptoBackend = rustCrypto;
@@ -2358,6 +2359,10 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             CryptoEvent.KeyBackupStatus,
             CryptoEvent.KeyBackupSessionsRemaining,
             CryptoEvent.KeyBackupFailed,
+            CryptoEvent.KeyBackupDecryptionKeyCached,
+            CryptoEvent.KeysChanged,
+            CryptoEvent.DevicesUpdated,
+            CryptoEvent.WillUpdateDevices,
         ]);
     }
 
@@ -2392,6 +2397,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      *
      * @returns base64-encoded ed25519 key. Null if crypto is
      *    disabled.
+     *
+     * @deprecated Prefer {@link CryptoApi.getOwnDeviceKeys}
      */
     public getDeviceEd25519Key(): string | null {
         return this.crypto?.getDeviceEd25519Key() ?? null;
@@ -2402,6 +2409,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      *
      * @returns base64-encoded curve25519 key. Null if crypto is
      *    disabled.
+     *
+     * @deprecated Use {@link CryptoApi.getOwnDeviceKeys}
      */
     public getDeviceCurve25519Key(): string | null {
         return this.crypto?.getDeviceCurve25519Key() ?? null;
@@ -3245,6 +3254,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * @param roomId - The room ID to enable encryption in.
      * @param config - The encryption config for the room.
      * @returns A promise that will resolve when encryption is set up.
+     *
+     * @deprecated Not supported for Rust Cryptography. To enable encryption in a room, send an `m.room.encryption`
+     * state event.
      */
     public setRoomEncryption(roomId: string, config: IRoomEncryption): Promise<void> {
         if (!this.crypto) {
@@ -3257,6 +3269,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * Whether encryption is enabled for a room.
      * @param roomId - the room id to query.
      * @returns whether encryption is enabled.
+     *
+     * @deprecated Not correctly supported for Rust Cryptography. Use {@link CryptoApi.isEncryptionEnabledInRoom} and/or
+     *    {@link Room.hasEncryptionStateEvent}.
      */
     public isRoomEncrypted(roomId: string): boolean {
         const room = this.getRoom(roomId);
@@ -3268,15 +3283,14 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         // if there is an 'm.room.encryption' event in this room, it should be
         // encrypted (independently of whether we actually support encryption)
-        const ev = room.currentState.getStateEvents(EventType.RoomEncryption, "");
-        if (ev) {
+        if (room.hasEncryptionStateEvent()) {
             return true;
         }
 
         // we don't have an m.room.encrypted event, but that might be because
         // the server is hiding it from us. Check the store to see if it was
         // previously encrypted.
-        return this.roomList.isRoomEncrypted(roomId);
+        return this.crypto?.isRoomEncrypted(roomId) ?? false;
     }
 
     /**
@@ -3639,6 +3653,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     /**
      * Marks all group sessions as needing to be backed up and schedules them to
      * upload in the background as soon as possible.
+     *
+     * (This is done automatically as part of {@link CryptoApi.resetKeyBackup},
+     * so there is probably no need to call this manually.)
      */
     public async scheduleAllGroupSessionsForBackup(): Promise<void> {
         if (!this.crypto) {
@@ -3651,6 +3668,10 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     /**
      * Marks all group sessions as needing to be backed up without scheduling
      * them to upload in the background.
+     *
+     * (This is done automatically as part of {@link CryptoApi.resetKeyBackup},
+     * so there is probably no need to call this manually.)
+     *
      * @returns Promise which resolves to the number of sessions requiring a backup.
      */
     public flagAllGroupSessionsForBackup(): Promise<number> {
@@ -3898,7 +3919,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         }
 
         let totalKeyCount = 0;
-        let keys: IMegolmSessionData[] = [];
+        let totalFailures = 0;
+        let totalImported = 0;
 
         const path = this.makeKeyBackupPath(targetRoomId, targetSessionId, backupInfo.version);
 
@@ -3934,25 +3956,61 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
                 { prefix: ClientPrefix.V3 },
             );
 
-            if ((res as IRoomsKeysResponse).rooms) {
-                const rooms = (res as IRoomsKeysResponse).rooms;
-                for (const [roomId, roomData] of Object.entries(rooms)) {
-                    if (!roomData.sessions) continue;
+            // We have finished fetching the backup, go to next step
+            if (progressCallback) {
+                progressCallback({
+                    stage: "load_keys",
+                });
+            }
 
-                    totalKeyCount += Object.keys(roomData.sessions).length;
-                    const roomKeys = await backupDecryptor.decryptSessions(roomData.sessions);
-                    for (const k of roomKeys) {
-                        k.room_id = roomId;
-                        keys.push(k);
-                    }
-                }
+            if ((res as IRoomsKeysResponse).rooms) {
+                // We have a full backup here, it can get quite big, so we need to decrypt and import it in chunks.
+
+                // Get the total count as a first pass
+                totalKeyCount = this.getTotalKeyCount(res as IRoomsKeysResponse);
+                // Now decrypt and import the keys in chunks
+                await this.handleDecryptionOfAFullBackup(
+                    res as IRoomsKeysResponse,
+                    backupDecryptor,
+                    200,
+                    async (chunk) => {
+                        // We have a chunk of decrypted keys: import them
+                        try {
+                            await this.cryptoBackend!.importBackedUpRoomKeys(chunk, {
+                                untrusted,
+                            });
+                            totalImported += chunk.length;
+                        } catch (e) {
+                            totalFailures += chunk.length;
+                            // We failed to import some keys, but we should still try to import the rest?
+                            // Log the error and continue
+                            logger.error("Error importing keys from backup", e);
+                        }
+
+                        if (progressCallback) {
+                            progressCallback({
+                                total: totalKeyCount,
+                                successes: totalImported,
+                                stage: "load_keys",
+                                failures: totalFailures,
+                            });
+                        }
+                    },
+                );
             } else if ((res as IRoomKeysResponse).sessions) {
+                // For now we don't chunk for a single room backup, but we could in the future.
+                // Currently it is not used by the application.
                 const sessions = (res as IRoomKeysResponse).sessions;
                 totalKeyCount = Object.keys(sessions).length;
-                keys = await backupDecryptor.decryptSessions(sessions);
+                const keys = await backupDecryptor.decryptSessions(sessions);
                 for (const k of keys) {
                     k.room_id = targetRoomId!;
                 }
+                await this.cryptoBackend.importBackedUpRoomKeys(keys, {
+                    progressCallback,
+                    untrusted,
+                });
+                totalImported = keys.length;
             } else {
                 totalKeyCount = 1;
                 try {
@@ -3961,7 +4019,12 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
                     });
                     key.room_id = targetRoomId!;
                     key.session_id = targetSessionId!;
-                    keys.push(key);
+
+                    await this.cryptoBackend.importBackedUpRoomKeys([key], {
+                        progressCallback,
+                        untrusted,
+                    });
+                    totalImported = 1;
                 } catch (e) {
                     this.logger.debug("Failed to decrypt megolm session from backup", e);
                 }
@@ -3970,16 +4033,88 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             backupDecryptor.free();
         }
 
-        await this.cryptoBackend.importRoomKeys(keys, {
-            progressCallback,
-            untrusted,
-            source: "backup",
-        });
-
         /// in case entering the passphrase would add a new signature?
         await this.cryptoBackend.checkKeyBackupAndEnable();
 
-        return { total: totalKeyCount, imported: keys.length };
+        return { total: totalKeyCount, imported: totalImported };
+    }
+
+    /**
+     * This method calculates the total number of keys present in the response of a `/room_keys/keys` call.
+     *
+     * @param res - The response from the server containing the keys to be counted.
+     *
+     * @returns The total number of keys in the backup.
+     */
+    private getTotalKeyCount(res: IRoomsKeysResponse): number {
+        const rooms = res.rooms;
+        let totalKeyCount = 0;
+        for (const roomData of Object.values(rooms)) {
+            if (!roomData.sessions) continue;
+            totalKeyCount += Object.keys(roomData.sessions).length;
+        }
+        return totalKeyCount;
+    }
+
+    /**
+     * This method handles the decryption of a full backup, i.e a call to `/room_keys/keys`.
+     * It will decrypt the keys in chunks and call the `block` callback for each chunk.
+     *
+     * @param res - The response from the server containing the keys to be decrypted.
+     * @param backupDecryptor - An instance of the BackupDecryptor class used to decrypt the keys.
+     * @param chunkSize - The size of the chunks to be processed at a time.
+     * @param block - A callback function that is called for each chunk of keys.
+     *
+     * @returns A promise that resolves when the decryption is complete.
+     */
+    private async handleDecryptionOfAFullBackup(
+        res: IRoomsKeysResponse,
+        backupDecryptor: BackupDecryptor,
+        chunkSize: number,
+        block: (chunk: IMegolmSessionData[]) => Promise<void>,
+    ): Promise<void> {
+        const rooms = (res as IRoomsKeysResponse).rooms;
+
+        let groupChunkCount = 0;
+        let chunkGroupByRoom: Map<string, IKeyBackupRoomSessions> = new Map();
+
+        const handleChunkCallback = async (roomChunks: Map<string, IKeyBackupRoomSessions>): Promise<void> => {
+            const currentChunk: IMegolmSessionData[] = [];
+            for (const roomId of roomChunks.keys()) {
+                const decryptedSessions = await backupDecryptor.decryptSessions(roomChunks.get(roomId)!);
+                for (const sessionId in decryptedSessions) {
+                    const k = decryptedSessions[sessionId];
+                    k.room_id = roomId;
+                    currentChunk.push(k);
+                }
+            }
+            await block(currentChunk);
+        };
+
+        for (const [roomId, roomData] of Object.entries(rooms)) {
+            if (!roomData.sessions) continue;
+
+            chunkGroupByRoom.set(roomId, {});
+
+            for (const [sessionId, session] of Object.entries(roomData.sessions)) {
+                const sessionsForRoom = chunkGroupByRoom.get(roomId)!;
+                sessionsForRoom[sessionId] = session;
+                groupChunkCount += 1;
+                if (groupChunkCount >= chunkSize) {
+                    // We have enough chunks to decrypt
+                    await handleChunkCallback(chunkGroupByRoom);
+                    chunkGroupByRoom = new Map();
+                    // There might be remaining keys for that room, so add back an entry for the current room.
+                    chunkGroupByRoom.set(roomId, {});
+                    groupChunkCount = 0;
+                }
+            }
+        }
+
+        // Handle remaining chunk if needed
+        if (groupChunkCount > 0) {
+            await handleChunkCallback(chunkGroupByRoom);
+        }
     }
 
     public deleteKeysFromBackup(roomId: undefined, sessionId: undefined, version?: string): Promise<void>;
@@ -4006,7 +4141,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             throw new Error("End-to-end encryption disabled");
         }
 
-        const roomEncryption = this.roomList.getRoomEncryption(roomId);
+        const roomEncryption = this.crypto?.getRoomEncryption(roomId);
         if (!roomEncryption) {
             // unknown room, or unencrypted room
             this.logger.error("Unknown room.  Not sharing decryption keys");
@@ -4328,9 +4463,10 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             throw new Error("cannot cancel an event with status " + event.status);
         }
 
-        // if the event is currently being encrypted then
+        // If the event is currently being encrypted then remove it from the pending list, to indicate that it should
+        // not be sent.
         if (event.status === EventStatus.ENCRYPTING) {
-            this.pendingEventEncryption.delete(event.getId()!);
+            this.eventsBeingEncrypted.delete(event.getId()!);
         } else if (this.scheduler && event.status === EventStatus.QUEUED) {
             // tell the scheduler to forget about it, if it's queued
             this.scheduler.removeEventFromQueue(event);
@@ -4629,96 +4765,102 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * encrypts the event if necessary; adds the event to the queue, or sends it; marks the event as sent/unsent
      * @returns returns a promise which resolves with the result of the send request
      */
-    protected encryptAndSendEvent(room: Room | null, event: MatrixEvent): Promise<ISendEventResponse> {
-        let cancelled = false;
-        // Add an extra Promise.resolve() to turn synchronous exceptions into promise rejections,
-        // so that we can handle synchronous and asynchronous exceptions with the
-        // same code path.
-        return Promise.resolve()
-            .then(() => {
-                const encryptionPromise = this.encryptEventIfNeeded(event, room ?? undefined);
-                if (!encryptionPromise) return null; // doesn't need encryption
+    protected async encryptAndSendEvent(room: Room | null, event: MatrixEvent): Promise<ISendEventResponse> {
+        try {
+            let cancelled: boolean;
+            this.eventsBeingEncrypted.add(event.getId()!);
+            try {
+                await this.encryptEventIfNeeded(event, room ?? undefined);
+            } finally {
+                cancelled = !this.eventsBeingEncrypted.delete(event.getId()!);
+            }
 
-                this.pendingEventEncryption.set(event.getId()!, encryptionPromise);
-                this.updatePendingEventStatus(room, event, EventStatus.ENCRYPTING);
-                return encryptionPromise.then(() => {
-                    if (!this.pendingEventEncryption.has(event.getId()!)) {
-                        // cancelled via MatrixClient::cancelPendingEvent
-                        cancelled = true;
-                        return;
-                    }
-                    this.updatePendingEventStatus(room, event, EventStatus.SENDING);
-                });
-            })
-            .then(() => {
-                if (cancelled) return {} as ISendEventResponse;
-                let promise: Promise<ISendEventResponse> | null = null;
-                if (this.scheduler) {
-                    // if this returns a promise then the scheduler has control now and will
-                    // resolve/reject when it is done. Internally, the scheduler will invoke
-                    // processFn which is set to this._sendEventHttpRequest so the same code
-                    // path is executed regardless.
-                    promise = this.scheduler.queueEvent(event);
-                    if (promise && this.scheduler.getQueueForEvent(event)!.length > 1) {
-                        // event is processed FIFO so if the length is 2 or more we know
-                        // this event is stuck behind an earlier event.
-                        this.updatePendingEventStatus(room, event, EventStatus.QUEUED);
-                    }
-                }
+            if (cancelled) {
+                // cancelled via MatrixClient::cancelPendingEvent
+                return {} as ISendEventResponse;
+            }
 
-                if (!promise) {
-                    promise = this.sendEventHttpRequest(event);
-                    if (room) {
-                        promise = promise.then((res) => {
-                            room.updatePendingEvent(event, EventStatus.SENT, res["event_id"]);
-                            return res;
-                        });
-                    }
-                }
+            // encryptEventIfNeeded may have updated the status from SENDING to ENCRYPTING. If so, we need
+            // to put it back.
+            if (event.status === EventStatus.ENCRYPTING) {
+                this.updatePendingEventStatus(room, event, EventStatus.SENDING);
+            }
 
-                return promise;
-            })
-            .catch((err) => {
-                this.logger.error("Error sending event", err.stack || err);
-                try {
-                    // set the error on the event before we update the status:
-                    // updating the status emits the event, so the state should be
-                    // consistent at that point.
-                    event.error = err;
-                    this.updatePendingEventStatus(room, event, EventStatus.NOT_SENT);
-                } catch (e) {
-                    this.logger.error("Exception in error handler!", (<Error>e).stack || err);
+            let promise: Promise<ISendEventResponse> | null = null;
+            if (this.scheduler) {
+                // if this returns a promise then the scheduler has control now and will
+                // resolve/reject when it is done. Internally, the scheduler will invoke
+                // processFn which is set to this._sendEventHttpRequest so the same code
+                // path is executed regardless.
+                promise = this.scheduler.queueEvent(event);
+                if (promise && this.scheduler.getQueueForEvent(event)!.length > 1) {
+                    // event is processed FIFO so if the length is 2 or more we know
+                    // this event is stuck behind an earlier event.
+                    this.updatePendingEventStatus(room, event, EventStatus.QUEUED);
                 }
-                if (err instanceof MatrixError) {
-                    err.event = event;
+            }
+
+            if (!promise) {
+                promise = this.sendEventHttpRequest(event);
+                if (room) {
+                    promise = promise.then((res) => {
+                        room.updatePendingEvent(event, EventStatus.SENT, res["event_id"]);
+                        return res;
+                    });
                 }
-                throw err;
-            });
+            }
+
+            return await promise;
+        } catch (err) {
+            this.logger.error("Error sending event", err);
+            try {
+                // set the error on the event before we update the status:
+                // updating the status emits the event, so the state should be
+                // consistent at that point.
+                event.error = <MatrixError>err;
+                this.updatePendingEventStatus(room, event, EventStatus.NOT_SENT);
+            } catch (e) {
+                this.logger.error("Exception in error handler!", e);
+            }
+            if (err instanceof MatrixError) {
+                err.event = event;
+            }
+            throw err;
+        }
     }
 
-    private encryptEventIfNeeded(event: MatrixEvent, room?: Room): Promise<void> | null {
+    private async encryptEventIfNeeded(event: MatrixEvent, room?: Room): Promise<void> {
+        // If the room is unknown, we cannot encrypt for it
+        if (!room) return;
+
+        if (!(await this.shouldEncryptEventForRoom(event, room))) return;
+
+        if (!this.cryptoBackend && this.usingExternalCrypto) {
+            // The client has opted to allow sending messages to encrypted
+            // rooms even if the room is encrypted, and we haven't set up
+            // crypto. This is useful for users of matrix-org/pantalaimon
+            return;
+        }
+
+        if (!this.cryptoBackend) {
+            throw new Error("This room is configured to use encryption, but your client does not support encryption.");
+        }
+
+        this.updatePendingEventStatus(room, event, EventStatus.ENCRYPTING);
+        await this.cryptoBackend.encryptEvent(event, room);
+    }
+
+    /**
+     * Determine whether a given event should be encrypted when we send it to the given room.
+     *
+     * This takes into account event type and room configuration.
+     */
+    private async shouldEncryptEventForRoom(event: MatrixEvent, room: Room): Promise<boolean> {
         if (event.isEncrypted()) {
             // this event has already been encrypted; this happens if the
             // encryption step succeeded, but the send step failed on the first
             // attempt.
-            return null;
-        }
-
-        if (event.isRedaction()) {
-            // Redactions do not support encryption in the spec at this time,
-            // whilst it mostly worked in some clients, it wasn't compliant.
-            return null;
-        }
-
-        if (!room || !this.isRoomEncrypted(event.getRoomId()!)) {
-            return null;
-        }
-
-        if (!this.cryptoBackend && this.usingExternalCrypto) {
-            // The client has opted to allow sending messages to encrypted
-            // rooms even if the room is encrypted, and we haven't setup
-            // crypto. This is useful for users of matrix-org/pantalaimon
-            return null;
+            return false;
         }
 
         if (event.getType() === EventType.Reaction) {
@@ -4732,14 +4874,23 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             // The reaction key / content / emoji value does warrant encrypting, but
             // this will be handled separately by encrypting just this value.
             // See https://github.com/matrix-org/matrix-doc/pull/1849#pullrequestreview-248763642
-            return null;
+            return false;
         }
 
-        if (!this.cryptoBackend) {
-            throw new Error("This room is configured to use encryption, but your client does not support encryption.");
+        if (event.isRedaction()) {
+            // Redactions do not support encryption in the spec at this time.
+            // Whilst it mostly worked in some clients, it wasn't compliant.
+            return false;
         }
 
-        return this.cryptoBackend.encryptEvent(event, room);
+        // If the room has an m.room.encryption event, we should encrypt.
+        if (room.hasEncryptionStateEvent()) return true;
+
+        // If we have a crypto impl, and *it* thinks we should encrypt, then we should.
+        if (await this.cryptoBackend?.isEncryptionEnabledInRoom(room.roomId)) return true;
+
+        // Otherwise, no need to encrypt.
+        return false;
     }
 
     /**
@@ -4754,7 +4905,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         eventType?: EventType | string | null,
     ): EventType | string | null | undefined {
         if (eventType === EventType.Reaction) return eventType;
-        return this.isRoomEncrypted(roomId) ? EventType.RoomMessageEncrypted : eventType;
+        return this.getRoom(roomId)?.hasEncryptionStateEvent() ? EventType.RoomMessageEncrypted : eventType;
     }
 
     protected updatePendingEventStatus(room: Room | null, event: MatrixEvent, newStatus: EventStatus): void {
@@ -5167,7 +5318,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         const room = this.getRoom(event.getRoomId());
         if (room && this.credentials.userId) {
-            room.addLocalEchoReceipt(this.credentials.userId, event, receiptType);
+            room.addLocalEchoReceipt(this.credentials.userId, event, receiptType, unthreaded);
         }
         return promise;
     }
@@ -5684,6 +5835,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * @param allowDirectLinks - If true, return any non-mxc URLs
      * directly. Fetching such URLs will leak information about the user to
      * anyone they share a room with. If false, will return null for such URLs.
+     * @param allowRedirects - If true, the caller supports the URL being 307 or
+     * 308 redirected to another resource upon request. If false, redirects
+     * are not expected.
      * @returns the avatar URL or null.
      */
     public mxcUrlToHttp(
@@ -5692,8 +5846,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         height?: number,
         resizeMethod?: string,
         allowDirectLinks?: boolean,
+        allowRedirects?: boolean,
     ): string | null {
-        return getHttpUriForMxc(this.baseUrl, mxcUrl, width, height, resizeMethod, allowDirectLinks);
+        return getHttpUriForMxc(this.baseUrl, mxcUrl, width, height, resizeMethod, allowDirectLinks, allowRedirects);
     }
 
     /**
@@ -5972,14 +6127,14 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
                 const resOlder: IRelationsResponse = await this.fetchRelations(
                     timelineSet.room.roomId,
                     thread.id,
-                    THREAD_RELATION_TYPE.name,
+                    null,
                     null,
                     { dir: Direction.Backward, from: res.start, recurse: recurse || undefined },
                 );
                 const resNewer: IRelationsResponse = await this.fetchRelations(
                     timelineSet.room.roomId,
                     thread.id,
-                    THREAD_RELATION_TYPE.name,
+                    null,
                     null,
                     { dir: Direction.Forward, from: res.end, recurse: recurse || undefined },
                 );
@@ -5987,10 +6142,11 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
                     // Order events from most recent to oldest (reverse-chronological).
                     // We start with the last event, since that's the point at which we have known state.
                     // events_after is already backwards; events_before is forwards.
-                    ...resNewer.chunk.reverse().map(mapper),
+                    ...resNewer.chunk.reverse().filter(getRelationsThreadFilter(thread.id)).map(mapper),
                     event,
-                    ...resOlder.chunk.map(mapper),
+                    ...resOlder.chunk.filter(getRelationsThreadFilter(thread.id)).map(mapper),
                 ];
+
                 for (const event of events) {
                     await timelineSet.thread?.processEvent(event);
                 }
@@ -6365,6 +6521,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
                         const stateEvents = res.state.filter(noUnsafeEventProps).map(this.getEventMapper());
                         roomState.setUnknownStateEvents(stateEvents);
                     }
+
                     const token = res.end;
                     const matrixEvents = res.chunk.filter(noUnsafeEventProps).map(this.getEventMapper());
 
@@ -6392,7 +6549,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             }
 
             const recurse = this.canSupport.get(Feature.RelationsRecursion) !== ServerSupport.Unsupported;
-            promise = this.fetchRelations(eventTimeline.getRoomId() ?? "", thread.id, THREAD_RELATION_TYPE.name, null, {
+            promise = this.fetchRelations(eventTimeline.getRoomId() ?? "", thread.id, null, null, {
                 dir,
                 limit: opts.limit,
                 from: token ?? undefined,
@@ -6400,7 +6557,10 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             })
                 .then(async (res) => {
                     const mapper = this.getEventMapper();
-                    const matrixEvents = res.chunk.filter(noUnsafeEventProps).map(mapper);
+                    const matrixEvents = res.chunk
+                        .filter(noUnsafeEventProps)
+                        .filter(getRelationsThreadFilter(thread.id))
+                        .map(mapper);
 
                     // Process latest events first
                     for (const event of matrixEvents.slice().reverse()) {
@@ -7448,16 +7608,11 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             return this.serverVersionsPromise;
         }
 
+        // We send an authenticated request as of MSC4026
         this.serverVersionsPromise = this.http
-            .request<IServerVersions>(
-                Method.Get,
-                "/_matrix/client/versions",
-                undefined, // queryParams
-                undefined, // data
-                {
-                    prefix: "",
-                },
-            )
+            .authedRequest<IServerVersions>(Method.Get, "/_matrix/client/versions", undefined, undefined, {
+                prefix: "",
+            })
             .catch((e) => {
                 // Need to unset this if it fails, otherwise we'll never retry
                 this.serverVersionsPromise = undefined;
@@ -7595,7 +7750,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     public async relations(
         roomId: string,
         eventId: string,
-        relationType?: RelationType | string | null,
+        relationType: RelationType | string | null,
         eventType?: EventType | string | null,
         opts: IRelationsRequestOpts = { dir: Direction.Backward },
     ): Promise<{
@@ -7730,6 +7885,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      */
     public setAccessToken(token: string): void {
         this.http.opts.accessToken = token;
+        // The /versions response can vary for different users so clear the cache
+        this.serverVersionsPromise = undefined;
     }
 
     /**
@@ -8045,50 +8202,23 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
     /**
      * Make a request for an `m.login.token` to be issued as per
-     * [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882).
-     * The server may require User-Interactive auth.
+     * https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv1loginget_token
      *
-     * Compatibility with unstable implementations of MSC3882 is deprecated and will be removed in a future release.
+     * The server may require User-Interactive auth.
      *
      * @param auth - Optional. Auth data to supply for User-Interactive auth.
      * @returns Promise which resolves: On success, the token response
      * or UIA auth data.
      */
     public async requestLoginToken(auth?: AuthDict): Promise<UIAResponse<LoginTokenPostResponse>> {
-        // use capabilities to determine which revision of the MSC is being used
-        const capabilities = await this.getCapabilities();
-
-        let endpoint: string;
-        if (capabilities[GET_LOGIN_TOKEN_CAPABILITY.name]) {
-            // use the stable endpoint
-            endpoint = `${ClientPrefix.V1}/login/get_token`;
-        } else if (capabilities[GET_LOGIN_TOKEN_CAPABILITY.altName!]) {
-            // newer unstable r1 endpoint
-            endpoint = `${ClientPrefix.Unstable}/org.matrix.msc3882/login/get_token`;
-        } else {
-            // old unstable r0 endpoint
-            endpoint = `${ClientPrefix.Unstable}/org.matrix.msc3882/login/token`;
-        }
-
         const body: UIARequest<{}> = { auth };
-        const res = await this.http.authedRequest<UIAResponse<LoginTokenPostResponse>>(
+        return this.http.authedRequest<UIAResponse<LoginTokenPostResponse>>(
             Method.Post,
-            endpoint,
+            "/login/get_token",
             undefined, // no query params
             body,
-            { prefix: "" },
+            { prefix: ClientPrefix.V1 },
         );
-
-        // the representation of expires_in changed from unstable revision 0 to unstable revision 1 so we cross populate
-        if ("login_token" in res) {
-            if (typeof res.expires_in_ms === "number") {
-                res.expires_in = Math.floor(res.expires_in_ms / 1000);
-            } else if (typeof res.expires_in === "number") {
-                res.expires_in_ms = res.expires_in * 1000;
-            }
-        }
-
-        return res;
     }
 
     /**
@@ -8145,7 +8275,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     public fetchRelations(
         roomId: string,
         eventId: string,
-        relationType?: RelationType | string | null,
+        relationType: RelationType | string | null,
         eventType?: EventType | string | null,
         opts: IRelationsRequestOpts = { dir: Direction.Backward },
     ): Promise<IRelationsResponse> {
@@ -9858,6 +9988,19 @@ export function fixNotificationCountOnDecryption(cli: MatrixClient, event: Matri
     const room = cli.getRoom(event.getRoomId());
     if (!room || !ourUserId || !eventId) return;
 
+    // Due to threads, we can get relation events (eg. edits & reactions) that never get
+    // added to a timeline and so cannot be found in their own room (their edit / reaction
+    // still applies to the event it needs to, so it doesn't matter too much). However, if
+    // we try to process notification about this event, we'll get very confused because we
+    // won't be able to find the event in the room, so will assume it must be unread, even
+    // if it's actually read. We therefore skip anything that isn't in the room. This isn't
+    // *great*, so if we can fix the homeless events (eg. with MSC4023) then we should probably
+    // remove this workaround.
+    if (!room.findEventById(eventId)) {
+        logger.info(`Decrypted event ${event.getId()} is not in room ${room.roomId}: ignoring`);
+        return;
+    }
+
     const isThreadEvent = !!event.threadRootId && !event.isThreadRoot;
 
     let hasReadEvent;
@@ -9942,7 +10085,7 @@ export function threadIdForReceipt(event: MatrixEvent): string {
  * @returns true if this event is considered to be in the main timeline as far
  *               as receipts are concerned.
  */
-function inMainTimelineForReceipt(event: MatrixEvent): boolean {
+export function inMainTimelineForReceipt(event: MatrixEvent): boolean {
     if (!event.threadRootId) {
         // Not in a thread: then it is in the main timeline
         return true;

src/common-crypto/CryptoBackend.ts

@@ -17,7 +17,7 @@ limitations under the License.
 import type { IDeviceLists, IToDeviceEvent } from "../sync-accumulator";
 import { IClearEvent, MatrixEvent } from "../models/event";
 import { Room } from "../models/room";
-import { CryptoApi } from "../crypto-api";
+import { CryptoApi, ImportRoomKeysOpts } from "../crypto-api";
 import { CrossSigningInfo, UserTrustLevel } from "../crypto/CrossSigning";
 import { IEncryptedEventInfo } from "../crypto/api";
 import { KeyBackupInfo, KeyBackupSession } from "../crypto-api/keybackup";
@@ -108,6 +108,15 @@ export interface CryptoBackend extends SyncCryptoCallbacks, CryptoApi {
      * @param privKey - The private decryption key.
      */
     getBackupDecryptor(backupInfo: KeyBackupInfo, privKey: ArrayLike<number>): Promise<BackupDecryptor>;
+
+    /**
+     * Import a list of room keys restored from backup
+     *
+     * @param keys - a list of session export objects
+     * @param opts - options object
+     * @returns a promise which resolves once the keys have been imported
+     */
+    importBackedUpRoomKeys(keys: IMegolmSessionData[], opts?: ImportRoomKeysOpts): Promise<void>;
 }
 
 /** The methods which crypto implementations should expose to the Sync api

src/content-repo.ts

@@ -28,6 +28,9 @@ import { encodeParams } from "./utils";
  * directly. Fetching such URLs will leak information about the user to
  * anyone they share a room with. If false, will return the emptry string
  * for such URLs.
+ * @param allowRedirects - If true, the caller supports the URL being 307 or
+ * 308 redirected to another resource upon request. If false, redirects
+ * are not expected.
  * @returns The complete URL to the content.
  */
 export function getHttpUriForMxc(
@@ -37,6 +40,7 @@ export function getHttpUriForMxc(
     height?: number,
     resizeMethod?: string,
     allowDirectLinks = false,
+    allowRedirects?: boolean,
 ): string {
     if (typeof mxc !== "string" || !mxc) {
         return "";
@@ -67,6 +71,11 @@ export function getHttpUriForMxc(
         prefix = "/_matrix/media/v3/thumbnail/";
     }
 
+    if (typeof allowRedirects === "boolean") {
+        // We add this after, so we don't convert everything to a thumbnail request.
+        params["allow_redirect"] = JSON.stringify(allowRedirects);
+    }
+
     const fragmentOffset = serverAndMediaId.indexOf("#");
     let fragment = "";
     if (fragmentOffset >= 0) {

src/crypto-api.ts

@@ -18,7 +18,7 @@ import type { IMegolmSessionData } from "./@types/crypto";
 import { Room } from "./models/room";
 import { DeviceMap } from "./models/device";
 import { UIAuthCallback } from "./interactive-auth";
-import { AddSecretStorageKeyOpts, SecretStorageCallbacks, SecretStorageKeyDescription } from "./secret-storage";
+import { PassphraseInfo, SecretStorageCallbacks, SecretStorageKeyDescription } from "./secret-storage";
 import { VerificationRequest } from "./crypto-api/verification";
 import { BackupTrustInfo, KeyBackupCheck, KeyBackupInfo } from "./crypto-api/keybackup";
 import { ISignatures } from "./@types/signed";
@@ -46,6 +46,25 @@ export interface CryptoApi {
      */
     getVersion(): string;
 
+    /**
+     * Get the public part of the device keys for the current device.
+     *
+     * @returns The public device keys.
+     */
+    getOwnDeviceKeys(): Promise<OwnDeviceKeys>;
+
+    /**
+     * Check if we believe the given room to be encrypted.
+     *
+     * This method returns true if the room has been configured with encryption. The setting is persistent, so that
+     * even if the encryption event is removed from the room state, it still returns true. This helps to guard against
+     * a downgrade attack wherein a server admin attempts to remove encryption.
+     *
+     * @returns `true` if the room with the supplied ID is encrypted. `false` if the room is not encrypted, or is unknown to
+     * us.
+     */
+    isEncryptionEnabledInRoom(roomId: string): Promise<boolean>;
+
     /**
      * Perform any background tasks that can be done before a message is ready to
      * send, in order to speed up sending of the message.
@@ -162,7 +181,7 @@ export interface CryptoApi {
     /**
      * Mark the given device as locally verified.
      *
-     * Marking a devices as locally verified has much the same effect as completing the verification dance, or receiving
+     * Marking a device as locally verified has much the same effect as completing the verification dance, or receiving
      * a cross-signing signature for it.
      *
      * @param userId - owner of the device
@@ -175,6 +194,21 @@ export interface CryptoApi {
      */
     setDeviceVerified(userId: string, deviceId: string, verified?: boolean): Promise<void>;
 
+    /**
+     * Cross-sign one of our own devices.
+     *
+     * This will create a signature for the device using our self-signing key, and publish that signature.
+     * Cross-signing a device indicates, to our other devices and to other users, that we have verified that it really
+     * belongs to us.
+     *
+     * Requires that cross-signing has been set up on this device (normally by calling {@link bootstrapCrossSigning}).
+     *
+     * *Note*: Do not call this unless you have verified, somehow, that the device is genuine!
+     *
+     * @param deviceId - ID of the device to be signed.
+     */
+    crossSignDevice(deviceId: string): Promise<void>;
+
     /**
      * Checks whether cross signing:
      * - is enabled on this account and trusted by this device
@@ -564,9 +598,9 @@ export class DeviceVerificationStatus {
  */
 export interface ImportRoomKeyProgressData {
     stage: string; // TODO: Enum
-    successes: number;
-    failures: number;
-    total: number;
+    successes?: number;
+    failures?: number;
+    total?: number;
 }
 
 /**
@@ -575,9 +609,10 @@ export interface ImportRoomKeyProgressData {
 export interface ImportRoomKeysOpts {
     /** Reports ongoing progress of the import process. Can be used for feedback. */
     progressCallback?: (stage: ImportRoomKeyProgressData) => void;
-    // TODO, the rust SDK will always such imported keys as untrusted
+    /** @deprecated the rust SDK will always such imported keys as untrusted */
     untrusted?: boolean;
-    source?: String; // TODO: Enum (backup, file, ??)
+    /** @deprecated not useful externally */
+    source?: string;
 }
 
 /**
@@ -687,10 +722,15 @@ export interface CrossSigningKeyInfo {
 }
 
 /**
- * Recovery key created by {@link CryptoApi#createRecoveryKeyFromPassphrase}
+ * Recovery key created by {@link CryptoApi#createRecoveryKeyFromPassphrase} or {@link CreateSecretStorageOpts#createSecretStorageKey}.
  */
 export interface GeneratedSecretStorageKey {
-    keyInfo?: AddSecretStorageKeyOpts;
+    keyInfo?: {
+        /** If the key was derived from a passphrase, information (algorithm, salt, etc) on that derivation. */
+        passphrase?: PassphraseInfo;
+        /** Optional human-readable name for the key, to be stored in account_data. */
+        name?: string;
+    };
     /** The raw generated private key. */
     privateKey: Uint8Array;
     /** The generated key, encoded for display to the user per https://spec.matrix.org/v1.7/client-server-api/#key-representation. */
@@ -749,5 +789,13 @@ export enum EventShieldReason {
     MISMATCHED_SENDER_KEY,
 }
 
+/** The result of a call to {@link CryptoApi.getOwnDeviceKeys} */
+export interface OwnDeviceKeys {
+    /** Public part of the Ed25519 fingerprint key for the current device, base64 encoded. */
+    ed25519: string;
+    /** Public part of the Curve25519 identity key for the current device, base64 encoded. */
+    curve25519: string;
+}
+
 export * from "./crypto-api/verification";
 export * from "./crypto-api/keybackup";

src/crypto/OlmDevice.ts

@@ -64,7 +64,7 @@ export interface InboundGroupSessionData {
     room_id: string; // eslint-disable-line camelcase
     /** pickled Olm.InboundGroupSession */
     session: string;
-    keysClaimed: Record<string, string>;
+    keysClaimed?: Record<string, string>;
     /** Devices involved in forwarding this session to us (normally empty). */
     forwardingCurve25519KeyChain: string[];
     /** whether this session is untrusted. */

src/crypto/RoomList.ts

@@ -29,6 +29,13 @@ export interface IRoomEncryption {
 }
 /* eslint-enable camelcase */
 
+/**
+ * Information about the encryption settings of rooms. Loads this information
+ * from the supplied crypto store when `init()` is called, and saves it to the
+ * crypto store whenever it is updated via `setRoomEncryption()`. Can supply
+ * full information about a room's encryption via `getRoomEncryption()`, or just
+ * answer whether or not a room has encryption via `isRoomEncrypted`.
+ */
 export class RoomList {
     // Object of roomId -> room e2e info object (body of the m.room.encryption event)
     private roomEncryption: Record<string, IRoomEncryption> = {};
@@ -43,7 +50,7 @@ export class RoomList {
         });
     }
 
-    public getRoomEncryption(roomId: string): IRoomEncryption {
+    public getRoomEncryption(roomId: string): IRoomEncryption | null {
         return this.roomEncryption[roomId] || null;
     }
 

src/crypto/SecretSharing.ts

@@ -38,7 +38,10 @@ interface ISecretRequestInternal {
 export class SecretSharing {
     private requests = new Map<string, ISecretRequestInternal>();
 
-    public constructor(private readonly baseApis: MatrixClient, private readonly cryptoCallbacks: ICryptoCallbacks) {}
+    public constructor(
+        private readonly baseApis: MatrixClient,
+        private readonly cryptoCallbacks: ICryptoCallbacks,
+    ) {}
 
     /**
      * Request a secret from another device

src/crypto/SecretStorage.ts

@@ -73,11 +73,7 @@ export class SecretStorage<B extends MatrixClient | undefined = MatrixClient> im
     /**
      * Add a key for encrypting secrets.
      */
-    public addKey(
-        algorithm: string,
-        opts: AddSecretStorageKeyOpts = {},
-        keyId?: string,
-    ): Promise<SecretStorageKeyObject> {
+    public addKey(algorithm: string, opts: AddSecretStorageKeyOpts, keyId?: string): Promise<SecretStorageKeyObject> {
         return this.storageImpl.addKey(algorithm, opts, keyId);
     }
 

src/crypto/algorithms/base.ts

@@ -210,7 +210,11 @@ export abstract class DecryptionAlgorithm {
 export class DecryptionError extends Error {
     public readonly detailedString: string;
 
-    public constructor(public readonly code: string, msg: string, details?: Record<string, string | Error>) {
+    public constructor(
+        public readonly code: string,
+        msg: string,
+        details?: Record<string, string | Error>,
+    ) {
         super(msg);
         this.code = code;
         this.name = "DecryptionError";
@@ -242,7 +246,11 @@ export class UnknownDeviceError extends Error {
      * @param msg - message describing the problem
      * @param devices - set of unknown devices per user we're warning about
      */
-    public constructor(msg: string, public readonly devices: DeviceInfoMap, public event?: MatrixEvent) {
+    public constructor(
+        msg: string,
+        public readonly devices: DeviceInfoMap,
+        public event?: MatrixEvent,
+    ) {
         super(msg);
         this.name = "UnknownDeviceError";
         this.devices = devices;

src/crypto/algorithms/megolm.ts

@@ -164,7 +164,10 @@ class OutboundSessionInfo {
      * @param sharedHistory - whether the session can be freely shared with
      *    other group members, according to the room history visibility settings
      */
-    public constructor(public readonly sessionId: string, public readonly sharedHistory = false) {
+    public constructor(
+        public readonly sessionId: string,
+        public readonly sharedHistory = false,
+    ) {
         this.creationTime = new Date().getTime();
     }
 

src/crypto/backup.ts

@@ -124,7 +124,10 @@ export class BackupManager {
     // The backup manager will schedule backup of keys when active (`scheduleKeyBackupSend`), this allows cancel when client is stopped
     private clientRunning = true;
 
-    public constructor(private readonly baseApis: MatrixClient, public readonly getKey: GetKey) {
+    public constructor(
+        private readonly baseApis: MatrixClient,
+        public readonly getKey: GetKey,
+    ) {
         this.checkedForBackup = false;
         this.sendingBackups = false;
     }
@@ -773,7 +776,10 @@ const UNSTABLE_MSC3270_NAME = new UnstableValue(
 export class Aes256 implements BackupAlgorithm {
     public static algorithmName = UNSTABLE_MSC3270_NAME.name;
 
-    public constructor(public readonly authData: IAes256AuthData, private readonly key: Uint8Array) {}
+    public constructor(
+        public readonly authData: IAes256AuthData,
+        private readonly key: Uint8Array,
+    ) {}
 
     public static async init(authData: IAes256AuthData, getKey: () => Promise<Uint8Array>): Promise<Aes256> {
         if (!authData) {