v31.3.0-rc.1

* Switch prepublishOnly to prepack to catch errors earlier

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix merge-release-notes.js parsing

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Improve release drafts and make release-drafter handle offcycle releases better

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Tweak release-drafter config

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Automate downstream dependency bumping

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove duplicated docs

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Delint

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

---------

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

.github/CODEOWNERS

@@ -6,3 +6,10 @@
 /src/matrixrtc               @matrix-org/element-call-reviewers
 /spec/*/webrtc            @matrix-org/element-call-reviewers
 /spec/*/matrixrtc            @matrix-org/element-call-reviewers
+
+/src/crypto               @matrix-org/element-crypto-web-reviewers
+/src/rust-crypto          @matrix-org/element-crypto-web-reviewers
+/spec/integ/crypto        @matrix-org/element-crypto-web-reviewers
+/spec/unit/crypto.spec.ts @matrix-org/element-crypto-web-reviewers
+/spec/unit/crypto         @matrix-org/element-crypto-web-reviewers
+/spec/unit/rust-crypto    @matrix-org/element-crypto-web-reviewers

.github/release-drafter.yml

@@ -22,10 +22,14 @@ version-resolver:
 exclude-labels:
     - "T-Task"
     - "X-Reverted"
+    - "backport staging"
 exclude-contributors:
     - "RiotRobot"
 template: |
     $CHANGES
+#no-changes-template: ""
 prerelease: true
 prerelease-identifier: rc
 include-pre-releases: false
+stable-ref: master
+staging-ref: staging

.github/workflows/docs-pr-netlify.yaml

@@ -11,13 +11,11 @@ jobs:
         if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
         runs-on: ubuntu-latest
         steps:
-            # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
-            # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
             - name: 📥 Download artifact
-              uses: dawidd6/action-download-artifact@f29d1b6a8930683e80acedfbe6baa2930cd646b4 # v2
+              uses: actions/download-artifact@v4
               with:
-                  workflow: static_analysis.yml
-                  run_id: ${{ github.event.workflow_run.id }}
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  run-id: ${{ github.event.workflow_run.id }}
                   name: docs
                   path: docs
 

.github/workflows/downstream-artifacts.yml

@@ -5,13 +5,9 @@ on:
 
     pull_request: {}
 
-    # For now at least, we don't run this or the cypress-tests against pushes
+    # For now at least, we don't run this or the downstream-end-to-end-tests against pushes
     # to develop or master.
     #
-    # Note that if we later choose to do so, we'll need to find a way to stop
-    # the results in Cypress Cloud from clobbering those from the 'develop'
-    # branch of matrix-react-sdk.
-    #
     #push:
     #    branches: [develop, master]
 concurrency:
@@ -20,7 +16,7 @@ concurrency:
 jobs:
     build-element-web:
         name: Build element-web
-        uses: matrix-org/matrix-react-sdk/.github/workflows/element-web.yaml@v3.86.0
+        uses: matrix-org/matrix-react-sdk/.github/workflows/element-web.yaml@v3.90.0
         with:
             matrix-js-sdk-sha: ${{ github.sha }}
             react-sdk-repository: matrix-org/matrix-react-sdk

.github/workflows/downstream-end-to-end-tests.yml

@@ -1,5 +1,5 @@
 # Triggers after the "Downstream artifacts" build has finished, to run the
-# matrix-react-sdk playwright & cypress tests (with access to repo secrets)
+# matrix-react-sdk playwright tests (with access to repo secrets)
 
 name: matrix-react-sdk End to End Tests
 on:
@@ -13,28 +13,6 @@ concurrency:
     cancel-in-progress: ${{ github.event.workflow_run.event == 'pull_request' }}
 
 jobs:
-    cypress:
-        name: Cypress
-
-        # We only want to run the cypress tests on merge queue to prevent regressions
-        # from creeping in. They take a long time to run and consume multiple concurrent runners.
-        if: github.event.workflow_run.event == 'merge_group'
-
-        uses: matrix-org/matrix-react-sdk/.github/workflows/cypress.yaml@develop
-        permissions:
-            actions: read
-            issues: read
-            statuses: write
-            pull-requests: read
-        secrets:
-            # secrets are not automatically shared with called workflows, so share the cypress dashboard key, and the Kiwi login details
-            KNAPSACK_PRO_TEST_SUITE_TOKEN_CYPRESS_RUST: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_CYPRESS_RUST}}
-            KNAPSACK_PRO_TEST_SUITE_TOKEN_CYPRESS_LEGACY: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_CYPRESS_LEGACY}}
-            TCMS_USERNAME: ${{ secrets.TCMS_USERNAME }}
-            TCMS_PASSWORD: ${{ secrets.TCMS_PASSWORD }}
-        with:
-            react-sdk-repository: matrix-org/matrix-react-sdk
-
     playwright:
         name: Playwright
         # We only want to run the playwright tests on merge queue to prevent regressions
@@ -49,15 +27,17 @@ jobs:
             deployments: write
         with:
             react-sdk-repository: matrix-org/matrix-react-sdk
+        secrets:
+            ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
 
-    # We want to make the cypress tests a required check for the merge queue.
+    # We want to make the Playwright tests a required check for the merge queue.
     #
-    # Unfortunately, github doesn't distinguish between "checks needed for branch
+    # Unfortunately, GitHub doesn't distinguish between "checks needed for branch
     # protection" (ie, the things that must pass before the PR will even be added
     # to the merge queue) and "checks needed in the merge queue". We just have to add
     # the check to the branch protection list.
     #
-    # Ergo, if we know we're not going to run the cypress tests, we need to add a
+    # Ergo, if we know we're not going to run the Playwright tests, we need to add a
     # passing status check manually.
     mark_skipped:
         if: github.event.workflow_run.event != 'merge_group'
@@ -65,18 +45,6 @@ jobs:
             statuses: write
         runs-on: ubuntu-latest
         steps:
-            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
-              with:
-                  authToken: "${{ secrets.GITHUB_TOKEN }}"
-                  state: success
-                  description: Cypress skipped
-
-                  # Keep in step with the `context` that is updated by `Sibz/github-status-action`
-                  # in matrix-org/matrix-react-sdk/.github/workflows/cypress.yaml.
-                  context: "${{ github.workflow }} / cypress"
-
-                  sha: "${{ github.event.workflow_run.head_sha }}"
-
             - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               with:
                   authToken: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/release-drafter-workflow.yml

@@ -0,0 +1,86 @@
+name: Release Drafter
+on:
+    workflow_call:
+        inputs:
+            include-changes:
+                description: Project to include changelog entries from in this release.
+                type: string
+                required: false
+concurrency: release-drafter-action
+jobs:
+    draft:
+        runs-on: ubuntu-latest
+        steps:
+            - name: 🧮 Checkout code
+              uses: actions/checkout@v4
+              with:
+                  ref: staging
+                  fetch-depth: 0
+
+            - uses: actions/setup-node@v4
+              with:
+                  node-version-file: package.json
+                  cache: "yarn"
+
+            - name: Install Deps
+              run: "yarn install --frozen-lockfile"
+
+            - uses: t3chguy/release-drafter@105e541c2c3d857f032bd522c0764694758fabad
+              id: draft-release
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                  disable-autolabeler: true
+
+            - name: Get actions scripts
+              uses: actions/checkout@v4
+              with:
+                  repository: matrix-org/matrix-js-sdk
+                  persist-credentials: false
+                  path: .action-repo
+                  sparse-checkout: |
+                      .github/actions
+                      scripts/release
+
+            - name: Ingest upstream changes
+              if: inputs.include-changes
+              uses: actions/github-script@v7
+              env:
+                  RELEASE_ID: ${{ steps.release.outputs.id }}
+                  DEPENDENCY: ${{ inputs.include-changes }}
+                  VERSION: ${{ steps.draft-release.outputs.tag_name }}
+              with:
+                  retries: 3
+                  script: |
+                      const { RELEASE_ID: releaseId, DEPENDENCY, VERSION } = process.env;
+                      const { owner, repo } = context.repo;
+                      const script = require("./.action-repo/scripts/release/merge-release-notes.js");
+
+                      let deps = [];
+                      if (DEPENDENCY.includes("/")) {
+                          deps.push(DEPENDENCY.replace("$VERSION", VERSION))
+                      } else {
+                          const fromVersion = JSON.parse(await exec.exec("git show origin/master:package.json")).dependencies[DEPENDENCY];
+                          const toVersion = require("./package.json").dependencies[DEPENDENCY];
+
+                          if (toVersion.endsWith("#develop")) {
+                              core.warning(`${DEPENDENCY} will be kept at ${fromVersion}`, { title: "Develop dependency found" });
+                          } else {
+                              deps.push([DEPENDENCY, fromVersion, toVersion]);
+                          }
+                      }
+
+                      if (deps.length) {
+                          const notes = await script({
+                              github,
+                              releaseId,
+                              dependencies: deps,
+                          });
+
+                          await github.rest.repos.updateRelease({
+                              owner,
+                              repo,
+                              release_id: releaseId,
+                              body: notes,
+                          });
+                      }

.github/workflows/release-drafter.yml

@@ -2,20 +2,8 @@ name: Release Drafter
 on:
     push:
         branches: [staging]
-    workflow_dispatch:
-        inputs:
-            previous-version:
-                description: What release to use as a base for release note purposes
-                required: false
-                type: string
+    workflow_dispatch: {}
 concurrency: ${{ github.workflow }}
 jobs:
     draft:
-        runs-on: ubuntu-latest
-        steps:
-            - uses: release-drafter/release-drafter@e64b19c4c46173209ed9f2e5a2f4ca7de89a0e86 # v5
-              env:
-                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-              with:
-                  disable-autolabeler: true
-                  previous-version: ${{ inputs.previous-version }}
+        uses: matrix-org/matrix-js-sdk/.github/workflows/release-drafter-workflow.yml@develop

.github/workflows/release-make.yml

@@ -20,10 +20,8 @@ on:
                 description: Publish to npm
                 type: boolean
                 default: false
-            dependencies:
-                description: |
-                    List of dependencies to update in `npm-dep=version` format.
-                    `version` can be `"current"` to leave it at the current version.
+            downstreams:
+                description: List of github projects (owner/repo) which should have their dependency bumped to the newly released version (in JSON string array string syntax)
                 type: string
                 required: false
             include-changes:
@@ -53,7 +51,7 @@ jobs:
             - name: Load GPG key
               id: gpg
               if: inputs.gpg-fingerprint
-              uses: crazy-max/ghaction-import-gpg@82a020f1f7f605c65dd2449b392a52c3fcfef7ef # v6
+              uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4 # v6
               with:
                   gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
                   passphrase: ${{ secrets.GPG_PASSPHRASE }}
@@ -88,17 +86,11 @@ jobs:
               id: prepare
               run: |
                   echo "VERSION=$VERSION" >> $GITHUB_ENV
-                  {
-                      echo "RELEASE_NOTES<<EOF"
-                      echo "$BODY"
-                      echo "EOF"
-                  } >> $GITHUB_ENV
 
                   HAS_DIST=0
                   jq -e .scripts.dist package.json >/dev/null 2>&1 && HAS_DIST=1
                   echo "has-dist-script=$HAS_DIST" >> $GITHUB_OUTPUT
               env:
-                  BODY: ${{ steps.release.outputs.body }}
                   VERSION: ${{ steps.release.outputs.tag_name }}
 
             - name: Finalise version
@@ -132,76 +124,23 @@ jobs:
             - name: Install dependencies
               run: "yarn install --frozen-lockfile"
 
-            - name: Update dependencies
-              id: update-dependencies
-              if: inputs.dependencies
-              run: |
-                  UPDATED=()
-                  while IFS= read -r DEPENDENCY; do
-                      [ -z "$DEPENDENCY" ] && continue
-                      IFS="=" read -r PACKAGE UPDATE_VERSION <<< "$DEPENDENCY"
-
-                      CURRENT_VERSION=$(cat package.json | jq -r .dependencies[\"$PACKAGE\"])
-                      echo "Current $PACKAGE version is $CURRENT_VERSION"
-
-                      if [ "$CURRENT_VERSION" == "null" ]
-                      then
-                          echo "Unable to find $PACKAGE in package.json"
-                          exit 1
-                      fi
-
-                      if [ "$UPDATE_VERSION" == "current" ] || [ "$UPDATE_VERSION" == "$CURRENT_VERSION" ]
-                      then
-                          echo "Not updating dependency $PACKAGE"
-                          continue
-                      fi
-
-                      echo "Upgrading $PACKAGE to $UPDATE_VERSION..."
-                      yarn upgrade "$PACKAGE@$UPDATE_VERSION" --exact
-                      git add -u
-                      git commit -m "Upgrade $PACKAGE to $UPDATE_VERSION"
-                      UPDATED+=("$PACKAGE")
-                  done <<< "$DEPENDENCIES"
-
-                  JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${UPDATED[@]}")
-                  echo "updated=$JSON" >> $GITHUB_OUTPUT
-              env:
-                  DEPENDENCIES: ${{ inputs.dependencies }}
-
-            - name: Prevent develop dependencies
-              if: inputs.dependencies
+            - name: Handle develop dependencies
               run: |
                   ret=0
-                  cat package.json | jq '.dependencies[]' | grep -q '#develop' || ret=$?
-                  if [ "$ret" -eq 0 ]; then
-                      echo "package.json contains develop dependencies. Refusing to release."
-                      exit
-                  fi
+                  cat package.json | jq -r '.dependencies | to_entries | .[] | "\(.key) \(.value)"' | grep '#develop$' | while read -r dep ; do
+                      IFS=" "
+                      PACKAGE=${dep[0]}
+                      VERSION=${dep[1]}
+
+                      echo "::warning title=Develop dependency found::$DEPENDENCY will be kept at $VERSION"
+                      yarn upgrade "$PACKAGE@$VERSION" --exact
+                      git add -u
+                      git commit -m "Keep $PACKAGE at $VERSION"
+                  done
 
             - name: Bump package.json version
               run: yarn version --no-git-tag-version --new-version "${VERSION#v}"
 
-            - name: Ingest upstream changes
-              if: |
-                  inputs.include-changes &&
-                  (!inputs.dependencies || contains(fromJSON(steps.update-dependencies.outputs.updated), inputs.include-changes))
-              uses: actions/github-script@v7
-              env:
-                  RELEASE_ID: ${{ steps.release.outputs.id }}
-                  DEPENDENCY: ${{ inputs.include-changes }}
-              with:
-                  retries: 3
-                  script: |
-                      const { RELEASE_ID: releaseId, DEPENDENCY, VERSION } = process.env;
-                      const { owner, repo } = context.repo;
-                      const script = require("./.action-repo/scripts/release/merge-release-notes.js");
-                      const notes = await script({
-                          github,
-                          releaseId,
-                          dependencies: [DEPENDENCY.replace("$VERSION", VERSION)],
-                      });
-                      core.exportVariable("RELEASE_NOTES", notes);
-
             - name: Add to CHANGELOG.md
               if: inputs.final
               run: |
@@ -219,6 +158,8 @@ jobs:
                   cat CHANGELOG.md.old >> CHANGELOG.md
                   rm CHANGELOG.md.old
                   git add CHANGELOG.md
+              env:
+                  RELEASE_NOTES: ${{ steps.release.outputs.body }}
 
             - name: Run pre-release script to update package.json fields
               run: |
@@ -335,15 +276,16 @@ jobs:
         secrets:
             NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
-    update-labels:
-        name: Advance release blocker labels
+    post-release:
+        name: Post release steps
         needs: release
         runs-on: ubuntu-latest
         steps:
             - id: repository
               run: echo "REPO=${GITHUB_REPOSITORY#*/}" >> $GITHUB_OUTPUT
 
-            - uses: garganshu/github-label-updater@3770d15ebfed2fe2cb06a241047bc340f774a7d1 # v1.0.0
+            - name: Advance release blocker labels
+              uses: garganshu/github-label-updater@3770d15ebfed2fe2cb06a241047bc340f774a7d1 # v1.0.0
               with:
                   owner: ${{ github.repository_owner }}
                   repo: ${{ steps.repository.outputs.REPO }}
@@ -351,3 +293,39 @@ jobs:
                   filter-labels: X-Upcoming-Release-Blocker
                   remove-labels: X-Upcoming-Release-Blocker
                   add-labels: X-Release-Blocker
+
+            - name: Wait for master->develop gitflow merge
+              if: inputs.final
+              uses: t3chguy/wait-on-check-action@18541021811b56544d90e0f073401c2b99e249d6 # fork
+              with:
+                  ref: master
+                  repo-token: ${{ secrets.GITHUB_TOKEN }}
+                  wait-interval: 10
+                  check-name: merge
+                  allowed-conclusions: success
+
+    bump-downstreams:
+        name: Update npm dependency in downstream projects
+        needs: npm
+        runs-on: ubuntu-latest
+        if: inputs.downstreams
+        strategy:
+            matrix:
+                repo: ${{ fromJSON(inputs.downstreams) }}
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  repository: ${{ matrix.repo }}
+                  ref: staging
+                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+
+            - name: Bump dependency
+              env:
+                  DEPENDENCY: ${{ needs.npm.outputs.id }}
+              run: |
+                  git config --global user.email "releases@riot.im"
+                  git config --global user.name "RiotRobot"
+                  yarn upgrade "$DEPENDENCY" --exact
+                  git add package.json yarn.lock
+                  git commit -am"Upgrade dependency to $DEPENDENCY"
+                  git push origin staging

.github/workflows/release-npm.yml

@@ -8,6 +8,8 @@ jobs:
     npm:
         name: Publish to npm
         runs-on: ubuntu-latest
+        outputs:
+            id: ${{ steps.npm-publish.outputs.id }}
         steps:
             - name: 🧮 Checkout code
               uses: actions/checkout@v4

.github/workflows/release.yml

@@ -28,6 +28,7 @@ jobs:
         with:
             final: ${{ inputs.mode == 'final' }}
             npm: ${{ inputs.npm }}
+            downstreams: '["matrix-org/matrix-react-sdk", "element-hq/element-web"]'
 
     docs:
         name: Publish Documentation

.github/workflows/sonarcloud.yml

@@ -5,6 +5,8 @@ on:
         secrets:
             SONAR_TOKEN:
                 required: true
+            ELEMENT_BOT_TOKEN:
+                required: true
         inputs:
             sharded:
                 type: boolean
@@ -15,45 +17,14 @@ on:
                 required: false
                 description: "Extra args to pass to SonarCloud"
 jobs:
-    # This is a workaround for https://github.com/SonarSource/SonarJS/issues/578
-    prepare:
-        name: Prepare
-        if: |
-            github.event.workflow_run.conclusion == 'success' &&
-            github.event.workflow_run.event != 'merge_group' &&
-            inputs.sharded
-        runs-on: ubuntu-latest
-        outputs:
-            extra_args: |
-                -Dsonar.javascript.lcov.reportPaths=${{ steps.extra_args.outputs.reportPaths }}
-                -Dsonar.testExecutionReportPaths=${{ steps.extra_args.outputs.testExecutionReportPaths }}
-        steps:
-            # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
-            # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
-            - name: 📥 Download artifact
-              uses: dawidd6/action-download-artifact@f29d1b6a8930683e80acedfbe6baa2930cd646b4 # v2
-              with:
-                  workflow: tests.yaml
-                  run_id: ${{ github.event.workflow_run.id }}
-                  name: coverage
-                  path: coverage
-
-            - id: extra_args
-              run: |
-                  coverage=$(find coverage -type f -name '*lcov.info' | tr '\n' ',' | sed 's/,$//g')
-                  echo "reportPaths=$coverage" >> $GITHUB_OUTPUT
-                  reports=$(find coverage -type f -name 'jest-sonar-report*.xml' | tr '\n' ',' | sed 's/,$//g')
-                  echo "testExecutionReportPaths=$reports" >> $GITHUB_OUTPUT
-
     sonarqube:
         runs-on: ubuntu-latest
-        needs: prepare
         if: |
-            always() && !failure() && !cancelled() &&
-            github.event.workflow_run.conclusion == 'success'
+            github.event.workflow_run.conclusion == 'success' &&
+            github.event.workflow_run.event != 'merge_group'
         steps:
             # We create the status here and then update it to success/failure in the `report` stage
-            # This provides an easy link to this workflow_run from the PR before Cypress is done.
+            # This provides an easy link to this workflow_run from the PR before Sonarcloud is done.
             - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               with:
                   authToken: ${{ secrets.GITHUB_TOKEN }}
@@ -62,22 +33,55 @@ jobs:
                   sha: ${{ github.event.workflow_run.head_sha }}
                   target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
 
+            - name: "🧮 Checkout code"
+              uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+              with:
+                  repository: ${{ github.event.workflow_run.head_repository.full_name }}
+                  ref: ${{ github.event.workflow_run.head_branch }} # checkout commit that triggered this workflow
+                  fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
+
+            - name: 📥 Download artifact
+              uses: actions/download-artifact@v4
+              if: ${{ !inputs.sharded }}
+              with:
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  run-id: ${{ github.event.workflow_run.id }}
+                  name: coverage
+                  path: coverage
+            - name: 📥 Download sharded artifacts
+              uses: actions/download-artifact@v4
+              if: inputs.sharded
+              with:
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  run-id: ${{ github.event.workflow_run.id }}
+                  pattern: coverage-*
+                  path: coverage
+                  merge-multiple: true
+
+            - id: extra_args
+              run: |
+                  coverage=$(find coverage -type f -name '*lcov.info' | tr '\n' ',' | sed 's/,$//g')
+                  echo "reportPaths=$coverage" >> $GITHUB_OUTPUT
+                  reports=$(find coverage -type f -name 'jest-sonar-report*.xml' | tr '\n' ',' | sed 's/,$//g')
+                  echo "testExecutionReportPaths=$reports" >> $GITHUB_OUTPUT
+
             - name: "🩻 SonarCloud Scan"
               id: sonarcloud
               uses: matrix-org/sonarcloud-workflow-action@v2.7
               # workflow_run fails report against the develop commit always, we don't want that for PRs
               continue-on-error: ${{ github.event.workflow_run.head_branch != 'develop' }}
               with:
+                  skip_checkout: true
                   repository: ${{ github.event.workflow_run.head_repository.full_name }}
                   is_pr: ${{ github.event.workflow_run.event == 'pull_request' }}
                   version_cmd: "cat package.json | jq -r .version"
                   branch: ${{ github.event.workflow_run.head_branch }}
                   revision: ${{ github.event.workflow_run.head_sha }}
                   token: ${{ secrets.SONAR_TOKEN }}
-                  coverage_run_id: ${{ github.event.workflow_run.id }}
-                  coverage_workflow_name: tests.yml
-                  coverage_extract_path: coverage
-                  extra_args: ${{ inputs.extra_args }} ${{ needs.prepare.outputs.extra_args }}
+                  extra_args: |
+                      ${{ inputs.extra_args }}
+                      -Dsonar.javascript.lcov.reportPaths=${{ steps.extra_args.outputs.reportPaths }}
+                      -Dsonar.testExecutionReportPaths=${{ steps.extra_args.outputs.testExecutionReportPaths }}
 
             - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               if: always()

.github/workflows/sonarqube.yml

@@ -14,5 +14,6 @@ jobs:
         uses: matrix-org/matrix-js-sdk/.github/workflows/sonarcloud.yml@develop
         secrets:
             SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+            ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
         with:
             sharded: true

.github/workflows/static_analysis.yml

@@ -91,7 +91,7 @@ jobs:
                   find _docs -mindepth 1 -maxdepth 1 -type d -execdir mv {} stable \; -quit
 
             - name: Upload Artifact
-              uses: actions/upload-artifact@v3
+              uses: actions/upload-artifact@v4
               with:
                   name: docs
                   path: _docs

.github/workflows/tests.yml

@@ -56,9 +56,9 @@ jobs:
 
             - name: Upload Artifact
               if: env.ENABLE_COVERAGE == 'true'
-              uses: actions/upload-artifact@v3
+              uses: actions/upload-artifact@v4
               with:
-                  name: coverage
+                  name: coverage-${{ matrix.specs }}-${{ matrix.node == 'lts/*' && 'lts' || matrix.node }}
                   path: |
                       coverage
                       !coverage/lcov-report

.prettierignore

@@ -25,5 +25,6 @@ out
 # This file is owned, parsed, and generated by allchange, which doesn't comply with prettier
 /CHANGELOG.md
 
-# This file is also autogenerated
+# These files are also autogenerated
 /spec/test-utils/test-data/index.ts
+/spec/test-utils/test_indexeddb_cryptostore_dump/dump.json

CHANGELOG.md

@@ -1,3 +1,24 @@
+Changes in [31.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.2.0) (2024-01-31)
+==================================================================================================
+## ✨ Features
+
+* Emit events during migration from libolm ([#3982](https://github.com/matrix-org/matrix-js-sdk/pull/3982)). Contributed by @richvdh.
+* Support for migration from from libolm ([#3978](https://github.com/matrix-org/matrix-js-sdk/pull/3978)). Contributed by @richvdh.
+
+## 🐛 Bug Fixes
+
+* ElementR | backup: call expensive `roomKeyCounts` less often ([#4015](https://github.com/matrix-org/matrix-js-sdk/pull/4015)). Contributed by @BillCarsonFr.
+* Decrypt and Import full backups in chunk with progress ([#4005](https://github.com/matrix-org/matrix-js-sdk/pull/4005)). Contributed by @BillCarsonFr.
+* Fix new threads not appearing. ([#4009](https://github.com/matrix-org/matrix-js-sdk/pull/4009)). Contributed by @dbkr.
+
+
+Changes in [31.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.1.0) (2024-01-19)
+==================================================================================================
+## ✨ Features
+
+* Broaden spec version support ([#4016](https://github.com/matrix-org/matrix-js-sdk/pull/4016)). Contributed by @RiotRobot.
+
+
 Changes in [31.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.0.0) (2024-01-16)
 ==================================================================================================
 ## 🚨 BREAKING CHANGES
@@ -69,6 +90,7 @@ Changes in [30.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30
 
 ## 🚨 BREAKING CHANGES
  * Refactor & make base64 functions browser-safe ([\#3818](https://github.com/matrix-org/matrix-js-sdk/pull/3818)).
+ * `IndexedDBStore.startup()` must be called after using it on `sdk.createClient` now.
 
 ## 🦖 Deprecations
  * Deprecate `MatrixEvent.toJSON` ([\#3801](https://github.com/matrix-org/matrix-js-sdk/pull/3801)).

README.md

@@ -11,7 +11,7 @@
 This is the [Matrix](https://matrix.org) Client-Server SDK for JavaScript and TypeScript. This SDK can be run in a
 browser or in Node.js.
 
-#### Minimum Matrix server version: v1.5
+#### Minimum Matrix server version: v1.1
 
 The Matrix specification is constantly evolving - while this SDK aims for maximum backwards compatibility, it only
 guarantees that a feature will be supported for at least 4 spec releases. For example, if a feature the js-sdk supports

docs/SUMMARY.md

@@ -4,6 +4,5 @@
 
 # Deep dive
 
--   [Release Process](release.md)
 -   [Storage notes](storage-notes.md)
 -   [Unverified devices](warning-on-unverified-devices.md)

docs/release.md

@@ -1,24 +0,0 @@
-# Release Process
-
-## Hotfix and off-cycle releases
-
-1. Prepare the `staging` branch by using the backport automation and manually merging
-2. Go to [Releasing](#Releasing)
-
-## Release candidates
-
-1. Prepare the `staging` branch by running the [branch cut automation](https://github.com/vector-im/element-web/actions/workflows/release_prepare.yml)
-2. Go to [Releasing](#Releasing)
-
-## Releasing
-
-1. Open the [Releases page](https://github.com/matrix-org/matrix-js-sdk/releases) and inspect the draft release there
-2. Make any modifications to the release notes and tag/version as required
-3. Run [workflow](https://github.com/matrix-org/matrix-js-sdk/actions/workflows/release.yml) with the type set appropriately
-
-## Artifacts
-
-Releasing the Matrix JS SDK has just two artifacts:
-
--   Package published to [npm](https://github.com/matrix-org/matrix-js-sdk)
--   Docs published to [Github Pages](https://matrix-org.github.io/matrix-js-sdk/)

package.json

@@ -1,12 +1,12 @@
 {
     "name": "matrix-js-sdk",
-    "version": "31.0.0",
+    "version": "31.3.0-rc.1",
     "description": "Matrix Client-Server SDK for Javascript",
     "engines": {
         "node": ">=18.0.0"
     },
     "scripts": {
-        "prepublishOnly": "yarn build",
+        "prepack": "yarn build",
         "start": "echo THIS IS FOR LEGACY PURPOSES ONLY. && babel src -w -s -d lib --verbose --extensions \".ts,.js\"",
         "clean": "rimraf lib",
         "build": "yarn build:dev",
@@ -52,7 +52,7 @@
     ],
     "dependencies": {
         "@babel/runtime": "^7.12.5",
-        "@matrix-org/matrix-sdk-crypto-wasm": "^3.5.0",
+        "@matrix-org/matrix-sdk-crypto-wasm": "^4.3.0",
         "another-json": "^0.2.0",
         "bs58": "^5.0.0",
         "content-type": "^1.0.4",
@@ -93,20 +93,20 @@
         "@types/uuid": "9",
         "@typescript-eslint/eslint-plugin": "^6.0.0",
         "@typescript-eslint/parser": "^6.0.0",
-        "allchange": "^1.0.6",
+        "allchange": "^1.3.0",
         "babel-jest": "^29.0.0",
         "debug": "^4.3.4",
         "domexception": "^4.0.0",
-        "eslint": "8.55.0",
+        "eslint": "8.56.0",
         "eslint-config-google": "^0.14.0",
         "eslint-config-prettier": "^9.0.0",
         "eslint-import-resolver-typescript": "^3.5.1",
         "eslint-plugin-import": "^2.26.0",
         "eslint-plugin-jest": "^27.1.6",
-        "eslint-plugin-jsdoc": "^46.0.0",
+        "eslint-plugin-jsdoc": "^48.0.0",
         "eslint-plugin-matrix-org": "^1.0.0",
         "eslint-plugin-tsdoc": "^0.2.17",
-        "eslint-plugin-unicorn": "^49.0.0",
+        "eslint-plugin-unicorn": "^50.0.0",
         "fake-indexeddb": "^5.0.2",
         "fetch-mock": "9.11.0",
         "fetch-mock-jest": "^1.5.1",
@@ -117,7 +117,7 @@
         "jest-mock": "^29.0.0",
         "lint-staged": "^15.0.2",
         "matrix-mock-request": "^2.5.0",
-        "prettier": "3.1.1",
+        "prettier": "3.2.4",
         "rimraf": "^5.0.0",
         "ts-node": "^10.9.1",
         "typedoc": "^0.24.0",

scripts/release/merge-release-notes.js

@@ -2,6 +2,31 @@
 
 const fs = require("fs");
 
+// Dependency can be the name of an entry in package.json, in which case the owner, repo & version will be looked up in its own package.json
+// Or it can be a string in the form owner/repo@tag
+// Or it can be a tuple of dependency, from version, to version, in which case a list of releases in that range (to inclusive) will be returned
+async function getReleases(github, dependency) {
+    if (Array.isArray(dependency)) {
+        const [dep, fromVersion, toVersion] = dependency;
+        const upstreamPackageJson = getDependencyPackageJson(dep);
+        const [owner, repo] = upstreamPackageJson.repository.url.split("/").slice(-2);
+
+        const response = await github.rest.repos.listReleases({
+            owner,
+            repo,
+            per_page: 100,
+        });
+        const releases = response.data.filter((release) => !release.draft && !release.prerelease);
+
+        const fromVersionIndex = releases.findIndex((release) => release.tag_name === `v${fromVersion}`);
+        const toVersionIndex = releases.findIndex((release) => release.tag_name === `v${toVersion}`);
+
+        return releases.slice(toVersionIndex, fromVersionIndex);
+    }
+
+    return [await getRelease(github, dependency)];
+}
+
 async function getRelease(github, dependency) {
     let owner;
     let repo;
@@ -11,7 +36,7 @@ async function getRelease(github, dependency) {
         repo = dependency.split("/")[1].split("@")[0];
         tag = dependency.split("@")[1];
     } else {
-        const upstreamPackageJson = JSON.parse(fs.readFileSync(`./node_modules/${dependency}/package.json`, "utf8"));
+        const upstreamPackageJson = getDependencyPackageJson(dependency);
         [owner, repo] = upstreamPackageJson.repository.url.split("/").slice(-2);
         tag = `v${upstreamPackageJson.version}`;
     }
@@ -24,25 +49,45 @@ async function getRelease(github, dependency) {
     return response.data;
 }
 
+function getDependencyPackageJson(dependency) {
+    return JSON.parse(fs.readFileSync(`./node_modules/${dependency}/package.json`, "utf8"));
+}
+
 const HEADING_PREFIX = "## ";
 
+const categories = [
+    "🔒 SECURITY FIXES",
+    "🚨 BREAKING CHANGESd",
+    "🦖 Deprecations",
+    "✨ Features",
+    "🐛 Bug Fixes",
+    "🧰 Maintenance",
+];
+
+const parseReleaseNotes = (body, sections) => {
+    let heading = null;
+    for (const line of body.split("\n")) {
+        const trimmed = line.trim();
+        if (trimmed.startsWith(HEADING_PREFIX)) {
+            heading = trimmed.slice(HEADING_PREFIX.length);
+            if (!categories.includes(heading)) heading = null;
+            continue;
+        }
+        if (heading && trimmed) {
+            sections[heading].push(trimmed);
+        }
+    }
+};
+
 const main = async ({ github, releaseId, dependencies }) => {
     const { GITHUB_REPOSITORY } = process.env;
     const [owner, repo] = GITHUB_REPOSITORY.split("/");
 
-    const sections = new Map();
-    let heading = null;
+    const sections = Object.fromEntries(categories.map((cat) => [cat, []]));
     for (const dependency of dependencies) {
-        const release = await getRelease(github, dependency);
-        for (const line of release.body.split("\n")) {
-            if (line.startsWith(HEADING_PREFIX)) {
-                heading = line.trim();
-                sections.set(heading, []);
-                continue;
-            }
-            if (heading && line) {
-                sections.get(heading).push(line.trim());
-            }
+        const releases = await getReleases(github, dependency);
+        for (const release of releases) {
+            parseReleaseNotes(release.body, sections);
         }
     }
 
@@ -52,36 +97,22 @@ const main = async ({ github, releaseId, dependencies }) => {
         release_id: releaseId,
     });
 
-    const headings = ["🚨 BREAKING CHANGES", "🦖 Deprecations", "✨ Features", "🐛 Bug Fixes", "🧰 Maintenance"].map(
-        (h) => HEADING_PREFIX + h,
-    );
+    const intro = release.body.split(HEADING_PREFIX, 2)[0].trim();
 
-    heading = null;
-    const output = [];
-    for (const line of [...release.body.split("\n"), null]) {
-        if (line === null || line.startsWith(HEADING_PREFIX)) {
-            // If we have a heading, and it's not the first in the list of pending headings, output the section.
-            // If we're processing the last line (null) then output all remaining sections.
-            while (headings.length > 0 && (line === null || (heading && headings[0] !== heading))) {
-                const heading = headings.shift();
-                if (sections.has(heading)) {
-                    output.push(heading);
-                    output.push(...sections.get(heading));
-                }
-            }
-
-            if (heading && sections.has(heading)) {
-                const lastIsBlank = !output.at(-1)?.trim();
-                if (lastIsBlank) output.pop();
-                output.push(...sections.get(heading));
-                if (lastIsBlank) output.push("");
-            }
-            heading = line;
-        }
-        output.push(line);
+    let output = "";
+    if (intro) {
+        output = intro + "\n\n";
     }
 
-    return output.join("\n");
+    for (const section in sections) {
+        const lines = sections[section];
+        if (!lines.length) continue;
+        output += HEADING_PREFIX + section + "\n\n";
+        output += lines.join("\n");
+        output += "\n\n";
+    }
+
+    return output;
 };
 
 // This is just for testing locally

spec/integ/crypto/cross-signing.spec.ts

@@ -398,7 +398,8 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
 
     describe("crossSignDevice", () => {
         beforeEach(async () => {
-            jest.useFakeTimers();
+            // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
             // make sure that there is another device which we can sign
             e2eKeyResponder.addDeviceKeys(SIGNED_TEST_DEVICE_DATA);

spec/integ/crypto/crypto.spec.ts

@@ -29,6 +29,7 @@ import {
     getSyncResponse,
     InitCrypto,
     mkEventCustom,
+    mkMembershipCustom,
     syncPromise,
 } from "../../test-utils/test-utils";
 import * as testData from "../../test-utils/test-data";
@@ -38,6 +39,7 @@ import {
     BOB_TEST_USER_ID,
     SIGNED_CROSS_SIGNING_KEYS_DATA,
     SIGNED_TEST_DEVICE_DATA,
+    TEST_ROOM_ID,
     TEST_ROOM_ID as ROOM_ID,
     TEST_USER_ID,
 } from "../../test-utils/test-data";
@@ -230,9 +232,6 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
     /** an object which intercepts `/keys/upload` requests from {@link #aliceClient} to catch the uploaded keys */
     let keyReceiver: E2EKeyReceiver;
 
-    /** an object which intercepts `/keys/query` requests on the test homeserver */
-    let keyResponder: E2EKeyResponder;
-
     /** an object which intercepts `/sync` requests from {@link #aliceClient} */
     let syncResponder: ISyncResponder;
 
@@ -368,6 +367,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 accessToken: "akjgkrgjs",
                 deviceId: "xzcvb",
                 cryptoCallbacks: createCryptoCallbacks(),
+                logger: logger.getChild("aliceClient"),
             });
 
             /* set up listeners for /keys/upload and /sync */
@@ -701,7 +701,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
     it("prepareToEncrypt", async () => {
         const homeserverUrl = aliceClient.getHomeserverUrl();
-        keyResponder = new E2EKeyResponder(homeserverUrl);
+        const keyResponder = new E2EKeyResponder(homeserverUrl);
         keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
 
         const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
@@ -732,7 +732,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
     it("Alice sends a megolm message with GlobalErrorOnUnknownDevices=false", async () => {
         aliceClient.setGlobalErrorOnUnknownDevices(false);
         const homeserverUrl = aliceClient.getHomeserverUrl();
-        keyResponder = new E2EKeyResponder(homeserverUrl);
+        const keyResponder = new E2EKeyResponder(homeserverUrl);
         keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
 
         const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
@@ -760,7 +760,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
     it("We should start a new megolm session after forceDiscardSession", async () => {
         aliceClient.setGlobalErrorOnUnknownDevices(false);
         const homeserverUrl = aliceClient.getHomeserverUrl();
-        keyResponder = new E2EKeyResponder(homeserverUrl);
+        const keyResponder = new E2EKeyResponder(homeserverUrl);
         keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
 
         const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
@@ -1063,8 +1063,9 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             await startClientAndAwaitFirstSync();
             const p2pSession = await establishOlmSession(aliceClient, keyReceiver, syncResponder, testOlmAccount);
 
-            // We need to fake the timers to advance the time
-            jest.useFakeTimers();
+            // We need to fake the timers to advance the time, but the wasm bindings of matrix-sdk-crypto rely on a
+            // working `queueMicrotask`
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
             const syncResponse = getSyncResponse(["@bob:xyz"]);
 
@@ -2069,7 +2070,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
         it("Sending an event initiates a member list sync", async () => {
             const homeserverUrl = aliceClient.getHomeserverUrl();
-            keyResponder = new E2EKeyResponder(homeserverUrl);
+            const keyResponder = new E2EKeyResponder(homeserverUrl);
             keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
 
             const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
@@ -2092,7 +2093,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
         it("loading the membership list inhibits a later load", async () => {
             const homeserverUrl = aliceClient.getHomeserverUrl();
-            keyResponder = new E2EKeyResponder(homeserverUrl);
+            const keyResponder = new E2EKeyResponder(homeserverUrl);
             keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
 
             const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
@@ -2189,7 +2190,8 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
     describe("key upload request", () => {
         beforeEach(() => {
-            jest.useFakeTimers();
+            // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
         });
 
         afterEach(() => {
@@ -2389,8 +2391,9 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             expect(devicesInfo.get(user)?.size).toBeFalsy();
         });
 
-        it("Get devices from tacked users", async () => {
-            jest.useFakeTimers();
+        it("Get devices from tracked users", async () => {
+            // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
             expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
             await startClientAndAwaitFirstSync();
@@ -2745,7 +2748,8 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
         describe("Manage Key Backup", () => {
             beforeEach(async () => {
-                jest.useFakeTimers();
+                // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+                jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
             });
 
             afterEach(() => {
@@ -2899,7 +2903,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             // anything that we don't have a specific matcher for silently returns a 404
             fetchMock.catch(404);
 
-            keyResponder = new E2EKeyResponder(aliceClient.getHomeserverUrl());
+            const keyResponder = new E2EKeyResponder(aliceClient.getHomeserverUrl());
             keyResponder.addCrossSigningData(SIGNED_CROSS_SIGNING_KEYS_DATA);
             keyResponder.addDeviceKeys(SIGNED_TEST_DEVICE_DATA);
             keyResponder.addKeyReceiver(BOB_TEST_USER_ID, keyReceiver);
@@ -2935,4 +2939,180 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             expect(hasCrossSigningKeysForUser).toBe(false);
         });
     });
+
+    /** Guards against downgrade attacks from servers hiding or manipulating the crypto settings. */
+    describe("Persistent encryption settings", () => {
+        let persistentStoreClient: MatrixClient;
+        let client2: MatrixClient;
+
+        beforeEach(async () => {
+            const homeserverurl = "https://alice-server.com";
+            const userId = "@alice:localhost";
+
+            const keyResponder = new E2EKeyResponder(homeserverurl);
+            keyResponder.addKeyReceiver(userId, keyReceiver);
+
+            // For legacy crypto, these tests only work properly with a proper (indexeddb-based) CryptoStore, so
+            // rather than using the existing `aliceClient`, create a new client. Once we drop legacy crypto, we can
+            // just use `aliceClient` here.
+            persistentStoreClient = await makeNewClient(homeserverurl, userId, "persistentStoreClient");
+            await persistentStoreClient.startClient({});
+        });
+
+        afterEach(async () => {
+            persistentStoreClient.stopClient();
+            client2?.stopClient();
+        });
+
+        test("Sending a message in a room where the server is hiding the state event does not send a plaintext event", async () => {
+            // Alice is in an encrypted room
+            const encryptionState = mkEncryptionEvent({ algorithm: "m.megolm.v1.aes-sha2" });
+            syncResponder.sendOrQueueSyncResponse(getSyncResponseWithState([encryptionState]));
+            await syncPromise(persistentStoreClient);
+
+            // Send a message, and expect to get an `m.room.encrypted` event.
+            await Promise.all([persistentStoreClient.sendTextMessage(ROOM_ID, "test"), expectEncryptedSendMessage()]);
+
+            // We now replace the client, and allow the new one to resync, *without* the encryption event.
+            client2 = await replaceClient(persistentStoreClient);
+            syncResponder.sendOrQueueSyncResponse(getSyncResponseWithState([]));
+            await client2.startClient({});
+            await syncPromise(client2);
+            logger.log(client2.getUserId() + ": restarted");
+
+            await expectSendMessageToFail(client2);
+        });
+
+        test("Changes to the rotation period should be ignored", async () => {
+            // Alice is in an encrypted room, where the rotation period is set to 2 messages
+            const encryptionState = mkEncryptionEvent({ algorithm: "m.megolm.v1.aes-sha2", rotation_period_msgs: 2 });
+            syncResponder.sendOrQueueSyncResponse(getSyncResponseWithState([encryptionState]));
+            await syncPromise(persistentStoreClient);
+
+            // Send a message, and expect to get an `m.room.encrypted` event.
+            const [, msg1Content] = await Promise.all([
+                persistentStoreClient.sendTextMessage(ROOM_ID, "test1"),
+                expectEncryptedSendMessage(),
+            ]);
+
+            // Replace the state with one which bumps the rotation period. This should be ignored, though it's not
+            // clear that is correct behaviour (see https://github.com/element-hq/element-meta/issues/69)
+            const encryptionState2 = mkEncryptionEvent({
+                algorithm: "m.megolm.v1.aes-sha2",
+                rotation_period_msgs: 100,
+            });
+            syncResponder.sendOrQueueSyncResponse({
+                next_batch: "1",
+                rooms: { join: { [TEST_ROOM_ID]: { timeline: { events: [encryptionState2], prev_batch: "" } } } },
+            });
+            await syncPromise(persistentStoreClient);
+
+            // Send two more messages. The first should use the same megolm session as the first; the second should
+            // use a different one.
+            const [, msg2Content] = await Promise.all([
+                persistentStoreClient.sendTextMessage(ROOM_ID, "test2"),
+                expectEncryptedSendMessage(),
+            ]);
+            expect(msg2Content.session_id).toEqual(msg1Content.session_id);
+            const [, msg3Content] = await Promise.all([
+                persistentStoreClient.sendTextMessage(ROOM_ID, "test3"),
+                expectEncryptedSendMessage(),
+            ]);
+            expect(msg3Content.session_id).not.toEqual(msg1Content.session_id);
+        });
+
+        test("Changes to the rotation period should be ignored after a client restart", async () => {
+            // Alice is in an encrypted room, where the rotation period is set to 2 messages
+            const encryptionState = mkEncryptionEvent({ algorithm: "m.megolm.v1.aes-sha2", rotation_period_msgs: 2 });
+            syncResponder.sendOrQueueSyncResponse(getSyncResponseWithState([encryptionState]));
+            await syncPromise(persistentStoreClient);
+
+            // Send a message, and expect to get an `m.room.encrypted` event.
+            await Promise.all([persistentStoreClient.sendTextMessage(ROOM_ID, "test1"), expectEncryptedSendMessage()]);
+
+            // We now replace the client, and allow the new one to resync with a *different* encryption event.
+            client2 = await replaceClient(persistentStoreClient);
+            const encryptionState2 = mkEncryptionEvent({
+                algorithm: "m.megolm.v1.aes-sha2",
+                rotation_period_msgs: 100,
+            });
+            syncResponder.sendOrQueueSyncResponse(getSyncResponseWithState([encryptionState2]));
+            await client2.startClient({});
+            await syncPromise(client2);
+            logger.log(client2.getUserId() + ": restarted");
+
+            // Now send another message, which should (for now) be rejected.
+            await expectSendMessageToFail(client2);
+        });
+
+        /** Shut down `oldClient`, and build a new MatrixClient for the same user. */
+        async function replaceClient(oldClient: MatrixClient) {
+            oldClient.stopClient();
+            syncResponder.sendOrQueueSyncResponse({}); // flush pending request from old client
+            return makeNewClient(oldClient.getHomeserverUrl(), oldClient.getSafeUserId(), "client2");
+        }
+
+        async function makeNewClient(
+            homeserverUrl: string,
+            userId: string,
+            loggerPrefix: string,
+        ): Promise<MatrixClient> {
+            const client = createClient({
+                baseUrl: homeserverUrl,
+                userId: userId,
+                accessToken: "akjgkrgjs",
+                deviceId: "xzcvb",
+                cryptoCallbacks: createCryptoCallbacks(),
+                logger: logger.getChild(loggerPrefix),
+
+                // For legacy crypto, these tests only work with a proper persistent cryptoStore.
+                cryptoStore: new IndexedDBCryptoStore(indexedDB, "test"),
+            });
+            await initCrypto(client);
+            mockInitialApiRequests(client.getHomeserverUrl());
+            return client;
+        }
+
+        function mkEncryptionEvent(content: Object) {
+            return mkEventCustom({
+                sender: persistentStoreClient.getSafeUserId(),
+                type: "m.room.encryption",
+                state_key: "",
+                content: content,
+            });
+        }
+
+        /** Sync response which includes `TEST_ROOM_ID`, where alice is a member
+         *
+         * @param stateEvents - Additional state events for the test room
+         */
+        function getSyncResponseWithState(stateEvents: Array<Object>) {
+            const roomResponse = {
+                state: {
+                    events: [
+                        mkMembershipCustom({ membership: "join", sender: persistentStoreClient.getSafeUserId() }),
+                        ...stateEvents,
+                    ],
+                },
+                timeline: {
+                    events: [],
+                    prev_batch: "",
+                },
+            };
+
+            return {
+                next_batch: "1",
+                rooms: { join: { [TEST_ROOM_ID]: roomResponse } },
+            };
+        }
+
+        /** Send a message with the given client, and check that it is not sent in plaintext */
+        async function expectSendMessageToFail(aliceClient2: MatrixClient) {
+            // The precise failure mode here is somewhat up for debate (https://github.com/element-hq/element-meta/issues/69).
+            // For now, the attempt to send is rejected with an exception. The text is different between old and new stacks.
+            await expect(aliceClient2.sendTextMessage(ROOM_ID, "test")).rejects.toThrow(
+                /unconfigured room !room:id|Room !room:id was previously configured to use encryption/,
+            );
+        }
+    });
 });

spec/integ/crypto/megolm-backup.spec.ts

@@ -17,8 +17,18 @@ limitations under the License.
 import fetchMock from "fetch-mock-jest";
 import "fake-indexeddb/auto";
 import { IDBFactory } from "fake-indexeddb";
+import { Mocked } from "jest-mock";
 
-import { createClient, CryptoEvent, ICreateClientOpts, IEvent, MatrixClient, TypedEventEmitter } from "../../../src";
+import {
+    createClient,
+    CryptoApi,
+    CryptoEvent,
+    ICreateClientOpts,
+    IEvent,
+    IMegolmSessionData,
+    MatrixClient,
+    TypedEventEmitter,
+} from "../../../src";
 import { SyncResponder } from "../../test-utils/SyncResponder";
 import { E2EKeyReceiver } from "../../test-utils/E2EKeyReceiver";
 import { E2EKeyResponder } from "../../test-utils/E2EKeyResponder";
@@ -31,7 +41,7 @@ import {
     syncPromise,
 } from "../../test-utils/test-utils";
 import * as testData from "../../test-utils/test-data";
-import { KeyBackupInfo } from "../../../src/crypto-api/keybackup";
+import { KeyBackupInfo, KeyBackupSession } from "../../../src/crypto-api/keybackup";
 import { IKeyBackup } from "../../../src/crypto/backup";
 import { flushPromises } from "../../test-utils/flushPromises";
 import { defer, IDeferred } from "../../../src/utils";
@@ -119,7 +129,8 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
     let e2eKeyResponder: E2EKeyResponder;
 
     beforeEach(async () => {
-        jest.useFakeTimers();
+        // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+        jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
         // anything that we don't have a specific matcher for silently returns a 404
         fetchMock.catch(404);
@@ -286,17 +297,21 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
     });
 
     describe("recover from backup", () => {
-        it("can restore from backup (Curve25519 version)", async function () {
+        let aliceCrypto: CryptoApi;
+
+        beforeEach(async () => {
             fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
 
             aliceClient = await initTestClient();
-            const aliceCrypto = aliceClient.getCrypto()!;
+            aliceCrypto = aliceClient.getCrypto()!;
             await aliceClient.startClient();
 
             // tell Alice to trust the dummy device that signed the backup
             await waitForDeviceList();
             await aliceCrypto.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
+        });
 
+        it("can restore from backup (Curve25519 version)", async function () {
             const fullBackup = {
                 rooms: {
                     [ROOM_ID]: {
@@ -340,17 +355,179 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
             expect(afterCache.imported).toStrictEqual(1);
         });
 
+        /**
+         * Creates a mock backup response of a GET `room_keys/keys` with a given number of keys per room.
+         * @param keysPerRoom The number of keys per room
+         */
+        function createBackupDownloadResponse(keysPerRoom: number[]) {
+            const response: {
+                rooms: {
+                    [roomId: string]: {
+                        sessions: {
+                            [sessionId: string]: KeyBackupSession;
+                        };
+                    };
+                };
+            } = { rooms: {} };
+
+            const expectedTotal = keysPerRoom.reduce((a, b) => a + b, 0);
+            for (let i = 0; i < keysPerRoom.length; i++) {
+                const roomId = `!room${i}:example.com`;
+                response.rooms[roomId] = { sessions: {} };
+                for (let j = 0; j < keysPerRoom[i]; j++) {
+                    const sessionId = `session${j}`;
+                    // Put the same fake session data, not important for that test
+                    response.rooms[roomId].sessions[sessionId] = testData.CURVE25519_KEY_BACKUP_DATA;
+                }
+            }
+            return { response, expectedTotal };
+        }
+
+        it("Should import full backup in chunks", async function () {
+            const importMockImpl = jest.fn();
+            // @ts-ignore - mock a private method for testing purpose
+            aliceCrypto.importBackedUpRoomKeys = importMockImpl;
+
+            // We need several rooms with several sessions to test chunking
+            const { response, expectedTotal } = createBackupDownloadResponse([45, 300, 345, 12, 130]);
+
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys", response);
+
+            const check = await aliceCrypto.checkKeyBackupAndEnable();
+
+            const progressCallback = jest.fn();
+            const result = await aliceClient.restoreKeyBackupWithRecoveryKey(
+                testData.BACKUP_DECRYPTION_KEY_BASE58,
+                undefined,
+                undefined,
+                check!.backupInfo!,
+                {
+                    progressCallback,
+                },
+            );
+
+            expect(result.imported).toStrictEqual(expectedTotal);
+            // Should be called 5 times: 200*4 plus one chunk with the remaining 32
+            expect(importMockImpl).toHaveBeenCalledTimes(5);
+            for (let i = 0; i < 4; i++) {
+                expect(importMockImpl.mock.calls[i][0].length).toEqual(200);
+            }
+            expect(importMockImpl.mock.calls[4][0].length).toEqual(32);
+
+            expect(progressCallback).toHaveBeenCalledWith({
+                stage: "fetch",
+            });
+
+            // Should be called 4 times and report 200/400/600/800
+            for (let i = 0; i < 4; i++) {
+                expect(progressCallback).toHaveBeenCalledWith({
+                    total: expectedTotal,
+                    successes: (i + 1) * 200,
+                    stage: "load_keys",
+                    failures: 0,
+                });
+            }
+
+            // The last chunk
+            expect(progressCallback).toHaveBeenCalledWith({
+                total: expectedTotal,
+                successes: 832,
+                stage: "load_keys",
+                failures: 0,
+            });
+        });
+
+        it("Should continue to process backup if a chunk import fails and report failures", async function () {
+            // @ts-ignore - mock a private method for testing purpose
+            aliceCrypto.importBackedUpRoomKeys = jest
+                .fn()
+                .mockImplementationOnce(() => {
+                    // Fail to import first chunk
+                    throw new Error("test error");
+                })
+                // Ok for other chunks
+                .mockResolvedValue(undefined);
+
+            const { response, expectedTotal } = createBackupDownloadResponse([100, 300]);
+
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys", response);
+
+            const check = await aliceCrypto.checkKeyBackupAndEnable();
+
+            const progressCallback = jest.fn();
+            const result = await aliceClient.restoreKeyBackupWithRecoveryKey(
+                testData.BACKUP_DECRYPTION_KEY_BASE58,
+                undefined,
+                undefined,
+                check!.backupInfo!,
+                {
+                    progressCallback,
+                },
+            );
+
+            expect(result.total).toStrictEqual(expectedTotal);
+            // A chunk failed to import
+            expect(result.imported).toStrictEqual(200);
+
+            expect(progressCallback).toHaveBeenCalledWith({
+                total: expectedTotal,
+                successes: 0,
+                stage: "load_keys",
+                failures: 200,
+            });
+
+            expect(progressCallback).toHaveBeenCalledWith({
+                total: expectedTotal,
+                successes: 200,
+                stage: "load_keys",
+                failures: 200,
+            });
+        });
+
+        it("Should continue if some keys fails to decrypt", async function () {
+            // @ts-ignore - mock a private method for testing purpose
+            aliceCrypto.importBackedUpRoomKeys = jest.fn();
+
+            const decryptionFailureCount = 2;
+
+            const mockDecryptor = {
+                // DecryptSessions does not reject on decryption failure, but just skip the key
+                decryptSessions: jest.fn().mockImplementation((sessions) => {
+                    // simulate fail to decrypt 2 keys out of all
+                    const decrypted = [];
+                    const keys = Object.keys(sessions);
+                    for (let i = 0; i < keys.length - decryptionFailureCount; i++) {
+                        decrypted.push({
+                            session_id: keys[i],
+                        } as unknown as Mocked<IMegolmSessionData>);
+                    }
+                    return decrypted;
+                }),
+                free: jest.fn(),
+            };
+
+            // @ts-ignore - mock a private method for testing purpose
+            aliceCrypto.getBackupDecryptor = jest.fn().mockResolvedValue(mockDecryptor);
+
+            const { response, expectedTotal } = createBackupDownloadResponse([100]);
+
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys", response);
+
+            const check = await aliceCrypto.checkKeyBackupAndEnable();
+
+            const result = await aliceClient.restoreKeyBackupWithRecoveryKey(
+                testData.BACKUP_DECRYPTION_KEY_BASE58,
+                undefined,
+                undefined,
+                check!.backupInfo!,
+            );
+
+            expect(result.total).toStrictEqual(expectedTotal);
+            // A chunk failed to import
+            expect(result.imported).toStrictEqual(expectedTotal - decryptionFailureCount);
+        });
+
         it("recover specific session from backup", async function () {
-            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
-
-            aliceClient = await initTestClient();
-            const aliceCrypto = aliceClient.getCrypto()!;
-            await aliceClient.startClient();
-
-            // tell Alice to trust the dummy device that signed the backup
-            await waitForDeviceList();
-            await aliceCrypto.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
-
             fetchMock.get(
                 "express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id",
                 testData.CURVE25519_KEY_BACKUP_DATA,
@@ -371,16 +548,6 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
         });
 
         it("Fails on bad recovery key", async function () {
-            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
-
-            aliceClient = await initTestClient();
-            const aliceCrypto = aliceClient.getCrypto()!;
-            await aliceClient.startClient();
-
-            // tell Alice to trust the dummy device that signed the backup
-            await waitForDeviceList();
-            await aliceCrypto.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
-
             const fullBackup = {
                 rooms: {
                     [ROOM_ID]: {

spec/integ/crypto/rust-crypto.spec.ts

@@ -16,8 +16,12 @@ limitations under the License.
 
 import "fake-indexeddb/auto";
 import { IDBFactory } from "fake-indexeddb";
+import fetchMock from "fetch-mock-jest";
 
-import { createClient } from "../../../src";
+import { createClient, CryptoEvent, IndexedDBCryptoStore } from "../../../src";
+import { populateStore } from "../../test-utils/test_indexeddb_cryptostore_dump";
+
+jest.setTimeout(15000);
 
 afterEach(() => {
     // reset fake-indexeddb after each test, to make sure we don't leak connections
@@ -88,6 +92,70 @@ describe("MatrixClient.initRustCrypto", () => {
         await matrixClient.initRustCrypto();
         await matrixClient.initRustCrypto();
     });
+
+    it("should migrate from libolm", async () => {
+        fetchMock.get("path:/_matrix/client/v3/room_keys/version", {
+            auth_data: {
+                public_key: "q+HZiJdHl2Yopv9GGvv7EYSzDMrAiRknK4glSdoaomI",
+                signatures: {
+                    "@vdhtest200713:matrix.org": {
+                        "ed25519:gh9fGr39eNZUdWynEMJ/q/WZq/Pk/foFxHXFBFm18ZI":
+                            "reDp6Mu+j+tfUL3/T6f5OBT3N825Lzpc43vvG+RvjX6V+KxXzodBQArgCoeEHLtL9OgSBmNrhTkSOX87MWCKAw",
+                        "ed25519:KMFSTJSMLB":
+                            "F8tyV5W6wNi0GXTdSg+gxSCULQi0EYxdAAqfkyNq58KzssZMw5i+PRA0aI2b+D7NH/aZaJrtiYNHJ0gWLSQvAw",
+                    },
+                },
+            },
+            version: "7",
+            algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+            etag: "1",
+            count: 79,
+        });
+
+        const testStoreName = "test-store";
+        await populateStore(testStoreName);
+        const cryptoStore = new IndexedDBCryptoStore(indexedDB, testStoreName);
+
+        const matrixClient = createClient({
+            baseUrl: "http://test.server",
+            userId: "@vdhtest200713:matrix.org",
+            deviceId: "KMFSTJSMLB",
+            cryptoStore,
+            pickleKey: "+1k2Ppd7HIisUY824v7JtV3/oEE4yX0TqtmNPyhaD7o",
+        });
+
+        const progressListener = jest.fn();
+        matrixClient.addListener(CryptoEvent.LegacyCryptoStoreMigrationProgress, progressListener);
+
+        await matrixClient.initRustCrypto();
+
+        // Do some basic checks on the imported data
+        const deviceKeys = await matrixClient.getCrypto()!.getOwnDeviceKeys();
+        expect(deviceKeys.curve25519).toEqual("LKv0bKbc0EC4h0jknbemv3QalEkeYvuNeUXVRgVVTTU");
+        expect(deviceKeys.ed25519).toEqual("qK70DEqIXq7T+UU3v/al47Ab4JkMEBLpNrTBMbS5rrw");
+
+        expect(await matrixClient.getCrypto()!.getActiveSessionBackupVersion()).toEqual("7");
+
+        expect(await matrixClient.getCrypto()!.isEncryptionEnabledInRoom("!CWLUCoEWXSFyTCOtfL:matrix.org")).toBe(true);
+
+        // check the progress callback
+        expect(progressListener.mock.calls.length).toBeGreaterThan(50);
+
+        // The first call should have progress == 0
+        const [firstProgress, totalSteps] = progressListener.mock.calls[0];
+        expect(totalSteps).toBeGreaterThan(3000);
+        expect(firstProgress).toEqual(0);
+
+        for (let i = 1; i < progressListener.mock.calls.length - 1; i++) {
+            const [progress, total] = progressListener.mock.calls[i];
+            expect(total).toEqual(totalSteps);
+            expect(progress).toBeGreaterThan(progressListener.mock.calls[i - 1][0]);
+            expect(progress).toBeLessThanOrEqual(totalSteps);
+        }
+
+        // The final call should have progress == total == -1
+        expect(progressListener).toHaveBeenLastCalledWith(-1, -1);
+    }, 60000);
 });
 
 describe("MatrixClient.clearStores", () => {

spec/integ/crypto/verification.spec.ts

@@ -85,7 +85,8 @@ import { encodeBase64 } from "../../../src/base64";
 
 // The verification flows use javascript timers to set timeouts. We tell jest to use mock timer implementations
 // to ensure that we don't end up with dangling timeouts.
-jest.useFakeTimers();
+// But the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
 beforeAll(async () => {
     // we use the libolm primitives in the test, so init the Olm library
@@ -743,6 +744,8 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
             expect(toDeviceMessage.transaction_id).toEqual(transactionId);
             expect(toDeviceMessage.code).toEqual("m.user");
             expect(request.phase).toEqual(VerificationPhase.Cancelled);
+            expect(request.cancellationCode).toEqual("m.user");
+            expect(request.cancellingUserId).toEqual("@alice:localhost");
         });
 
         it("can cancel during the SAS phase", async () => {
@@ -1285,7 +1288,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
             await new Promise((resolve) => {
                 setTimeout(resolve, 500);
             });
-            jest.useFakeTimers();
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
             // the backup secret should not be cached
             const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
@@ -1309,7 +1312,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
             await new Promise((resolve) => {
                 setTimeout(resolve, 500);
             });
-            jest.useFakeTimers();
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
             // the backup secret should not be cached
             const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
@@ -1334,7 +1337,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
             await new Promise((resolve) => {
                 setTimeout(resolve, 500);
             });
-            jest.useFakeTimers();
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
             // the backup secret should not be cached
             const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
@@ -1355,7 +1358,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
             await new Promise((resolve) => {
                 setTimeout(resolve, 500);
             });
-            jest.useFakeTimers();
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
             // the backup secret should not be cached
             const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();

spec/integ/matrix-client-event-timeline.spec.ts

@@ -1555,7 +1555,7 @@ describe("MatrixClient event timelines", function () {
                 expect(threadIds).toContain(THREAD2_ROOT.event_id);
                 const [allThreads] = timelineSets!;
                 const timeline = allThreads.getLiveTimeline()!;
-                // Test threads are in chronological order
+                // Test threads are in chronological order (first thread should be first because it has a more recent reply)
                 expect(timeline.getEvents().map((it) => it.event.event_id)).toEqual([
                     THREAD_ROOT.event_id,
                     THREAD2_ROOT.event_id,

spec/test-utils/mockEndpoints.ts

@@ -24,11 +24,21 @@ import { KeyBackupInfo } from "../../src/crypto-api";
  * @param homeserverUrl - the homeserver url for the client under test
  */
 export function mockInitialApiRequests(homeserverUrl: string) {
-    fetchMock.getOnce(new URL("/_matrix/client/versions", homeserverUrl).toString(), { versions: ["v1.1"] });
-    fetchMock.getOnce(new URL("/_matrix/client/v3/pushrules/", homeserverUrl).toString(), {});
-    fetchMock.postOnce(new URL("/_matrix/client/v3/user/%40alice%3Alocalhost/filter", homeserverUrl).toString(), {
-        filter_id: "fid",
-    });
+    fetchMock.getOnce(
+        new URL("/_matrix/client/versions", homeserverUrl).toString(),
+        { versions: ["v1.1"] },
+        { overwriteRoutes: true },
+    );
+    fetchMock.getOnce(
+        new URL("/_matrix/client/v3/pushrules/", homeserverUrl).toString(),
+        {},
+        { overwriteRoutes: true },
+    );
+    fetchMock.postOnce(
+        new URL("/_matrix/client/v3/user/%40alice%3Alocalhost/filter", homeserverUrl).toString(),
+        { filter_id: "fid" },
+        { overwriteRoutes: true },
+    );
 }
 
 /**

spec/test-utils/test_indexeddb_cryptostore_dump/README.md

@@ -0,0 +1,53 @@
+## Dump of libolm indexeddb cryptostore
+
+This directory contains a dump of a real indexeddb store from a session using
+libolm crypto.
+
+The corresponding pickle key is `+1k2Ppd7HIisUY824v7JtV3/oEE4yX0TqtmNPyhaD7o`.
+
+It was created by pasting the following into the browser console:
+
+```javascript
+async function exportIndexedDb(name) {
+    const db = await new Promise((resolve, reject) => {
+        const dbReq = indexedDB.open(name);
+        dbReq.onerror = reject;
+        dbReq.onsuccess = () => resolve(dbReq.result);
+    });
+
+    const storeNames = db.objectStoreNames;
+    const exports = {};
+    for (const store of storeNames) {
+        exports[store] = [];
+        const txn = db.transaction(store, "readonly");
+        const objectStore = txn.objectStore(store);
+        await new Promise((resolve, reject) => {
+            const cursorReq = objectStore.openCursor();
+            cursorReq.onerror = reject;
+            cursorReq.onsuccess = (event) => {
+                const cursor = event.target.result;
+                if (cursor) {
+                    const entry = { value: cursor.value };
+                    if (!objectStore.keyPath) {
+                        entry.key = cursor.key;
+                    }
+                    exports[store].push(entry);
+                    cursor.continue();
+                } else {
+                    resolve();
+                }
+            };
+        });
+    }
+    return exports;
+}
+
+window.saveAs(
+    new Blob([JSON.stringify(await exportIndexedDb("matrix-js-sdk:crypto"), null, 2)], {
+        type: "application/json;charset=utf-8",
+    }),
+    "dump.json",
+);
+```
+
+The pickle key is extracted via `mxMatrixClientPeg.get().crypto.olmDevice.pickleKey`.

spec/test-utils/test_indexeddb_cryptostore_dump/index.ts

@@ -0,0 +1,136 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+
+/**
+ * Populate an IndexedDB store with the test data from this directory.
+ *
+ * @param name - Name of the IndexedDB database to create.
+ */
+export async function populateStore(name: string): Promise<IDBDatabase> {
+    const req = indexedDB.open(name, 11);
+
+    const db = await new Promise<IDBDatabase>((resolve, reject) => {
+        req.onupgradeneeded = (ev): void => {
+            const db = req.result;
+            const oldVersion = ev.oldVersion;
+            upgradeDatabase(oldVersion, db);
+        };
+
+        req.onerror = (ev): void => {
+            reject(req.error);
+        };
+
+        req.onsuccess = (): void => {
+            const db = req.result;
+            resolve(db);
+        };
+    });
+
+    await importData(db);
+
+    return db;
+}
+
+/** Create the schema for the indexed db store */
+function upgradeDatabase(oldVersion: number, db: IDBDatabase) {
+    if (oldVersion < 1) {
+        const outgoingRoomKeyRequestsStore = db.createObjectStore("outgoingRoomKeyRequests", { keyPath: "requestId" });
+        outgoingRoomKeyRequestsStore.createIndex("session", ["requestBody.room_id", "requestBody.session_id"]);
+        outgoingRoomKeyRequestsStore.createIndex("state", "state");
+    }
+
+    if (oldVersion < 2) {
+        db.createObjectStore("account");
+    }
+
+    if (oldVersion < 3) {
+        const sessionsStore = db.createObjectStore("sessions", { keyPath: ["deviceKey", "sessionId"] });
+        sessionsStore.createIndex("deviceKey", "deviceKey");
+    }
+
+    if (oldVersion < 4) {
+        db.createObjectStore("inbound_group_sessions", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 5) {
+        db.createObjectStore("device_data");
+    }
+
+    if (oldVersion < 6) {
+        db.createObjectStore("rooms");
+    }
+
+    if (oldVersion < 7) {
+        db.createObjectStore("sessions_needing_backup", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 8) {
+        db.createObjectStore("inbound_group_sessions_withheld", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 9) {
+        const problemsStore = db.createObjectStore("session_problems", { keyPath: ["deviceKey", "time"] });
+        problemsStore.createIndex("deviceKey", "deviceKey");
+
+        db.createObjectStore("notified_error_devices", { keyPath: ["userId", "deviceId"] });
+    }
+
+    if (oldVersion < 10) {
+        db.createObjectStore("shared_history_inbound_group_sessions", { keyPath: ["roomId"] });
+    }
+
+    if (oldVersion < 11) {
+        db.createObjectStore("parked_shared_history", { keyPath: ["roomId"] });
+    }
+}
+
+async function importData(db: IDBDatabase) {
+    const path = resolve("spec/test-utils/test_indexeddb_cryptostore_dump/dump.json");
+    const json: Record<string, Array<{ key?: any; value: any }>> = JSON.parse(
+        await readFile(path, { encoding: "utf8" }),
+    );
+
+    for (const [storeName, data] of Object.entries(json)) {
+        await new Promise((resolve, reject) => {
+            const store = db.transaction(storeName, "readwrite").objectStore(storeName);
+
+            function putEntry(idx: number) {
+                if (idx >= data.length) {
+                    resolve(undefined);
+                    return;
+                }
+
+                const { key, value } = data[idx];
+                try {
+                    const putReq = store.put(value, key);
+                    putReq.onsuccess = (_) => putEntry(idx + 1);
+                    putReq.onerror = (_) => reject(putReq.error);
+                } catch (e) {
+                    throw new Error(
+                        `Error populating '${storeName}' with key ${JSON.stringify(key)}, value ${JSON.stringify(
+                            value,
+                        )}: ${e}`,
+                    );
+                }
+            }
+
+            putEntry(0);
+        });
+    }
+}

spec/unit/content-repo.spec.ts

@@ -37,6 +37,21 @@ describe("ContentRepo", function () {
             );
         });
 
+        it("should allow redirects when requested on download URLs", function () {
+            const mxcUri = "mxc://server.name/resourceid";
+            expect(getHttpUriForMxc(baseUrl, mxcUri, undefined, undefined, undefined, false, true)).toEqual(
+                baseUrl + "/_matrix/media/v3/download/server.name/resourceid?allow_redirect=true",
+            );
+        });
+
+        it("should allow redirects when requested on thumbnail URLs", function () {
+            const mxcUri = "mxc://server.name/resourceid";
+            expect(getHttpUriForMxc(baseUrl, mxcUri, 32, 32, "scale", false, true)).toEqual(
+                baseUrl +
+                    "/_matrix/media/v3/thumbnail/server.name/resourceid?width=32&height=32&method=scale&allow_redirect=true",
+            );
+        });
+
         it("should return the empty string for null input", function () {
             expect(getHttpUriForMxc(null as any, "")).toEqual("");
         });

spec/unit/crypto/store/CryptoStore.spec.ts

@@ -74,6 +74,12 @@ describe.each([
             const N_SESSIONS_PER_DEVICE = 6;
             await createSessions(N_DEVICES, N_SESSIONS_PER_DEVICE);
 
+            let nSessions = 0;
+            await store.doTxn("readonly", [IndexedDBCryptoStore.STORE_SESSIONS], (txn) =>
+                store.countEndToEndSessions(txn, (n) => (nSessions = n)),
+            );
+            expect(nSessions).toEqual(N_DEVICES * N_SESSIONS_PER_DEVICE);
+
             // Then, get a batch and check it looks right.
             const batch = await store.getEndToEndSessionsBatch();
             expect(batch!.length).toEqual(N_DEVICES * N_SESSIONS_PER_DEVICE);
@@ -145,6 +151,13 @@ describe.each([
             const N_SESSIONS_PER_DEVICE = 6;
             await createSessions(N_DEVICES, N_SESSIONS_PER_DEVICE);
 
+            // Mark one of the sessions as needing backup
+            await store.doTxn("readwrite", IndexedDBCryptoStore.STORE_BACKUP, async (txn) => {
+                await store.markSessionsNeedingBackup([{ senderKey: pad43("device5"), sessionId: "session5" }], txn);
+            });
+
+            expect(await store.countEndToEndInboundGroupSessions()).toEqual(N_DEVICES * N_SESSIONS_PER_DEVICE);
+
             const batch = await store.getEndToEndInboundGroupSessionsBatch();
             expect(batch!.length).toEqual(N_DEVICES * N_SESSIONS_PER_DEVICE);
             for (let i = 0; i < N_DEVICES; i++) {
@@ -153,6 +166,9 @@ describe.each([
 
                     expect(r.senderKey).toEqual(pad43(`device${i}`));
                     expect(r.sessionId).toEqual(`session${j}`);
+
+                    // only the last session needs backup
+                    expect(r.needsBackup).toBe(i === 5 && j === 5);
                 }
             }
         });

spec/unit/crypto/verification/sas.spec.ts

@@ -189,10 +189,12 @@ describe("SAS verification", function () {
             const origSendToDevice = bob.client.sendToDevice.bind(bob.client);
             bob.client.sendToDevice = async (type, map) => {
                 if (type === "m.key.verification.accept") {
-                    macMethod = map.get(alice.client.getUserId()!)?.get(alice.client.deviceId!)
-                        ?.message_authentication_code;
-                    keyAgreement = map.get(alice.client.getUserId()!)?.get(alice.client.deviceId!)
-                        ?.key_agreement_protocol;
+                    macMethod = map
+                        .get(alice.client.getUserId()!)
+                        ?.get(alice.client.deviceId!)?.message_authentication_code;
+                    keyAgreement = map
+                        .get(alice.client.getUserId()!)
+                        ?.get(alice.client.deviceId!)?.key_agreement_protocol;
                 }
                 return origSendToDevice(type, map);
             };

spec/unit/matrix-client.spec.ts

@@ -55,6 +55,7 @@ import {
     RuleId,
     IPushRule,
     ConditionKind,
+    getHttpUriForMxc,
 } from "../../src";
 import { supportsMatrixCall } from "../../src/webrtc/call";
 import { makeBeaconEvent } from "../test-utils/beacon";
@@ -64,7 +65,7 @@ import {
     PolicyScope,
 } from "../../src/models/invites-ignorer";
 import { IOlmDevice } from "../../src/crypto/algorithms/megolm";
-import { QueryDict } from "../../src/utils";
+import { defer, QueryDict } from "../../src/utils";
 import { SyncState } from "../../src/sync";
 import * as featureUtils from "../../src/feature";
 import { StubStore } from "../../src/store/stub";
@@ -369,6 +370,21 @@ describe("MatrixClient", function () {
         client.stopClient();
     });
 
+    describe("mxcUrlToHttp", () => {
+        it("should call getHttpUriForMxc", () => {
+            const mxc = "mxc://server/example";
+            expect(client.mxcUrlToHttp(mxc)).toBe(getHttpUriForMxc(client.baseUrl, mxc));
+            expect(client.mxcUrlToHttp(mxc, 32)).toBe(getHttpUriForMxc(client.baseUrl, mxc, 32));
+            expect(client.mxcUrlToHttp(mxc, 32, 46)).toBe(getHttpUriForMxc(client.baseUrl, mxc, 32, 46));
+            expect(client.mxcUrlToHttp(mxc, 32, 46, "scale")).toBe(
+                getHttpUriForMxc(client.baseUrl, mxc, 32, 46, "scale"),
+            );
+            expect(client.mxcUrlToHttp(mxc, 32, 46, "scale", false, true)).toBe(
+                getHttpUriForMxc(client.baseUrl, mxc, 32, 46, "scale", false, true),
+            );
+        });
+    });
+
     describe("timestampToEvent", () => {
         const roomId = "!room:server.org";
         const eventId = "$eventId:example.org";
@@ -1434,25 +1450,11 @@ describe("MatrixClient", function () {
         const mockRoom = {
             getMyMembership: () => "join",
             updatePendingEvent: (event: MatrixEvent, status: EventStatus) => event.setStatus(status),
-            currentState: {
-                getStateEvents: (eventType, stateKey) => {
-                    if (eventType === EventType.RoomCreate) {
-                        expect(stateKey).toEqual("");
-                        return new MatrixEvent({
-                            content: {
-                                [RoomCreateTypeField]: RoomType.Space,
-                            },
-                        });
-                    } else if (eventType === EventType.RoomEncryption) {
-                        expect(stateKey).toEqual("");
-                        return new MatrixEvent({ content: {} });
-                    } else {
-                        throw new Error("Unexpected event type or state key");
-                    }
-                },
-            } as Room["currentState"],
+            hasEncryptionStateEvent: jest.fn().mockReturnValue(true),
         } as unknown as Room;
 
+        let mockCrypto: Mocked<Crypto>;
+
         let event: MatrixEvent;
         beforeEach(async () => {
             event = new MatrixEvent({
@@ -1467,11 +1469,12 @@ describe("MatrixClient", function () {
                 expect(getRoomId).toEqual(roomId);
                 return mockRoom;
             };
-            client.crypto = client["cryptoBackend"] = {
-                // mock crypto
-                encryptEvent: () => new Promise(() => {}),
+            mockCrypto = {
+                isEncryptionEnabledInRoom: jest.fn().mockResolvedValue(true),
+                encryptEvent: jest.fn(),
                 stop: jest.fn(),
-            } as unknown as Crypto;
+            } as unknown as Mocked<Crypto>;
+            client.crypto = client["cryptoBackend"] = mockCrypto;
         });
 
         function assertCancelled() {
@@ -1488,12 +1491,21 @@ describe("MatrixClient", function () {
         });
 
         it("should cancel an event which is encrypting", async () => {
+            const encryptEventDefer = defer();
+            mockCrypto.encryptEvent.mockReturnValue(encryptEventDefer.promise);
+
+            const statusPromise = testUtils.emitPromise(event, "Event.status");
             // @ts-ignore protected method access
-            client.encryptAndSendEvent(mockRoom, event);
-            await testUtils.emitPromise(event, "Event.status");
+            const encryptAndSendPromise = client.encryptAndSendEvent(mockRoom, event);
+            await statusPromise;
             expect(event.status).toBe(EventStatus.ENCRYPTING);
             client.cancelPendingEvent(event);
             assertCancelled();
+
+            // now let the encryption complete, and check that the message is not sent.
+            encryptEventDefer.resolve();
+            await encryptAndSendPromise;
+            assertCancelled();
         });
 
         it("should cancel an event which is not sent", () => {

spec/unit/matrixrtc/CallMembership.spec.ts

@@ -34,9 +34,12 @@ function makeMockEvent(originTs = 0): MatrixEvent {
 }
 
 describe("CallMembership", () => {
-    it("rejects membership with no expiry", () => {
+    it("rejects membership with no expiry and no expires_ts", () => {
         expect(() => {
-            new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { expires: undefined }));
+            new CallMembership(
+                makeMockEvent(),
+                Object.assign({}, membershipTemplate, { expires: undefined, expires_ts: undefined }),
+            );
         }).toThrow();
     });
 
@@ -57,6 +60,16 @@ describe("CallMembership", () => {
             new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { scope: undefined }));
         }).toThrow();
     });
+    it("rejects with malformatted expires_ts", () => {
+        expect(() => {
+            new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { expires_ts: "string" }));
+        }).toThrow();
+    });
+    it("rejects with malformatted expires", () => {
+        expect(() => {
+            new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { expires: "string" }));
+        }).toThrow();
+    });
 
     it("uses event timestamp if no created_ts", () => {
         const membership = new CallMembership(makeMockEvent(12345), membershipTemplate);
@@ -71,11 +84,19 @@ describe("CallMembership", () => {
         expect(membership.createdTs()).toEqual(67890);
     });
 
-    it("computes absolute expiry time", () => {
+    it("computes absolute expiry time based on expires", () => {
         const membership = new CallMembership(makeMockEvent(1000), membershipTemplate);
         expect(membership.getAbsoluteExpiry()).toEqual(5000 + 1000);
     });
 
+    it("computes absolute expiry time based on expires_ts", () => {
+        const membership = new CallMembership(
+            makeMockEvent(1000),
+            Object.assign({}, membershipTemplate, { expires: undefined, expires_ts: 6000 }),
+        );
+        expect(membership.getAbsoluteExpiry()).toEqual(5000 + 1000);
+    });
+
     it("considers memberships unexpired if local age low enough", () => {
         const fakeEvent = makeMockEvent(1000);
         fakeEvent.getLocalAge = jest.fn().mockReturnValue(3000);

spec/unit/matrixrtc/MatrixRTCSession.spec.ts

@@ -214,8 +214,8 @@ describe("MatrixRTCSession", () => {
         });
 
         it("sends a membership event when joining a call", () => {
+            jest.useFakeTimers();
             sess!.joinRoomSession([mockFocus]);
-
             expect(client.sendStateEvent).toHaveBeenCalledWith(
                 mockRoom!.roomId,
                 EventType.GroupCallMemberPrefix,
@@ -227,6 +227,7 @@ describe("MatrixRTCSession", () => {
                             call_id: "",
                             device_id: "AAAAAAA",
                             expires: 3600000,
+                            expires_ts: Date.now() + 3600000,
                             foci_active: [{ type: "mock" }],
                             membershipID: expect.stringMatching(".*"),
                         },
@@ -234,6 +235,7 @@ describe("MatrixRTCSession", () => {
                 },
                 "@alice:example.org",
             );
+            jest.useRealTimers();
         });
 
         it("does nothing if join called when already joined", () => {
@@ -291,6 +293,7 @@ describe("MatrixRTCSession", () => {
                                 call_id: "",
                                 device_id: "AAAAAAA",
                                 expires: 3600000 * 2,
+                                expires_ts: 1000 + 3600000 * 2,
                                 foci_active: [{ type: "mock" }],
                                 created_ts: 1000,
                                 membershipID: expect.stringMatching(".*"),
@@ -510,7 +513,7 @@ describe("MatrixRTCSession", () => {
         });
     });
 
-    it("Does not emits if no membership changes", () => {
+    it("Does not emit if no membership changes", () => {
         const mockRoom = makeMockRoom([membershipTemplate]);
         sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
 
@@ -591,6 +594,7 @@ describe("MatrixRTCSession", () => {
                             call_id: "",
                             device_id: "AAAAAAA",
                             expires: 3600000,
+                            expires_ts: Date.now() + 3600000,
                             foci_active: [mockFocus],
                             membershipID: expect.stringMatching(".*"),
                         },
@@ -605,7 +609,7 @@ describe("MatrixRTCSession", () => {
 
     it("fills in created_ts for other memberships on update", () => {
         client.sendStateEvent = jest.fn();
-
+        jest.useFakeTimers();
         const mockRoom = makeMockRoom([
             Object.assign({}, membershipTemplate, {
                 device_id: "OTHERDEVICE",
@@ -635,6 +639,7 @@ describe("MatrixRTCSession", () => {
                         call_id: "",
                         device_id: "AAAAAAA",
                         expires: 3600000,
+                        expires_ts: Date.now() + 3600000,
                         foci_active: [mockFocus],
                         membershipID: expect.stringMatching(".*"),
                     },
@@ -642,6 +647,7 @@ describe("MatrixRTCSession", () => {
             },
             "@alice:example.org",
         );
+        jest.useRealTimers();
     });
 
     it("collects keys from encryption events", () => {

spec/unit/queueToDevice.spec.ts

@@ -265,6 +265,7 @@ describe.each([[StoreType.Memory], [StoreType.IndexedDB]])("queueToDevice (%s st
         });
         const mockRoom = {
             updatePendingEvent: jest.fn(),
+            hasEncryptionStateEvent: jest.fn().mockReturnValue(false),
         } as unknown as Room;
         client.resendEvent(dummyEvent, mockRoom);
 

spec/unit/rendezvous/rendezvous.spec.ts

@@ -23,7 +23,7 @@ import {
     MSC3903ECDHPayload,
     MSC3903ECDHv2RendezvousChannel as MSC3903ECDHRendezvousChannel,
 } from "../../../src/rendezvous/channels";
-import { MatrixClient } from "../../../src";
+import { Device, MatrixClient } from "../../../src";
 import {
     MSC3886SimpleHttpRendezvousTransport,
     MSC3886SimpleHttpRendezvousTransportDetails,
@@ -31,16 +31,57 @@ import {
 import { DummyTransport } from "./DummyTransport";
 import { decodeBase64 } from "../../../src/base64";
 import { logger } from "../../../src/logger";
-import { DeviceInfo } from "../../../src/crypto/deviceinfo";
+import { CrossSigningKey, OwnDeviceKeys } from "../../../src/crypto-api";
+
+type UserID = string;
+type DeviceID = string;
+type Fingerprint = string;
+type SimpleDeviceMap = Record<UserID, Record<DeviceID, Fingerprint>>;
+
+function mockDevice(userId: UserID, deviceId: DeviceID, fingerprint: Fingerprint): Device {
+    return {
+        deviceId,
+        userId,
+        getFingerprint: () => fingerprint,
+    } as unknown as Device;
+}
+
+function mockDeviceMap(
+    userId: UserID,
+    deviceId: DeviceID,
+    deviceKey?: Fingerprint,
+    otherDevices: SimpleDeviceMap = {},
+): Map<string, Map<string, Device>> {
+    const deviceMap: Map<string, Map<string, Device>> = new Map();
+
+    const myDevices: Map<string, Device> = new Map();
+    if (deviceKey) {
+        myDevices.set(deviceId, mockDevice(userId, deviceId, deviceKey));
+    }
+    deviceMap.set(userId, myDevices);
+
+    for (const u in otherDevices) {
+        let userDevices = deviceMap.get(u);
+        if (!userDevices) {
+            userDevices = new Map();
+            deviceMap.set(u, userDevices);
+        }
+        for (const d in otherDevices[u]) {
+            userDevices.set(d, mockDevice(u, d, otherDevices[u][d]));
+        }
+    }
+
+    return deviceMap;
+}
 
 function makeMockClient(opts: {
-    userId: string;
-    deviceId: string;
-    deviceKey?: string;
+    userId: UserID;
+    deviceId: DeviceID;
+    deviceKey?: Fingerprint;
     getLoginTokenEnabled: boolean;
     msc3882r0Only: boolean;
     msc3886Enabled: boolean;
-    devices?: Record<string, Partial<DeviceInfo>>;
+    devices?: SimpleDeviceMap;
     verificationFunction?: (
         userId: string,
         deviceId: string,
@@ -48,50 +89,77 @@ function makeMockClient(opts: {
         blocked: boolean,
         known: boolean,
     ) => void;
-    crossSigningIds?: Record<string, string>;
-}): MatrixClient {
-    return {
-        getVersions() {
-            return {
-                unstable_features: {
-                    "org.matrix.msc3882": opts.getLoginTokenEnabled,
-                    "org.matrix.msc3886": opts.msc3886Enabled,
-                },
-            };
-        },
-        getCapabilities() {
-            return opts.msc3882r0Only
-                ? {}
-                : {
-                      capabilities: {
-                          "m.get_login_token": {
-                              enabled: opts.getLoginTokenEnabled,
+    crossSigningIds?: Partial<Record<CrossSigningKey, string>>;
+}): [MatrixClient, Map<string, Map<string, Device>>] {
+    const deviceMap = mockDeviceMap(opts.userId, opts.deviceId, opts.deviceKey, opts.devices);
+    return [
+        {
+            doesServerSupportUnstableFeature: jest.fn().mockImplementation((feature) => {
+                if (feature === "org.matrix.msc3886") {
+                    return opts.msc3886Enabled;
+                } else if (feature === "org.matrix.msc3882") {
+                    return opts.getLoginTokenEnabled;
+                } else {
+                    return false;
+                }
+            }),
+            getVersions() {
+                return {
+                    unstable_features: {
+                        "org.matrix.msc3882": opts.getLoginTokenEnabled,
+                        "org.matrix.msc3886": opts.msc3886Enabled,
+                    },
+                };
+            },
+            getCapabilities() {
+                return opts.msc3882r0Only
+                    ? {}
+                    : {
+                          capabilities: {
+                              "m.get_login_token": {
+                                  enabled: opts.getLoginTokenEnabled,
+                              },
                           },
-                      },
-                  };
-        },
-        getUserId() {
-            return opts.userId;
-        },
-        getDeviceId() {
-            return opts.deviceId;
-        },
-        getDeviceEd25519Key() {
-            return opts.deviceKey;
-        },
-        baseUrl: "https://example.com",
-        crypto: {
-            getStoredDevice(userId: string, deviceId: string) {
-                return opts.devices?.[deviceId] ?? null;
+                      };
             },
-            setDeviceVerification: opts.verificationFunction,
-            crossSigningInfo: {
-                getId(key: string) {
-                    return opts.crossSigningIds?.[key];
-                },
+            getUserId() {
+                return opts.userId;
             },
-        },
-    } as unknown as MatrixClient;
+            getSafeUserId() {
+                return opts.userId;
+            },
+            getDeviceId() {
+                return opts.deviceId;
+            },
+            baseUrl: "https://example.com",
+            getCrypto() {
+                return {
+                    getUserDeviceInfo(
+                        [userId]: string[],
+                        downloadUncached?: boolean,
+                    ): Promise<Map<string, Map<string, Device>>> {
+                        return Promise.resolve(deviceMap);
+                    },
+                    getCrossSigningKeyId(key: CrossSigningKey): string | null {
+                        return opts.crossSigningIds?.[key] ?? null;
+                    },
+                    setDeviceVerified(userId: string, deviceId: string, verified: boolean): Promise<void> {
+                        return Promise.resolve();
+                    },
+                    crossSignDevice(deviceId: string): Promise<void> {
+                        return Promise.resolve();
+                    },
+                    getOwnDeviceKeys(): Promise<OwnDeviceKeys> {
+                        return Promise.resolve({
+                            ed25519: opts.deviceKey!,
+                            curve25519: "aaaa",
+                        });
+                    },
+                };
+            },
+        } as unknown as MatrixClient,
+        deviceMap,
+    ];
 }
 
 function makeTransport(name: string, uri = "https://test.rz/123456") {
@@ -106,6 +174,7 @@ describe("Rendezvous", function () {
     let httpBackend: MockHttpBackend;
     let fetchFn: typeof global.fetch;
     let transports: DummyTransport<any, MSC3903ECDHPayload>[];
+    const userId: UserID = "@user:example.com";
 
     beforeEach(function () {
         httpBackend = new MockHttpBackend();
@@ -118,9 +187,9 @@ describe("Rendezvous", function () {
     });
 
     it("generate and cancel", async function () {
-        const alice = makeMockClient({
-            userId: "@alice:example.com",
-            deviceId: "DEVICEID",
+        const [alice] = makeMockClient({
+            userId,
+            deviceId: "ALICE",
             msc3886Enabled: false,
             getLoginTokenEnabled: true,
             msc3882r0Only: true,
@@ -194,8 +263,8 @@ describe("Rendezvous", function () {
 
         // alice is already signs in and generates a code
         const aliceOnFailure = jest.fn();
-        const alice = makeMockClient({
-            userId: "alice",
+        const [alice] = makeMockClient({
+            userId,
             deviceId: "ALICE",
             msc3886Enabled: false,
             getLoginTokenEnabled,
@@ -257,8 +326,8 @@ describe("Rendezvous", function () {
 
         // alice is already signs in and generates a code
         const aliceOnFailure = jest.fn();
-        const alice = makeMockClient({
-            userId: "alice",
+        const [alice] = makeMockClient({
+            userId,
             deviceId: "ALICE",
             getLoginTokenEnabled: true,
             msc3882r0Only: false,
@@ -316,8 +385,8 @@ describe("Rendezvous", function () {
 
         // alice is already signs in and generates a code
         const aliceOnFailure = jest.fn();
-        const alice = makeMockClient({
-            userId: "alice",
+        const [alice] = makeMockClient({
+            userId,
             deviceId: "ALICE",
             getLoginTokenEnabled: true,
             msc3882r0Only: false,
@@ -375,7 +444,7 @@ describe("Rendezvous", function () {
 
         // alice is already signs in and generates a code
         const aliceOnFailure = jest.fn();
-        const alice = makeMockClient({
+        const [alice] = makeMockClient({
             userId: "alice",
             deviceId: "ALICE",
             getLoginTokenEnabled: true,
@@ -436,7 +505,7 @@ describe("Rendezvous", function () {
 
         // alice is already signs in and generates a code
         const aliceOnFailure = jest.fn();
-        const alice = makeMockClient({
+        const [alice] = makeMockClient({
             userId: "alice",
             deviceId: "ALICE",
             getLoginTokenEnabled: true,
@@ -495,7 +564,7 @@ describe("Rendezvous", function () {
         await bobCompleteProm;
     });
 
-    async function completeLogin(devices: Record<string, Partial<DeviceInfo>>) {
+    async function completeLogin(devices: SimpleDeviceMap) {
         const aliceTransport = makeTransport("Alice", "https://test.rz/123456");
         const bobTransport = makeTransport("Bob", "https://test.rz/999999");
         transports.push(aliceTransport, bobTransport);
@@ -505,8 +574,8 @@ describe("Rendezvous", function () {
         // alice is already signs in and generates a code
         const aliceOnFailure = jest.fn();
         const aliceVerification = jest.fn();
-        const alice = makeMockClient({
-            userId: "alice",
+        const [alice, deviceMap] = makeMockClient({
+            userId,
             deviceId: "ALICE",
             getLoginTokenEnabled: true,
             msc3882r0Only: false,
@@ -575,13 +644,14 @@ describe("Rendezvous", function () {
             aliceRz,
             bobTransport,
             bobEcdh,
+            deviceMap,
         };
     }
 
     it("approve on existing device + verification", async function () {
         const { bobEcdh, aliceRz } = await completeLogin({
-            BOB: {
-                getFingerprint: () => "bbbb",
+            [userId]: {
+                BOB: "bbbb",
             },
         });
         const verifyProm = aliceRz.verifyNewDeviceOnExistingDevice();
@@ -607,33 +677,29 @@ describe("Rendezvous", function () {
     });
 
     it("device appears online within timeout", async function () {
-        const devices: Record<string, Partial<DeviceInfo>> = {};
-        const { aliceRz } = await completeLogin(devices);
-        // device appears after 1 second
+        const devices: SimpleDeviceMap = {};
+        const { aliceRz, deviceMap } = await completeLogin(devices);
+        // device appears before the timeout
         setTimeout(() => {
-            devices.BOB = {
-                getFingerprint: () => "bbbb",
-            };
+            deviceMap.get(userId)!.set("BOB", mockDevice(userId, "BOB", "bbbb"));
         }, 1000);
         await aliceRz.verifyNewDeviceOnExistingDevice(2000);
     });
 
     it("device appears online after timeout", async function () {
-        const devices: Record<string, Partial<DeviceInfo>> = {};
-        const { aliceRz } = await completeLogin(devices);
-        // device appears after 1 second
+        const devices: SimpleDeviceMap = {};
+        const { aliceRz, deviceMap } = await completeLogin(devices);
+        // device appears after the timeout
         setTimeout(() => {
-            devices.BOB = {
-                getFingerprint: () => "bbbb",
-            };
+            deviceMap.get(userId)!.set("BOB", mockDevice(userId, "BOB", "bbbb"));
         }, 1500);
         await expect(aliceRz.verifyNewDeviceOnExistingDevice(1000)).rejects.toThrow();
     });
 
     it("mismatched device key", async function () {
         const { aliceRz } = await completeLogin({
-            BOB: {
-                getFingerprint: () => "XXXX",
+            [userId]: {
+                BOB: "XXXX",
             },
         });
         await expect(aliceRz.verifyNewDeviceOnExistingDevice(1000)).rejects.toThrow(/different key/);

spec/unit/room.spec.ts

@@ -1460,7 +1460,7 @@ describe("Room", function () {
                 it("should reset the unread count when our non-synthetic receipt points to the latest event", () => {
                     // Given a room with 2 events, and an unread count set.
                     room.client.isInitialSyncComplete = jest.fn().mockReturnValue(true);
-                    room.timeline = [event1, event2];
+                    jest.spyOn(room, "timeline", "get").mockReturnValue([event1, event2]);
                     room.setUnread(NotificationCountType.Total, 45);
                     room.setUnread(NotificationCountType.Highlight, 57);
                     // Sanity check:
@@ -1479,7 +1479,7 @@ describe("Room", function () {
                 it("should not reset the unread count when someone else's receipt points to the latest event", () => {
                     // Given a room with 2 events, and an unread count set.
                     room.client.isInitialSyncComplete = jest.fn().mockReturnValue(true);
-                    room.timeline = [event1, event2];
+                    jest.spyOn(room, "timeline", "get").mockReturnValue([event1, event2]);
                     room.setUnread(NotificationCountType.Total, 45);
                     room.setUnread(NotificationCountType.Highlight, 57);
                     // Sanity check:
@@ -1498,7 +1498,7 @@ describe("Room", function () {
                 it("should not reset the unread count when our non-synthetic receipt points to an earlier event", () => {
                     // Given a room with 2 events, and an unread count set.
                     room.client.isInitialSyncComplete = jest.fn().mockReturnValue(true);
-                    room.timeline = [event1, event2];
+                    jest.spyOn(room, "timeline", "get").mockReturnValue([event1, event2]);
                     room.setUnread(NotificationCountType.Total, 45);
                     room.setUnread(NotificationCountType.Highlight, 57);
                     // Sanity check:
@@ -1517,7 +1517,7 @@ describe("Room", function () {
                 it("should not reset the unread count when our a synthetic receipt points to the latest event", () => {
                     // Given a room with 2 events, and an unread count set.
                     room.client.isInitialSyncComplete = jest.fn().mockReturnValue(true);
-                    room.timeline = [event1, event2];
+                    jest.spyOn(room, "timeline", "get").mockReturnValue([event1, event2]);
                     room.setUnread(NotificationCountType.Total, 45);
                     room.setUnread(NotificationCountType.Highlight, 57);
                     // Sanity check:

spec/unit/rust-crypto/KeyClaimManager.spec.ts

@@ -97,7 +97,10 @@ describe("KeyClaimManager", () => {
         await keyClaimManager.ensureSessionsForUsers(new LogSpan(logger, "test"), [u1, u2]);
 
         // check that all the calls were made
-        expect(olmMachine.getMissingSessions).toHaveBeenCalledWith([u1, u2]);
+        // We can't use directly toHaveBeenCalledWith because the UserId are cloned in the process.
+        const calledWith = olmMachine.getMissingSessions.mock.calls[0][0].map((u) => u.toString());
+        expect(calledWith).toEqual([u1.toString(), u2.toString()]);
+
         expect(fetchMock).toHaveFetched("https://example.com/_matrix/client/v3/keys/claim", {
             method: "POST",
             body: { k1: "v1" },
@@ -135,7 +138,10 @@ describe("KeyClaimManager", () => {
 
         // at this point, there should have been a single call to getMissingSessions, and a single fetch; and neither
         // call to ensureSessionsAsUsers should have completed
-        expect(olmMachine.getMissingSessions).toHaveBeenCalledWith([u1]);
+        // check that all the calls were made
+        // We can't use directly toHaveBeenCalledWith because the UserId are cloned in the process.
+        const calledWith = olmMachine.getMissingSessions.mock.calls[0][0].map((u) => u.toString());
+        expect(calledWith).toEqual([u1.toString()]);
         expect(olmMachine.getMissingSessions).toHaveBeenCalledTimes(1);
         expect(fetchMock).toHaveBeenCalledTimes(1);
         expect(req1Resolved).toBe(false);
@@ -147,7 +153,9 @@ describe("KeyClaimManager", () => {
         resolveMarkRequestAsSentCallback = await markRequestAsSentPromise;
 
         // the first request should now have completed, and we should have more calls and fetches
-        expect(olmMachine.getMissingSessions).toHaveBeenCalledWith([u2]);
+        // We can't use directly toHaveBeenCalledWith because the UserId are cloned in the process.
+        const calledWith2 = olmMachine.getMissingSessions.mock.calls[1][0].map((u) => u.toString());
+        expect(calledWith2).toEqual([u2.toString()]);
         expect(olmMachine.getMissingSessions).toHaveBeenCalledTimes(2);
         expect(fetchMock).toHaveBeenCalledTimes(2);
         expect(req1Resolved).toBe(true);

spec/unit/rust-crypto/backup.spec.ts

@@ -0,0 +1,144 @@
+import { Mocked } from "jest-mock";
+import fetchMock from "fetch-mock-jest";
+import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
+
+import { CryptoEvent, HttpApiEvent, HttpApiEventHandlerMap, MatrixHttpApi, TypedEventEmitter } from "../../../src";
+import { OutgoingRequestProcessor } from "../../../src/rust-crypto/OutgoingRequestProcessor";
+import * as testData from "../../test-utils/test-data";
+import * as TestData from "../../test-utils/test-data";
+import { IKeyBackup } from "../../../src/crypto/backup";
+import { IKeyBackupSession } from "../../../src/crypto/keybackup";
+import { RustBackupManager } from "../../../src/rust-crypto/backup";
+
+describe("Upload keys to backup", () => {
+    /** The backup manager under test */
+    let rustBackupManager: RustBackupManager;
+
+    let mockOlmMachine: Mocked<RustSdkCryptoJs.OlmMachine>;
+
+    let outgoingRequestProcessor: Mocked<OutgoingRequestProcessor>;
+
+    const httpAPi = new MatrixHttpApi(new TypedEventEmitter<HttpApiEvent, HttpApiEventHandlerMap>(), {
+        baseUrl: "http://server/",
+        prefix: "",
+        onlyData: true,
+    });
+
+    let idGenerator = 0;
+    function mockBackupRequest(keyCount: number): RustSdkCryptoJs.KeysBackupRequest {
+        const requestBody: IKeyBackup = {
+            rooms: {
+                "!room1:server": {
+                    sessions: {},
+                },
+            },
+        };
+        for (let i = 0; i < keyCount; i++) {
+            requestBody.rooms["!room1:server"].sessions["session" + i] = {} as IKeyBackupSession;
+        }
+        return {
+            id: "id" + idGenerator++,
+            body: JSON.stringify(requestBody),
+        } as unknown as Mocked<RustSdkCryptoJs.KeysBackupRequest>;
+    }
+
+    beforeEach(async () => {
+        jest.useFakeTimers();
+        idGenerator = 0;
+
+        mockOlmMachine = {
+            getBackupKeys: jest.fn().mockResolvedValue({
+                backupVersion: TestData.SIGNED_BACKUP_DATA.version!,
+                decryptionKey: RustSdkCryptoJs.BackupDecryptionKey.fromBase64(TestData.BACKUP_DECRYPTION_KEY_BASE64),
+            } as unknown as RustSdkCryptoJs.BackupKeys),
+            backupRoomKeys: jest.fn(),
+            isBackupEnabled: jest.fn().mockResolvedValue(true),
+            enableBackupV1: jest.fn(),
+            verifyBackup: jest.fn().mockResolvedValue({
+                trusted: jest.fn().mockResolvedValue(true),
+            } as unknown as RustSdkCryptoJs.SignatureVerification),
+            roomKeyCounts: jest.fn(),
+        } as unknown as Mocked<RustSdkCryptoJs.OlmMachine>;
+
+        outgoingRequestProcessor = {
+            makeOutgoingRequest: jest.fn(),
+        } as unknown as Mocked<OutgoingRequestProcessor>;
+
+        rustBackupManager = new RustBackupManager(mockOlmMachine, httpAPi, outgoingRequestProcessor);
+
+        fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+    });
+
+    afterEach(() => {
+        fetchMock.reset();
+        jest.useRealTimers();
+        jest.resetAllMocks();
+    });
+
+    it("Should call expensive roomKeyCounts only once per loop", async () => {
+        const remainingEmitted: number[] = [];
+
+        const zeroRemainingWasEmitted = new Promise<void>((resolve) => {
+            rustBackupManager.on(CryptoEvent.KeyBackupSessionsRemaining, (count) => {
+                remainingEmitted.push(count);
+                if (count == 0) {
+                    resolve();
+                }
+            });
+        });
+
+        // We want several batch of keys to check that we don't call expensive room key count several times
+        mockOlmMachine.backupRoomKeys
+            .mockResolvedValueOnce(mockBackupRequest(100))
+            .mockResolvedValueOnce(mockBackupRequest(100))
+            .mockResolvedValueOnce(mockBackupRequest(100))
+            .mockResolvedValueOnce(mockBackupRequest(100))
+            .mockResolvedValueOnce(mockBackupRequest(100))
+            .mockResolvedValueOnce(mockBackupRequest(100))
+            .mockResolvedValueOnce(mockBackupRequest(2))
+            .mockResolvedValue(null);
+
+        mockOlmMachine.roomKeyCounts.mockResolvedValue({
+            total: 602,
+            // First iteration won't call roomKeyCounts(); it will be called on the second iteration after 200 keys have been saved.
+            backedUp: 200,
+        });
+
+        await rustBackupManager.checkKeyBackupAndEnable(false);
+        await jest.runAllTimersAsync();
+
+        await zeroRemainingWasEmitted;
+
+        expect(outgoingRequestProcessor.makeOutgoingRequest).toHaveBeenCalledTimes(7);
+        expect(mockOlmMachine.roomKeyCounts).toHaveBeenCalledTimes(1);
+
+        // check event emission
+        expect(remainingEmitted[0]).toEqual(402);
+        expect(remainingEmitted[1]).toEqual(302);
+        expect(remainingEmitted[2]).toEqual(202);
+        expect(remainingEmitted[3]).toEqual(102);
+        expect(remainingEmitted[4]).toEqual(2);
+        expect(remainingEmitted[5]).toEqual(0);
+    });
+
+    it("Should not call expensive roomKeyCounts when only one iteration is needed", async () => {
+        const zeroRemainingWasEmitted = new Promise<void>((resolve) => {
+            rustBackupManager.on(CryptoEvent.KeyBackupSessionsRemaining, (count) => {
+                if (count == 0) {
+                    resolve();
+                }
+            });
+        });
+
+        // Only returns 2 keys on the first call, then none.
+        mockOlmMachine.backupRoomKeys.mockResolvedValueOnce(mockBackupRequest(2)).mockResolvedValue(null);
+
+        await rustBackupManager.checkKeyBackupAndEnable(false);
+        await jest.runAllTimersAsync();
+
+        await zeroRemainingWasEmitted;
+
+        expect(outgoingRequestProcessor.makeOutgoingRequest).toHaveBeenCalledTimes(1);
+        expect(mockOlmMachine.roomKeyCounts).toHaveBeenCalledTimes(0);
+    });
+});

spec/unit/rust-crypto/rust-crypto.spec.ts

@@ -15,7 +15,15 @@ limitations under the License.
 */
 
 import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
-import { KeysQueryRequest, OlmMachine } from "@matrix-org/matrix-sdk-crypto-wasm";
+import {
+    BaseMigrationData,
+    KeysQueryRequest,
+    Migration,
+    OlmMachine,
+    PickledInboundGroupSession,
+    PickledSession,
+    StoreHandle,
+} from "@matrix-org/matrix-sdk-crypto-wasm";
 import { mocked, Mocked } from "jest-mock";
 import fetchMock from "fetch-mock-jest";
 
@@ -25,6 +33,7 @@ import {
     CryptoEvent,
     Device,
     DeviceVerification,
+    encodeBase64,
     HttpApiEvent,
     HttpApiEventHandlerMap,
     IHttpOpts,
@@ -32,11 +41,12 @@ import {
     MatrixClient,
     MatrixEvent,
     MatrixHttpApi,
+    MemoryCryptoStore,
     TypedEventEmitter,
 } from "../../../src";
 import { mkEvent } from "../../test-utils/test-utils";
 import { CryptoBackend } from "../../../src/common-crypto/CryptoBackend";
-import { IEventDecryptionResult } from "../../../src/@types/crypto";
+import { IEventDecryptionResult, IMegolmSessionData } from "../../../src/@types/crypto";
 import { OutgoingRequestProcessor } from "../../../src/rust-crypto/OutgoingRequestProcessor";
 import {
     AccountDataClient,
@@ -59,10 +69,18 @@ import { logger } from "../../../src/logger";
 import { OutgoingRequestsManager } from "../../../src/rust-crypto/OutgoingRequestsManager";
 import { ClientEvent, ClientEventHandlerMap } from "../../../src/client";
 import { Curve25519AuthData } from "../../../src/crypto-api/keybackup";
+import { encryptAES } from "../../../src/crypto/aes";
+import { CryptoStore, SecretStorePrivateKeys } from "../../../src/crypto/store/base";
 
 const TEST_USER = "@alice:example.com";
 const TEST_DEVICE_ID = "TEST_DEVICE";
 
+beforeAll(async () => {
+    // Load the WASM upfront, before any of the tests. This can take some time, and doing it here means that it gets
+    // a separate timeout.
+    await RustSdkCryptoJs.initAsync();
+}, 15000);
+
 afterEach(() => {
     fetchMock.reset();
     jest.restoreAllMocks();
@@ -73,16 +91,23 @@ describe("initRustCrypto", () => {
         return {
             registerRoomKeyUpdatedCallback: jest.fn(),
             registerUserIdentityUpdatedCallback: jest.fn(),
-            getSecretsFromInbox: jest.fn().mockResolvedValue(["dGhpc2lzYWZha2VzZWNyZXQ="]),
+            getSecretsFromInbox: jest.fn().mockResolvedValue([]),
             deleteSecretsFromInbox: jest.fn(),
             registerReceiveSecretCallback: jest.fn(),
+            registerDevicesUpdatedCallback: jest.fn(),
             outgoingRequests: jest.fn(),
+            isBackupEnabled: jest.fn().mockResolvedValue(false),
+            verifyBackup: jest.fn().mockResolvedValue({ trusted: jest.fn().mockReturnValue(false) }),
+            getBackupKeys: jest.fn(),
         } as unknown as Mocked<OlmMachine>;
     }
 
     it("passes through the store params", async () => {
+        const mockStore = { free: jest.fn() } as unknown as StoreHandle;
+        jest.spyOn(StoreHandle, "open").mockResolvedValue(mockStore);
+
         const testOlmMachine = makeTestOlmMachine();
-        jest.spyOn(OlmMachine, "initialize").mockResolvedValue(testOlmMachine);
+        jest.spyOn(OlmMachine, "initFromStore").mockResolvedValue(testOlmMachine);
 
         await initRustCrypto({
             logger,
@@ -95,17 +120,16 @@ describe("initRustCrypto", () => {
             storePassphrase: "storePassphrase",
         });
 
-        expect(OlmMachine.initialize).toHaveBeenCalledWith(
-            expect.anything(),
-            expect.anything(),
-            "storePrefix",
-            "storePassphrase",
-        );
+        expect(StoreHandle.open).toHaveBeenCalledWith("storePrefix", "storePassphrase");
+        expect(OlmMachine.initFromStore).toHaveBeenCalledWith(expect.anything(), expect.anything(), mockStore);
     });
 
     it("suppresses the storePassphrase if storePrefix is unset", async () => {
+        const mockStore = { free: jest.fn() } as unknown as StoreHandle;
+        jest.spyOn(StoreHandle, "open").mockResolvedValue(mockStore);
+
         const testOlmMachine = makeTestOlmMachine();
-        jest.spyOn(OlmMachine, "initialize").mockResolvedValue(testOlmMachine);
+        jest.spyOn(OlmMachine, "initFromStore").mockResolvedValue(testOlmMachine);
 
         await initRustCrypto({
             logger,
@@ -118,12 +142,16 @@ describe("initRustCrypto", () => {
             storePassphrase: "storePassphrase",
         });
 
-        expect(OlmMachine.initialize).toHaveBeenCalledWith(expect.anything(), expect.anything(), undefined, undefined);
+        expect(StoreHandle.open).toHaveBeenCalledWith(undefined, undefined);
+        expect(OlmMachine.initFromStore).toHaveBeenCalledWith(expect.anything(), expect.anything(), mockStore);
     });
 
     it("Should get secrets from inbox on start", async () => {
-        const testOlmMachine = makeTestOlmMachine() as OlmMachine;
-        jest.spyOn(OlmMachine, "initialize").mockResolvedValue(testOlmMachine);
+        const mockStore = { free: jest.fn() } as unknown as StoreHandle;
+        jest.spyOn(StoreHandle, "open").mockResolvedValue(mockStore);
+
+        const testOlmMachine = makeTestOlmMachine();
+        jest.spyOn(OlmMachine, "initFromStore").mockResolvedValue(testOlmMachine);
 
         await initRustCrypto({
             logger,
@@ -138,6 +166,212 @@ describe("initRustCrypto", () => {
 
         expect(testOlmMachine.getSecretsFromInbox).toHaveBeenCalledWith("m.megolm_backup.v1");
     });
+
+    describe("libolm migration", () => {
+        let mockStore: RustSdkCryptoJs.StoreHandle;
+
+        beforeEach(() => {
+            // Stub out a bunch of stuff in the Rust library
+            mockStore = { free: jest.fn() } as unknown as StoreHandle;
+            jest.spyOn(StoreHandle, "open").mockResolvedValue(mockStore);
+
+            jest.spyOn(Migration, "migrateBaseData").mockResolvedValue(undefined);
+            jest.spyOn(Migration, "migrateOlmSessions").mockResolvedValue(undefined);
+            jest.spyOn(Migration, "migrateMegolmSessions").mockResolvedValue(undefined);
+
+            const testOlmMachine = makeTestOlmMachine();
+            jest.spyOn(OlmMachine, "initFromStore").mockResolvedValue(testOlmMachine);
+        });
+
+        it("migrates data from a legacy crypto store", async () => {
+            const PICKLE_KEY = "pickle1234";
+            const legacyStore = new MemoryCryptoStore();
+
+            // Populate the legacy store with some test data
+            const storeSecretKey = (type: string, key: string) =>
+                encryptAndStoreSecretKey(type, new TextEncoder().encode(key), PICKLE_KEY, legacyStore);
+
+            await legacyStore.storeAccount({}, "not a real account");
+            await storeSecretKey("m.megolm_backup.v1", "backup key");
+            await storeSecretKey("master", "master key");
+            await storeSecretKey("self_signing", "ssk");
+            await storeSecretKey("user_signing", "usk");
+            const nDevices = 6;
+            const nSessionsPerDevice = 10;
+            createSessions(legacyStore, nDevices, nSessionsPerDevice);
+            createMegolmSessions(legacyStore, nDevices, nSessionsPerDevice);
+            await legacyStore.markSessionsNeedingBackup([{ senderKey: pad43("device5"), sessionId: "session5" }]);
+
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", { version: "45" });
+
+            function legacyMigrationProgressListener(progress: number, total: number): void {
+                logger.log(`migrated ${progress} of ${total}`);
+            }
+
+            await initRustCrypto({
+                logger,
+                http: makeMatrixHttpApi(),
+                userId: TEST_USER,
+                deviceId: TEST_DEVICE_ID,
+                secretStorage: {} as ServerSideSecretStorage,
+                cryptoCallbacks: {} as CryptoCallbacks,
+                storePrefix: "storePrefix",
+                storePassphrase: "storePassphrase",
+                legacyCryptoStore: legacyStore,
+                legacyPickleKey: PICKLE_KEY,
+                legacyMigrationProgressListener,
+            });
+
+            // Check that the migration functions were correctly called
+            expect(Migration.migrateBaseData).toHaveBeenCalledWith(
+                expect.any(BaseMigrationData),
+                new Uint8Array(Buffer.from(PICKLE_KEY)),
+                mockStore,
+            );
+            const data = mocked(Migration.migrateBaseData).mock.calls[0][0];
+            expect(data.pickledAccount).toEqual("not a real account");
+            expect(data.userId!.toString()).toEqual(TEST_USER);
+            expect(data.deviceId!.toString()).toEqual(TEST_DEVICE_ID);
+            expect(atob(data.backupRecoveryKey!)).toEqual("backup key");
+            expect(data.backupVersion).toEqual("45");
+            expect(atob(data.privateCrossSigningMasterKey!)).toEqual("master key");
+            expect(atob(data.privateCrossSigningUserSigningKey!)).toEqual("usk");
+            expect(atob(data.privateCrossSigningSelfSigningKey!)).toEqual("ssk");
+
+            expect(Migration.migrateOlmSessions).toHaveBeenCalledTimes(2);
+            expect(Migration.migrateOlmSessions).toHaveBeenCalledWith(
+                expect.any(Array),
+                new Uint8Array(Buffer.from(PICKLE_KEY)),
+                mockStore,
+            );
+            // First call should have 50 entries; second should have 10
+            const sessions1: PickledSession[] = mocked(Migration.migrateOlmSessions).mock.calls[0][0];
+            expect(sessions1.length).toEqual(50);
+            const sessions2: PickledSession[] = mocked(Migration.migrateOlmSessions).mock.calls[1][0];
+            expect(sessions2.length).toEqual(10);
+            const sessions = [...sessions1, ...sessions2];
+            for (let i = 0; i < nDevices; i++) {
+                for (let j = 0; j < nSessionsPerDevice; j++) {
+                    const session = sessions[i * nSessionsPerDevice + j];
+                    expect(session.senderKey).toEqual(`device${i}`);
+                    expect(session.pickle).toEqual(`session${i}.${j}`);
+                    expect(session.creationTime).toEqual(new Date(1000));
+                    expect(session.lastUseTime).toEqual(new Date(1000));
+                }
+            }
+
+            expect(Migration.migrateMegolmSessions).toHaveBeenCalledTimes(2);
+            expect(Migration.migrateMegolmSessions).toHaveBeenCalledWith(
+                expect.any(Array),
+                new Uint8Array(Buffer.from(PICKLE_KEY)),
+                mockStore,
+            );
+            // First call should have 50 entries; second should have 10
+            const megolmSessions1: PickledInboundGroupSession[] = mocked(Migration.migrateMegolmSessions).mock
+                .calls[0][0];
+            expect(megolmSessions1.length).toEqual(50);
+            const megolmSessions2: PickledInboundGroupSession[] = mocked(Migration.migrateMegolmSessions).mock
+                .calls[1][0];
+            expect(megolmSessions2.length).toEqual(10);
+            const megolmSessions = [...megolmSessions1, ...megolmSessions2];
+            for (let i = 0; i < nDevices; i++) {
+                for (let j = 0; j < nSessionsPerDevice; j++) {
+                    const session = megolmSessions[i * nSessionsPerDevice + j];
+                    expect(session.senderKey).toEqual(pad43(`device${i}`));
+                    expect(session.pickle).toEqual("sessionPickle");
+                    expect(session.roomId!.toString()).toEqual("!room:id");
+                    expect(session.senderSigningKey).toEqual("sender_signing_key");
+
+                    // only one of the sessions needs backing up
+                    expect(session.backedUp).toEqual(i !== 5 || j !== 5);
+                }
+            }
+        }, 10000);
+
+        it("handles megolm sessions with no `keysClaimed`", async () => {
+            const legacyStore = new MemoryCryptoStore();
+            legacyStore.storeAccount({}, "not a real account");
+
+            legacyStore.storeEndToEndInboundGroupSession(
+                pad43(`device1`),
+                `session1`,
+                {
+                    forwardingCurve25519KeyChain: [],
+                    room_id: "!room:id",
+                    session: "sessionPickle",
+                },
+                undefined,
+            );
+
+            const PICKLE_KEY = "pickle1234";
+            await initRustCrypto({
+                logger,
+                http: makeMatrixHttpApi(),
+                userId: TEST_USER,
+                deviceId: TEST_DEVICE_ID,
+                secretStorage: {} as ServerSideSecretStorage,
+                cryptoCallbacks: {} as CryptoCallbacks,
+                storePrefix: "storePrefix",
+                storePassphrase: "storePassphrase",
+                legacyCryptoStore: legacyStore,
+                legacyPickleKey: PICKLE_KEY,
+            });
+
+            expect(Migration.migrateMegolmSessions).toHaveBeenCalledTimes(1);
+            expect(Migration.migrateMegolmSessions).toHaveBeenCalledWith(
+                expect.any(Array),
+                new Uint8Array(Buffer.from(PICKLE_KEY)),
+                mockStore,
+            );
+            const megolmSessions: PickledInboundGroupSession[] = mocked(Migration.migrateMegolmSessions).mock
+                .calls[0][0];
+            expect(megolmSessions.length).toEqual(1);
+            const session = megolmSessions[0];
+            expect(session.senderKey).toEqual(pad43(`device1`));
+            expect(session.pickle).toEqual("sessionPickle");
+            expect(session.roomId!.toString()).toEqual("!room:id");
+            expect(session.senderSigningKey).toBe(undefined);
+        }, 10000);
+
+        async function encryptAndStoreSecretKey(type: string, key: Uint8Array, pickleKey: string, store: CryptoStore) {
+            const encryptedKey = await encryptAES(encodeBase64(key), Buffer.from(pickleKey), type);
+            store.storeSecretStorePrivateKey(undefined, type as keyof SecretStorePrivateKeys, encryptedKey);
+        }
+
+        /** Create a bunch of fake Olm sessions and stash them in the DB. */
+        function createSessions(store: CryptoStore, nDevices: number, nSessionsPerDevice: number) {
+            for (let i = 0; i < nDevices; i++) {
+                for (let j = 0; j < nSessionsPerDevice; j++) {
+                    const sessionData = {
+                        deviceKey: `device${i}`,
+                        sessionId: `session${j}`,
+                        session: `session${i}.${j}`,
+                        lastReceivedMessageTs: 1000,
+                    };
+                    store.storeEndToEndSession(`device${i}`, `session${j}`, sessionData, undefined);
+                }
+            }
+        }
+
+        /** Create a bunch of fake Megolm sessions and stash them in the DB. */
+        function createMegolmSessions(store: CryptoStore, nDevices: number, nSessionsPerDevice: number) {
+            for (let i = 0; i < nDevices; i++) {
+                for (let j = 0; j < nSessionsPerDevice; j++) {
+                    store.storeEndToEndInboundGroupSession(
+                        pad43(`device${i}`),
+                        `session${j}`,
+                        {
+                            forwardingCurve25519KeyChain: [],
+                            keysClaimed: { ed25519: "sender_signing_key" },
+                            room_id: "!room:id",
+                            session: "sessionPickle",
+                        },
+                        undefined,
+                    );
+                }
+            }
+        }
+    });
 });
 
 describe("RustCrypto", () => {
@@ -165,7 +399,7 @@ describe("RustCrypto", () => {
             let importTotal = 0;
             const opt: ImportRoomKeysOpts = {
                 progressCallback: (stage) => {
-                    importTotal = stage.total;
+                    importTotal = stage.total ?? 0;
                 },
             };
             await rustCrypto.importRoomKeys(someRoomKeys, opt);
@@ -463,6 +697,58 @@ describe("RustCrypto", () => {
             await awaitCallToMakeOutgoingRequest();
             expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(2);
         });
+
+        it("should encode outgoing requests properly", async () => {
+            // we need a real OlmMachine, so replace the one created by beforeEach
+            rustCrypto = await makeTestRustCrypto();
+            const olmMachine: OlmMachine = rustCrypto["olmMachine"];
+
+            const outgoingRequestProcessor = {} as unknown as OutgoingRequestProcessor;
+            rustCrypto["outgoingRequestProcessor"] = outgoingRequestProcessor;
+            const outgoingRequestsManager = new OutgoingRequestsManager(logger, olmMachine, outgoingRequestProcessor);
+            rustCrypto["outgoingRequestsManager"] = outgoingRequestsManager;
+
+            // The second time we do a /keys/upload, the `device_keys` property
+            // should be absent from the request body
+            // cf. https://github.com/matrix-org/matrix-rust-sdk-crypto-wasm/issues/57
+            //
+            // On the first upload, we pretend that there are no OTKs, so it will
+            // try to upload more keys
+            let keysUploadCount = 0;
+            let deviceKeys: object;
+            let deviceKeysAbsent = false;
+            outgoingRequestProcessor.makeOutgoingRequest = jest.fn(async (request, uiaCallback?) => {
+                let resp: any = {};
+                if (request instanceof RustSdkCryptoJs.KeysUploadRequest) {
+                    if (keysUploadCount == 0) {
+                        deviceKeys = JSON.parse(request.body).device_keys;
+                        resp = { one_time_key_counts: { signed_curve25519: 0 } };
+                    } else {
+                        deviceKeysAbsent = !("device_keys" in JSON.parse(request.body));
+                        resp = { one_time_key_counts: { signed_curve25519: 50 } };
+                    }
+                    keysUploadCount++;
+                } else if (request instanceof RustSdkCryptoJs.KeysQueryRequest) {
+                    resp = {
+                        device_keys: {
+                            [TEST_USER]: {
+                                [TEST_DEVICE_ID]: deviceKeys,
+                            },
+                        },
+                    };
+                } else if (request instanceof RustSdkCryptoJs.UploadSigningKeysRequest) {
+                    // SigningKeysUploadRequest does not implement OutgoingRequest and does not need to be marked as sent.
+                    return;
+                }
+                if (request.id) {
+                    olmMachine.markRequestAsSent(request.id, request.type, JSON.stringify(resp));
+                }
+            });
+            await outgoingRequestsManager.doProcessOutgoingRequests();
+            await outgoingRequestsManager.doProcessOutgoingRequests();
+
+            expect(deviceKeysAbsent).toBe(true);
+        });
     });
 
     describe(".getEventEncryptionInfo", () => {
@@ -821,7 +1107,8 @@ describe("RustCrypto", () => {
     });
 
     it("should wait for a keys/query before returning devices", async () => {
-        jest.useFakeTimers();
+        // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+        jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
         fetchMock.post("path:/_matrix/client/v3/keys/upload", { one_time_key_counts: {} });
         fetchMock.post("path:/_matrix/client/v3/keys/query", {
@@ -846,6 +1133,33 @@ describe("RustCrypto", () => {
         rustCrypto.stop();
     });
 
+    it("should emit events on device changes", async () => {
+        jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
+
+        fetchMock.post("path:/_matrix/client/v3/keys/upload", { one_time_key_counts: {} });
+        fetchMock.post("path:/_matrix/client/v3/keys/query", {
+            device_keys: {
+                [testData.TEST_USER_ID]: {
+                    [testData.TEST_DEVICE_ID]: testData.SIGNED_TEST_DEVICE_DATA,
+                },
+            },
+        });
+
+        const rustCrypto = await makeTestRustCrypto(makeMatrixHttpApi(), testData.TEST_USER_ID);
+        const willUpdateCallback = jest.fn();
+        rustCrypto.on(CryptoEvent.WillUpdateDevices, willUpdateCallback);
+        const devicesUpdatedCallback = jest.fn();
+        rustCrypto.on(CryptoEvent.DevicesUpdated, devicesUpdatedCallback);
+
+        rustCrypto.onSyncCompleted({});
+
+        // wait for the devices to be updated
+        await rustCrypto.getUserDeviceInfo([testData.TEST_USER_ID]);
+        expect(willUpdateCallback).toHaveBeenCalledWith([testData.TEST_USER_ID], false);
+        expect(devicesUpdatedCallback).toHaveBeenCalledWith([testData.TEST_USER_ID], false);
+        rustCrypto.stop();
+    });
+
     describe("requestDeviceVerification", () => {
         it("throws an error if the device is unknown", async () => {
             const rustCrypto = await makeTestRustCrypto();
@@ -1026,6 +1340,34 @@ describe("RustCrypto", () => {
                 },
             });
         });
+
+        it("ignores invalid keys when restoring from backup", async () => {
+            const rustCrypto = await makeTestRustCrypto();
+            const olmMachine: OlmMachine = rustCrypto["olmMachine"];
+
+            await olmMachine.enableBackupV1(
+                (testData.SIGNED_BACKUP_DATA.auth_data as Curve25519AuthData).public_key,
+                testData.SIGNED_BACKUP_DATA.version!,
+            );
+
+            const backup = Array.from(testData.MEGOLM_SESSION_DATA_ARRAY);
+            // in addition to correct keys, we restore an invalid key
+            backup.push({ room_id: "!roomid", session_id: "sessionid" } as IMegolmSessionData);
+            const progressCallback = jest.fn();
+            await rustCrypto.importBackedUpRoomKeys(backup, { progressCallback });
+            expect(progressCallback).toHaveBeenCalledWith({
+                total: 3,
+                successes: 0,
+                stage: "load_keys",
+                failures: 1,
+            });
+            expect(progressCallback).toHaveBeenCalledWith({
+                total: 3,
+                successes: 1,
+                stage: "load_keys",
+                failures: 1,
+            });
+        });
     });
 });
 
@@ -1095,3 +1437,8 @@ class DummyAccountDataClient
         return {};
     }
 }
+
+/** Pad a string to 43 characters long */
+function pad43(x: string): string {
+    return x + ".".repeat(43 - x.length);
+}

spec/unit/rust-crypto/verification.spec.ts

@@ -17,8 +17,19 @@ limitations under the License.
 import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
 import { Mocked } from "jest-mock";
 
-import { isVerificationEvent, RustVerificationRequest } from "../../../src/rust-crypto/verification";
-import { OutgoingRequestProcessor } from "../../../src/rust-crypto/OutgoingRequestProcessor";
+import {
+    isVerificationEvent,
+    RustVerificationRequest,
+    verificationMethodIdentifierToMethod,
+} from "../../../src/rust-crypto/verification";
+import {
+    ShowSasCallbacks,
+    VerificationRequestEvent,
+    Verifier,
+    VerifierEvent,
+} from "../../../src/crypto-api/verification";
+import { OutgoingRequest, OutgoingRequestProcessor } from "../../../src/rust-crypto/OutgoingRequestProcessor";
+import { IDeviceKeys } from "../../../src/@types/crypto";
 import { EventType, MatrixEvent, MsgType } from "../../../src";
 
 describe("VerificationRequest", () => {
@@ -91,6 +102,354 @@ describe("VerificationRequest", () => {
             );
         });
     });
+
+    it("can verify with SAS", async () => {
+        const aliceUserId = "@alice:example.org";
+        const aliceDeviceId = "ABCDEFG";
+        const bobUserId = "@bob:example.org";
+        const bobDeviceId = "HIJKLMN";
+        const [aliceOlmMachine, aliceDeviceKeys, aliceCrossSigningKeys] = await initOlmMachineAndKeys(
+            aliceUserId,
+            aliceDeviceId,
+        );
+        const [bobOlmMachine, bobDeviceKeys, bobCrossSigningKeys] = await initOlmMachineAndKeys(bobUserId, bobDeviceId);
+
+        const aliceRequestLoop = makeRequestLoop(
+            aliceOlmMachine,
+            aliceDeviceKeys,
+            aliceCrossSigningKeys,
+            bobOlmMachine,
+            bobDeviceKeys,
+            bobCrossSigningKeys,
+        );
+        const bobRequestLoop = makeRequestLoop(
+            bobOlmMachine,
+            bobDeviceKeys,
+            bobCrossSigningKeys,
+            aliceOlmMachine,
+            aliceDeviceKeys,
+            aliceCrossSigningKeys,
+        );
+
+        try {
+            await aliceOlmMachine.updateTrackedUsers([new RustSdkCryptoJs.UserId(bobUserId)]);
+            await bobOlmMachine.updateTrackedUsers([new RustSdkCryptoJs.UserId(aliceUserId)]);
+
+            // Alice requests verification
+            const bobUserIdentity = await aliceOlmMachine.getIdentity(new RustSdkCryptoJs.UserId(bobUserId));
+
+            const roomId = new RustSdkCryptoJs.RoomId("!roomId:example.org");
+            const methods = [verificationMethodIdentifierToMethod("m.sas.v1")];
+            const innerVerificationRequest = await bobUserIdentity.requestVerification(
+                roomId,
+                new RustSdkCryptoJs.EventId("$m.key.verification.request"),
+                methods,
+            );
+            const aliceVerificationRequest = new RustVerificationRequest(
+                aliceOlmMachine,
+                innerVerificationRequest,
+                aliceRequestLoop as unknown as OutgoingRequestProcessor,
+                ["m.sas.v1"],
+            );
+
+            const verificationRequestContent = JSON.parse(await bobUserIdentity.verificationRequestContent(methods));
+            await bobOlmMachine.receiveVerificationEvent(
+                JSON.stringify({
+                    type: "m.room.message",
+                    sender: aliceUserId,
+                    event_id: "$m.key.verification.request",
+                    content: verificationRequestContent,
+                    origin_server_ts: Date.now(),
+                    unsigned: {
+                        age: 0,
+                    },
+                }),
+                roomId,
+            );
+
+            // Bob accepts
+            const bobInnerVerificationRequest = bobOlmMachine.getVerificationRequest(
+                new RustSdkCryptoJs.UserId(aliceUserId),
+                "$m.key.verification.request",
+            )!;
+            const bobVerificationRequest = new RustVerificationRequest(
+                bobOlmMachine,
+                bobInnerVerificationRequest,
+                bobRequestLoop as unknown as OutgoingRequestProcessor,
+                ["m.sas.v1"],
+            );
+
+            await bobVerificationRequest.accept();
+
+            // Alice starts the verification
+            const bobVerifierPromise: Promise<Verifier> = new Promise((resolve, reject) => {
+                bobVerificationRequest.on(VerificationRequestEvent.Change, () => {
+                    const verifier = bobVerificationRequest.verifier;
+                    if (verifier) {
+                        resolve(verifier);
+                    }
+                });
+            });
+            const aliceVerifier = await aliceVerificationRequest.startVerification("m.sas.v1");
+            const bobVerifier = await bobVerifierPromise;
+
+            // create a function to compare the SAS, and then let the verification run
+            let otherCallbacks: ShowSasCallbacks | undefined;
+            const compareSas = (callbacks: ShowSasCallbacks): void => {
+                if (otherCallbacks) {
+                    const ourDecimal = callbacks.sas.decimal!;
+                    const theirDecimal = otherCallbacks.sas.decimal!;
+                    if (ourDecimal.every((el, idx) => el == theirDecimal[idx])) {
+                        otherCallbacks.confirm();
+                        callbacks.confirm();
+                    } else {
+                        otherCallbacks.mismatch();
+                        callbacks.mismatch();
+                    }
+                } else {
+                    otherCallbacks = callbacks;
+                }
+            };
+            aliceVerifier.on(VerifierEvent.ShowSas, compareSas);
+            bobVerifier.on(VerifierEvent.ShowSas, compareSas);
+
+            await Promise.all([aliceVerifier.verify(), await bobVerifier.verify()]);
+        } finally {
+            await aliceRequestLoop.stop();
+            await bobRequestLoop.stop();
+        }
+    });
+
+    it("can handle simultaneous starts in SAS", async () => {
+        const aliceUserId = "@alice:example.org";
+        const aliceDeviceId = "ABCDEFG";
+        const bobUserId = "@bob:example.org";
+        const bobDeviceId = "HIJKLMN";
+        const [aliceOlmMachine, aliceDeviceKeys, aliceCrossSigningKeys] = await initOlmMachineAndKeys(
+            aliceUserId,
+            aliceDeviceId,
+        );
+        const [bobOlmMachine, bobDeviceKeys, bobCrossSigningKeys] = await initOlmMachineAndKeys(bobUserId, bobDeviceId);
+
+        let aliceStartRequest: RustSdkCryptoJs.RoomMessageRequest | undefined;
+        const aliceRequestLoop = makeRequestLoop(
+            aliceOlmMachine,
+            aliceDeviceKeys,
+            aliceCrossSigningKeys,
+            bobOlmMachine,
+            bobDeviceKeys,
+            bobCrossSigningKeys,
+            async (request): Promise<any> => {
+                // If the request is sending the m.key.verification.start
+                // event, we delay sending it until after Bob has also started
+                // a verification
+                if (
+                    !aliceStartRequest &&
+                    request instanceof RustSdkCryptoJs.RoomMessageRequest &&
+                    request.event_type == "m.key.verification.start"
+                ) {
+                    aliceStartRequest = request;
+                    return { event_id: "$m.key.verification.start" };
+                }
+            },
+        );
+        const bobRequestLoop = makeRequestLoop(
+            bobOlmMachine,
+            bobDeviceKeys,
+            bobCrossSigningKeys,
+            aliceOlmMachine,
+            aliceDeviceKeys,
+            aliceCrossSigningKeys,
+        );
+
+        try {
+            await aliceOlmMachine.updateTrackedUsers([new RustSdkCryptoJs.UserId(bobUserId)]);
+            await bobOlmMachine.updateTrackedUsers([new RustSdkCryptoJs.UserId(aliceUserId)]);
+
+            // Alice requests verification
+            const bobUserIdentity = await aliceOlmMachine.getIdentity(new RustSdkCryptoJs.UserId(bobUserId));
+
+            const roomId = new RustSdkCryptoJs.RoomId("!roomId:example.org");
+            const methods = [verificationMethodIdentifierToMethod("m.sas.v1")];
+            const innerVerificationRequest = await bobUserIdentity.requestVerification(
+                roomId,
+                new RustSdkCryptoJs.EventId("$m.key.verification.request"),
+                methods,
+            );
+            const aliceVerificationRequest = new RustVerificationRequest(
+                aliceOlmMachine,
+                innerVerificationRequest,
+                aliceRequestLoop as unknown as OutgoingRequestProcessor,
+                ["m.sas.v1"],
+            );
+
+            const verificationRequestContent = JSON.parse(await bobUserIdentity.verificationRequestContent(methods));
+            await bobOlmMachine.receiveVerificationEvent(
+                JSON.stringify({
+                    type: "m.room.message",
+                    sender: aliceUserId,
+                    event_id: "$m.key.verification.request",
+                    content: verificationRequestContent,
+                    origin_server_ts: Date.now(),
+                    unsigned: {
+                        age: 0,
+                    },
+                }),
+                roomId,
+            );
+
+            // Bob accepts
+            const bobInnerVerificationRequest = bobOlmMachine.getVerificationRequest(
+                new RustSdkCryptoJs.UserId(aliceUserId),
+                "$m.key.verification.request",
+            )!;
+            const bobVerificationRequest = new RustVerificationRequest(
+                bobOlmMachine,
+                bobInnerVerificationRequest,
+                bobRequestLoop as unknown as OutgoingRequestProcessor,
+                ["m.sas.v1"],
+            );
+
+            await bobVerificationRequest.accept();
+
+            // Alice and Bob both start the verification
+            const aliceVerifier = await aliceVerificationRequest.startVerification("m.sas.v1");
+            const bobVerifier = await bobVerificationRequest.startVerification("m.sas.v1");
+            // We can now send Alice's start message to Bob
+            await aliceRequestLoop.makeOutgoingRequest(aliceStartRequest!);
+
+            // create a function to compare the SAS, and then let the verification run
+            let otherCallbacks: ShowSasCallbacks | undefined;
+            const compareSas = (callbacks: ShowSasCallbacks) => {
+                if (otherCallbacks) {
+                    const ourDecimal = callbacks.sas.decimal!;
+                    const theirDecimal = otherCallbacks.sas.decimal!;
+                    if (ourDecimal.every((el, idx) => el == theirDecimal[idx])) {
+                        otherCallbacks.confirm();
+                        callbacks.confirm();
+                    } else {
+                        otherCallbacks.mismatch();
+                        callbacks.mismatch();
+                    }
+                } else {
+                    otherCallbacks = callbacks;
+                }
+            };
+            aliceVerifier.on(VerifierEvent.ShowSas, compareSas);
+            bobVerifier.on(VerifierEvent.ShowSas, compareSas);
+
+            await Promise.all([aliceVerifier.verify(), await bobVerifier.verify()]);
+        } finally {
+            await aliceRequestLoop.stop();
+            await bobRequestLoop.stop();
+        }
+    });
+
+    it("can verify by QR code", async () => {
+        const aliceUserId = "@alice:example.org";
+        const aliceDeviceId = "ABCDEFG";
+        const bobUserId = "@bob:example.org";
+        const bobDeviceId = "HIJKLMN";
+        const [aliceOlmMachine, aliceDeviceKeys, aliceCrossSigningKeys] = await initOlmMachineAndKeys(
+            aliceUserId,
+            aliceDeviceId,
+        );
+        const [bobOlmMachine, bobDeviceKeys, bobCrossSigningKeys] = await initOlmMachineAndKeys(bobUserId, bobDeviceId);
+
+        const aliceRequestLoop = makeRequestLoop(
+            aliceOlmMachine,
+            aliceDeviceKeys,
+            aliceCrossSigningKeys,
+            bobOlmMachine,
+            bobDeviceKeys,
+            bobCrossSigningKeys,
+        );
+        const bobRequestLoop = makeRequestLoop(
+            bobOlmMachine,
+            bobDeviceKeys,
+            bobCrossSigningKeys,
+            aliceOlmMachine,
+            aliceDeviceKeys,
+            aliceCrossSigningKeys,
+        );
+
+        try {
+            await aliceOlmMachine.updateTrackedUsers([new RustSdkCryptoJs.UserId(bobUserId)]);
+            await bobOlmMachine.updateTrackedUsers([new RustSdkCryptoJs.UserId(aliceUserId)]);
+
+            // Alice requests verification
+            const bobUserIdentity = await aliceOlmMachine.getIdentity(new RustSdkCryptoJs.UserId(bobUserId));
+
+            const roomId = new RustSdkCryptoJs.RoomId("!roomId:example.org");
+            const methods = [
+                verificationMethodIdentifierToMethod("m.reciprocate.v1"),
+                verificationMethodIdentifierToMethod("m.qr_code.show.v1"),
+            ];
+            const innerVerificationRequest = await bobUserIdentity.requestVerification(
+                roomId,
+                new RustSdkCryptoJs.EventId("$m.key.verification.request"),
+                methods,
+            );
+            const aliceVerificationRequest = new RustVerificationRequest(
+                aliceOlmMachine,
+                innerVerificationRequest,
+                aliceRequestLoop as unknown as OutgoingRequestProcessor,
+                ["m.reciprocate.v1", "m.qr_code.show.v1"],
+            );
+
+            const verificationRequestContent = JSON.parse(await bobUserIdentity.verificationRequestContent(methods));
+            await bobOlmMachine.receiveVerificationEvent(
+                JSON.stringify({
+                    type: "m.room.message",
+                    sender: aliceUserId,
+                    event_id: "$m.key.verification.request",
+                    content: verificationRequestContent,
+                    origin_server_ts: Date.now(),
+                    unsigned: {
+                        age: 0,
+                    },
+                }),
+                roomId,
+            );
+
+            // Bob accepts
+            const bobInnerVerificationRequest = bobOlmMachine.getVerificationRequest(
+                new RustSdkCryptoJs.UserId(aliceUserId),
+                "$m.key.verification.request",
+            )!;
+            const bobVerificationRequest = new RustVerificationRequest(
+                bobOlmMachine,
+                bobInnerVerificationRequest,
+                bobRequestLoop as unknown as OutgoingRequestProcessor,
+                ["m.reciprocate.v1", "m.qr_code.show.v1", "m.qr_code.scan.v1"],
+            );
+
+            await bobVerificationRequest.accept();
+
+            // Bob scans
+            const qrCode = await aliceVerificationRequest.generateQRCode();
+
+            const aliceVerifierPromise: Promise<Verifier> = new Promise((resolve, reject) => {
+                aliceVerificationRequest.on(VerificationRequestEvent.Change, () => {
+                    const verifier = aliceVerificationRequest.verifier;
+                    if (verifier) {
+                        resolve(verifier);
+                    }
+                });
+            });
+            const bobVerifier = await bobVerificationRequest.scanQRCode(qrCode!);
+
+            const aliceVerifier = await aliceVerifierPromise;
+            aliceVerifier.on(VerifierEvent.ShowReciprocateQr, (showQrCodeCallbacks) => {
+                showQrCodeCallbacks.confirm();
+            });
+
+            await Promise.all([aliceVerifier.verify(), await bobVerifier.verify()]);
+        } finally {
+            await aliceRequestLoop.stop();
+            await bobRequestLoop.stop();
+        }
+    });
 });
 
 describe("isVerificationEvent", () => {
@@ -152,3 +511,148 @@ function makeMockedInner(): Mocked<RustSdkCryptoJs.VerificationRequest> {
         },
     } as unknown as Mocked<RustSdkCryptoJs.VerificationRequest>;
 }
+
+interface CrossSigningKeys {
+    master_key: any;
+    self_signing_key: any;
+    user_signing_key: any;
+}
+
+/** create an Olm machine and device/cross-signing keys for a user */
+async function initOlmMachineAndKeys(
+    userId: string,
+    deviceId: string,
+): Promise<[RustSdkCryptoJs.OlmMachine, IDeviceKeys, CrossSigningKeys]> {
+    const olmMachine = await RustSdkCryptoJs.OlmMachine.initialize(
+        new RustSdkCryptoJs.UserId(userId),
+        new RustSdkCryptoJs.DeviceId(deviceId),
+        undefined,
+        undefined,
+    );
+    const { uploadKeysRequest, uploadSignaturesRequest, uploadSigningKeysRequest } =
+        await olmMachine.bootstrapCrossSigning(true);
+    const deviceKeys = JSON.parse(uploadKeysRequest.body).device_keys;
+    await olmMachine.markRequestAsSent(
+        uploadKeysRequest.id,
+        uploadKeysRequest.type,
+        '{"one_time_key_counts":{"signed_curve25519":100}}',
+    );
+    const crossSigningSignatures = JSON.parse(uploadSignaturesRequest.body);
+    for (const [keyId, signature] of Object.entries(crossSigningSignatures[userId][deviceId]["signatures"][userId])) {
+        deviceKeys["signatures"][userId][keyId] = signature;
+    }
+    const crossSigningKeys = JSON.parse(uploadSigningKeysRequest.body);
+    // note: the upload signatures request and upload signing keys requests
+    // don't need to be marked as sent in the Olm machine
+
+    return [olmMachine, deviceKeys, crossSigningKeys];
+}
+
+type CustomRequestHandler = (request: OutgoingRequest | RustSdkCryptoJs.UploadSigningKeysRequest) => Promise<any>;
+
+/** Loop for handling outgoing requests from an Olm machine.
+ *
+ * Simulates a server with two users: "us" and "them".  Handles key query
+ * requests, querying either our keys or the other user's keys.  Room messages
+ * are sent as incoming verification events to the other user.  A custom
+ * handler can be added to override default request processing (the handler
+ * should return a response body to inhibit default processing).
+ *
+ * Can also be used as an OutgoingRequestProcessor. */
+function makeRequestLoop(
+    ourOlmMachine: RustSdkCryptoJs.OlmMachine,
+    ourDeviceKeys: IDeviceKeys,
+    ourCrossSigningKeys: CrossSigningKeys,
+    theirOlmMachine: RustSdkCryptoJs.OlmMachine,
+    theirDeviceKeys: IDeviceKeys,
+    theirCrossSigningKeys: CrossSigningKeys,
+    customHandler?: CustomRequestHandler,
+) {
+    let stopRequestLoop = false;
+    const ourUserId = ourOlmMachine.userId.toString();
+    const ourDeviceId = ourOlmMachine.deviceId.toString();
+    const theirUserId = theirOlmMachine.userId.toString();
+    const theirDeviceId = theirOlmMachine.deviceId.toString();
+
+    function defaultHandler(request: OutgoingRequest | RustSdkCryptoJs.UploadSigningKeysRequest): any {
+        if (request instanceof RustSdkCryptoJs.KeysQueryRequest) {
+            const resp: Record<string, any> = {
+                device_keys: {},
+            };
+            const body = JSON.parse(request.body);
+            const query = body.device_keys;
+            const masterKeys: Record<string, any> = {};
+            const selfSigningKeys: Record<string, any> = {};
+            if (ourUserId in query) {
+                resp.device_keys[ourUserId] = { [ourDeviceId]: ourDeviceKeys };
+                masterKeys[ourUserId] = ourCrossSigningKeys.master_key;
+                selfSigningKeys[ourUserId] = ourCrossSigningKeys.self_signing_key;
+                resp.user_signing_keys = {
+                    [ourUserId]: ourCrossSigningKeys.user_signing_key,
+                };
+            }
+            if (theirUserId in query) {
+                resp.device_keys[theirUserId] = {
+                    [theirDeviceId]: theirDeviceKeys,
+                };
+                masterKeys[theirUserId] = theirCrossSigningKeys.master_key;
+                selfSigningKeys[theirUserId] = theirCrossSigningKeys.self_signing_key;
+            }
+            if (Object.keys(masterKeys).length) {
+                resp.master_keys = masterKeys;
+            }
+            if (Object.keys(selfSigningKeys).length) {
+                resp.self_signing_keys = selfSigningKeys;
+            }
+            return resp;
+        } else if (request instanceof RustSdkCryptoJs.RoomMessageRequest) {
+            theirOlmMachine.receiveVerificationEvent(
+                JSON.stringify({
+                    type: request.event_type,
+                    sender: ourUserId,
+                    event_id: "$" + request.event_type,
+                    content: JSON.parse(request.body),
+                    origin_server_ts: Date.now(),
+                    unsigned: {
+                        age: 0,
+                    },
+                }),
+                new RustSdkCryptoJs.RoomId(request.room_id),
+            );
+            return { event_id: "$" + request.event_type };
+        } else if (request instanceof RustSdkCryptoJs.SignatureUploadRequest) {
+            // this only gets called at the end after the verification
+            // succeeds, so we don't actually have to do anything.
+            return { failures: {} };
+        }
+        return {};
+    }
+
+    async function makeOutgoingRequest(
+        request: OutgoingRequest | RustSdkCryptoJs.UploadSigningKeysRequest,
+    ): Promise<any> {
+        const resp = (await customHandler?.(request)) ?? defaultHandler(request);
+        if (!(request instanceof RustSdkCryptoJs.UploadSigningKeysRequest) && request.id) {
+            await ourOlmMachine.markRequestAsSent(request.id!, request.type, JSON.stringify(resp));
+        }
+    }
+
+    async function runLoop() {
+        while (!stopRequestLoop) {
+            const requests = await ourOlmMachine.outgoingRequests();
+            for (const request of requests) {
+                await makeOutgoingRequest(request);
+            }
+        }
+    }
+
+    const loopCompletedPromise = runLoop();
+
+    return {
+        makeOutgoingRequest,
+        stop: async () => {
+            stopRequestLoop = true;
+            await loopCompletedPromise;
+        },
+    };
+}

src/@types/extensible_events.ts

@@ -23,9 +23,8 @@ import { isProvided } from "../extensible_events_v1/utilities";
 /**
  * Represents the stable and unstable values of a given namespace.
  */
-export type TSNamespace<N> = N extends NamespacedValue<infer S, infer U>
-    ? TSNamespaceValue<S> | TSNamespaceValue<U>
-    : never;
+export type TSNamespace<N> =
+    N extends NamespacedValue<infer S, infer U> ? TSNamespaceValue<S> | TSNamespaceValue<U> : never;
 
 /**
  * Represents a namespaced value, if the value is a string. Used to extract provided types

src/autodiscovery.ts

@@ -28,7 +28,7 @@ import {
     validateWellKnownAuthentication,
 } from "./oidc/validate";
 import { OidcError } from "./oidc/error";
-import { MINIMUM_MATRIX_VERSION } from "./version-support";
+import { SUPPORTED_MATRIX_VERSIONS } from "./version-support";
 
 // Dev note: Auto discovery is part of the spec.
 // See: https://matrix.org/docs/spec/client_server/r0.4.0.html#server-discovery
@@ -51,7 +51,10 @@ export enum AutoDiscoveryError {
     InvalidIs = "Invalid identity server discovery response",
     MissingWellknown = "No .well-known JSON file found",
     InvalidJson = "Invalid JSON",
-    HomeserverTooOld = "The homeserver does not meet the minimum version requirements",
+    UnsupportedHomeserverSpecVersion = "The homeserver does not meet the version requirements",
+
+    /** @deprecated Replaced by `UnsupportedHomeserverSpecVersion` */
+    HomeserverTooOld = UnsupportedHomeserverSpecVersion,
     // TODO: Implement when Sydent supports the `/versions` endpoint - https://github.com/matrix-org/sydent/issues/424
     //IdentityServerTooOld = "The identity server does not meet the minimum version requirements",
 }
@@ -112,7 +115,11 @@ export class AutoDiscovery {
 
     public static readonly ERROR_INVALID_JSON = AutoDiscoveryError.InvalidJson;
 
-    public static readonly ERROR_HOMESERVER_TOO_OLD = AutoDiscoveryError.HomeserverTooOld;
+    public static readonly ERROR_UNSUPPORTED_HOMESERVER_SPEC_VERSION =
+        AutoDiscoveryError.UnsupportedHomeserverSpecVersion;
+
+    /** @deprecated Replaced by ERROR_UNSUPPORTED_HOMESERVER_SPEC_VERSION */
+    public static readonly ERROR_HOMESERVER_TOO_OLD = AutoDiscovery.ERROR_UNSUPPORTED_HOMESERVER_SPEC_VERSION;
 
     public static readonly ALL_ERRORS = Object.keys(AutoDiscoveryError) as AutoDiscoveryError[];
 
@@ -216,10 +223,19 @@ export class AutoDiscovery {
             return Promise.resolve(clientConfig);
         }
 
-        // Step 3.1: Non-spec check to ensure the server will actually work for us
-        if (!hsVersions.raw!["versions"].includes(MINIMUM_MATRIX_VERSION)) {
-            logger.error("Homeserver does not meet minimum version requirements");
-            clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_HOMESERVER_TOO_OLD;
+        // Step 3.1: Non-spec check to ensure the server will actually work for us. We need to check if
+        // any of the versions in `SUPPORTED_MATRIX_VERSIONS` are listed in the /versions response.
+        const hsVersionSet = new Set(hsVersions.raw!["versions"]);
+        let supportedVersionFound = false;
+        for (const version of SUPPORTED_MATRIX_VERSIONS) {
+            if (hsVersionSet.has(version)) {
+                supportedVersionFound = true;
+                break;
+            }
+        }
+        if (!supportedVersionFound) {
+            logger.error("Homeserver does not meet version requirements");
+            clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_UNSUPPORTED_HOMESERVER_SPEC_VERSION;
 
             // Supply the base_url to the caller because they may be ignoring liveliness
             // errors, like this one.

src/client.ts

@@ -209,7 +209,7 @@ import { IgnoredInvites } from "./models/invites-ignorer";
 import { UIARequest, UIAResponse } from "./@types/uia";
 import { LocalNotificationSettings } from "./@types/local_notifications";
 import { buildFeatureSupportMap, Feature, ServerSupport } from "./feature";
-import { CryptoBackend } from "./common-crypto/CryptoBackend";
+import { BackupDecryptor, CryptoBackend } from "./common-crypto/CryptoBackend";
 import { RUST_SDK_STORE_PREFIX } from "./rust-crypto/constants";
 import { BootstrapCrossSigningOpts, CrossSigningKeyInfo, CryptoApi, ImportRoomKeysOpts } from "./crypto-api";
 import { DeviceInfoMap } from "./crypto/DeviceList";
@@ -962,7 +962,8 @@ type CryptoEvents =
     | CryptoEvent.KeysChanged
     | CryptoEvent.Warning
     | CryptoEvent.DevicesUpdated
-    | CryptoEvent.WillUpdateDevices;
+    | CryptoEvent.WillUpdateDevices
+    | CryptoEvent.LegacyCryptoStoreMigrationProgress;
 
 type MatrixEventEvents = MatrixEventEvent.Decrypted | MatrixEventEvent.Replaced | MatrixEventEvent.VisibilityChange;
 
@@ -1304,7 +1305,13 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     protected txnCtr = 0;
     protected mediaHandler = new MediaHandler(this);
     protected sessionId: string;
-    protected pendingEventEncryption = new Map<string, Promise<void>>();
+
+    /** IDs of events which are currently being encrypted.
+     *
+     * This is part of the cancellation mechanism: if the event is no longer listed here when encryption completes,
+     * that tells us that it has been cancelled, and we should not send it.
+     */
+    private eventsBeingEncrypted = new Set<string>();
 
     private useE2eForGroupCall = true;
     private toDeviceMessageQueue: ToDeviceMessageQueue;
@@ -1444,7 +1451,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         // correctly handle notification counts on encrypted rooms.
         // This fixes https://github.com/vector-im/element-web/issues/9421
         this.on(RoomEvent.Receipt, (event, room) => {
-            if (room && this.isRoomEncrypted(room.roomId)) {
+            if (room?.hasEncryptionStateEvent()) {
                 // Figure out if we've read something or if it's just informational
                 const content = event.getContent();
                 const isSelf =
@@ -2328,7 +2335,13 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             cryptoCallbacks: this.cryptoCallbacks,
             storePrefix: useIndexedDB ? RUST_SDK_STORE_PREFIX : null,
             storePassphrase: this.pickleKey,
+            legacyCryptoStore: this.cryptoStore,
+            legacyPickleKey: this.pickleKey ?? "DEFAULT_KEY",
+            legacyMigrationProgressListener: (progress, total) => {
+                this.emit(CryptoEvent.LegacyCryptoStoreMigrationProgress, progress, total);
+            },
         });
+
         rustCrypto.setSupportedVerificationMethods(this.verificationMethods);
 
         this.cryptoBackend = rustCrypto;
@@ -2347,6 +2360,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             CryptoEvent.KeyBackupSessionsRemaining,
             CryptoEvent.KeyBackupFailed,
             CryptoEvent.KeyBackupDecryptionKeyCached,
+            CryptoEvent.KeysChanged,
+            CryptoEvent.DevicesUpdated,
+            CryptoEvent.WillUpdateDevices,
         ]);
     }
 
@@ -3238,6 +3254,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * @param roomId - The room ID to enable encryption in.
      * @param config - The encryption config for the room.
      * @returns A promise that will resolve when encryption is set up.
+     *
+     * @deprecated Not supported for Rust Cryptography. To enable encryption in a room, send an `m.room.encryption`
+     * state event.
      */
     public setRoomEncryption(roomId: string, config: IRoomEncryption): Promise<void> {
         if (!this.crypto) {
@@ -3250,6 +3269,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * Whether encryption is enabled for a room.
      * @param roomId - the room id to query.
      * @returns whether encryption is enabled.
+     *
+     * @deprecated Not correctly supported for Rust Cryptography. Use {@link CryptoApi.isEncryptionEnabledInRoom} and/or
+     *    {@link Room.hasEncryptionStateEvent}.
      */
     public isRoomEncrypted(roomId: string): boolean {
         const room = this.getRoom(roomId);
@@ -3261,8 +3283,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         // if there is an 'm.room.encryption' event in this room, it should be
         // encrypted (independently of whether we actually support encryption)
-        const ev = room.currentState.getStateEvents(EventType.RoomEncryption, "");
-        if (ev) {
+        if (room.hasEncryptionStateEvent()) {
             return true;
         }
 
@@ -3898,7 +3919,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         }
 
         let totalKeyCount = 0;
-        let keys: IMegolmSessionData[] = [];
+        let totalFailures = 0;
+        let totalImported = 0;
 
         const path = this.makeKeyBackupPath(targetRoomId, targetSessionId, backupInfo.version);
 
@@ -3934,25 +3956,61 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
                 { prefix: ClientPrefix.V3 },
             );
 
-            if ((res as IRoomsKeysResponse).rooms) {
-                const rooms = (res as IRoomsKeysResponse).rooms;
-                for (const [roomId, roomData] of Object.entries(rooms)) {
-                    if (!roomData.sessions) continue;
+            // We have finished fetching the backup, go to next step
+            if (progressCallback) {
+                progressCallback({
+                    stage: "load_keys",
+                });
+            }
 
-                    totalKeyCount += Object.keys(roomData.sessions).length;
-                    const roomKeys = await backupDecryptor.decryptSessions(roomData.sessions);
-                    for (const k of roomKeys) {
-                        k.room_id = roomId;
-                        keys.push(k);
-                    }
-                }
+            if ((res as IRoomsKeysResponse).rooms) {
+                // We have a full backup here, it can get quite big, so we need to decrypt and import it in chunks.
+
+                // Get the total count as a first pass
+                totalKeyCount = this.getTotalKeyCount(res as IRoomsKeysResponse);
+                // Now decrypt and import the keys in chunks
+                await this.handleDecryptionOfAFullBackup(
+                    res as IRoomsKeysResponse,
+                    backupDecryptor,
+                    200,
+                    async (chunk) => {
+                        // We have a chunk of decrypted keys: import them
+                        try {
+                            await this.cryptoBackend!.importBackedUpRoomKeys(chunk, {
+                                untrusted,
+                            });
+                            totalImported += chunk.length;
+                        } catch (e) {
+                            totalFailures += chunk.length;
+                            // We failed to import some keys, but we should still try to import the rest?
+                            // Log the error and continue
+                            logger.error("Error importing keys from backup", e);
+                        }
+
+                        if (progressCallback) {
+                            progressCallback({
+                                total: totalKeyCount,
+                                successes: totalImported,
+                                stage: "load_keys",
+                                failures: totalFailures,
+                            });
+                        }
+                    },
+                );
             } else if ((res as IRoomKeysResponse).sessions) {
+                // For now we don't chunk for a single room backup, but we could in the future.
+                // Currently it is not used by the application.
                 const sessions = (res as IRoomKeysResponse).sessions;
                 totalKeyCount = Object.keys(sessions).length;
-                keys = await backupDecryptor.decryptSessions(sessions);
+                const keys = await backupDecryptor.decryptSessions(sessions);
                 for (const k of keys) {
                     k.room_id = targetRoomId!;
                 }
+                await this.cryptoBackend.importBackedUpRoomKeys(keys, {
+                    progressCallback,
+                    untrusted,
+                });
+                totalImported = keys.length;
             } else {
                 totalKeyCount = 1;
                 try {
@@ -3961,7 +4019,12 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
                     });
                     key.room_id = targetRoomId!;
                     key.session_id = targetSessionId!;
-                    keys.push(key);
+
+                    await this.cryptoBackend.importBackedUpRoomKeys([key], {
+                        progressCallback,
+                        untrusted,
+                    });
+                    totalImported = 1;
                 } catch (e) {
                     this.logger.debug("Failed to decrypt megolm session from backup", e);
                 }
@@ -3970,15 +4033,88 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             backupDecryptor.free();
         }
 
-        await this.cryptoBackend.importBackedUpRoomKeys(keys, {
-            progressCallback,
-            untrusted,
-        });
-
         /// in case entering the passphrase would add a new signature?
         await this.cryptoBackend.checkKeyBackupAndEnable();
 
-        return { total: totalKeyCount, imported: keys.length };
+        return { total: totalKeyCount, imported: totalImported };
+    }
+
+    /**
+     * This method calculates the total number of keys present in the response of a `/room_keys/keys` call.
+     *
+     * @param res - The response from the server containing the keys to be counted.
+     *
+     * @returns The total number of keys in the backup.
+     */
+    private getTotalKeyCount(res: IRoomsKeysResponse): number {
+        const rooms = res.rooms;
+        let totalKeyCount = 0;
+        for (const roomData of Object.values(rooms)) {
+            if (!roomData.sessions) continue;
+            totalKeyCount += Object.keys(roomData.sessions).length;
+        }
+        return totalKeyCount;
+    }
+
+    /**
+     * This method handles the decryption of a full backup, i.e a call to `/room_keys/keys`.
+     * It will decrypt the keys in chunks and call the `block` callback for each chunk.
+     *
+     * @param res - The response from the server containing the keys to be decrypted.
+     * @param backupDecryptor - An instance of the BackupDecryptor class used to decrypt the keys.
+     * @param chunkSize - The size of the chunks to be processed at a time.
+     * @param block - A callback function that is called for each chunk of keys.
+     *
+     * @returns A promise that resolves when the decryption is complete.
+     */
+    private async handleDecryptionOfAFullBackup(
+        res: IRoomsKeysResponse,
+        backupDecryptor: BackupDecryptor,
+        chunkSize: number,
+        block: (chunk: IMegolmSessionData[]) => Promise<void>,
+    ): Promise<void> {
+        const rooms = (res as IRoomsKeysResponse).rooms;
+
+        let groupChunkCount = 0;
+        let chunkGroupByRoom: Map<string, IKeyBackupRoomSessions> = new Map();
+
+        const handleChunkCallback = async (roomChunks: Map<string, IKeyBackupRoomSessions>): Promise<void> => {
+            const currentChunk: IMegolmSessionData[] = [];
+            for (const roomId of roomChunks.keys()) {
+                const decryptedSessions = await backupDecryptor.decryptSessions(roomChunks.get(roomId)!);
+                for (const sessionId in decryptedSessions) {
+                    const k = decryptedSessions[sessionId];
+                    k.room_id = roomId;
+                    currentChunk.push(k);
+                }
+            }
+            await block(currentChunk);
+        };
+
+        for (const [roomId, roomData] of Object.entries(rooms)) {
+            if (!roomData.sessions) continue;
+
+            chunkGroupByRoom.set(roomId, {});
+
+            for (const [sessionId, session] of Object.entries(roomData.sessions)) {
+                const sessionsForRoom = chunkGroupByRoom.get(roomId)!;
+                sessionsForRoom[sessionId] = session;
+                groupChunkCount += 1;
+                if (groupChunkCount >= chunkSize) {
+                    // We have enough chunks to decrypt
+                    await handleChunkCallback(chunkGroupByRoom);
+                    chunkGroupByRoom = new Map();
+                    // There might be remaining keys for that room, so add back an entry for the current room.
+                    chunkGroupByRoom.set(roomId, {});
+                    groupChunkCount = 0;
+                }
+            }
+        }
+
+        // Handle remaining chunk if needed
+        if (groupChunkCount > 0) {
+            await handleChunkCallback(chunkGroupByRoom);
+        }
     }
 
     public deleteKeysFromBackup(roomId: undefined, sessionId: undefined, version?: string): Promise<void>;
@@ -4327,9 +4463,10 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             throw new Error("cannot cancel an event with status " + event.status);
         }
 
-        // if the event is currently being encrypted then
+        // If the event is currently being encrypted then remove it from the pending list, to indicate that it should
+        // not be sent.
         if (event.status === EventStatus.ENCRYPTING) {
-            this.pendingEventEncryption.delete(event.getId()!);
+            this.eventsBeingEncrypted.delete(event.getId()!);
         } else if (this.scheduler && event.status === EventStatus.QUEUED) {
             // tell the scheduler to forget about it, if it's queued
             this.scheduler.removeEventFromQueue(event);
@@ -4628,96 +4765,102 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * encrypts the event if necessary; adds the event to the queue, or sends it; marks the event as sent/unsent
      * @returns returns a promise which resolves with the result of the send request
      */
-    protected encryptAndSendEvent(room: Room | null, event: MatrixEvent): Promise<ISendEventResponse> {
-        let cancelled = false;
-        // Add an extra Promise.resolve() to turn synchronous exceptions into promise rejections,
-        // so that we can handle synchronous and asynchronous exceptions with the
-        // same code path.
-        return Promise.resolve()
-            .then(() => {
-                const encryptionPromise = this.encryptEventIfNeeded(event, room ?? undefined);
-                if (!encryptionPromise) return null; // doesn't need encryption
+    protected async encryptAndSendEvent(room: Room | null, event: MatrixEvent): Promise<ISendEventResponse> {
+        try {
+            let cancelled: boolean;
+            this.eventsBeingEncrypted.add(event.getId()!);
+            try {
+                await this.encryptEventIfNeeded(event, room ?? undefined);
+            } finally {
+                cancelled = !this.eventsBeingEncrypted.delete(event.getId()!);
+            }
 
-                this.pendingEventEncryption.set(event.getId()!, encryptionPromise);
-                this.updatePendingEventStatus(room, event, EventStatus.ENCRYPTING);
-                return encryptionPromise.then(() => {
-                    if (!this.pendingEventEncryption.has(event.getId()!)) {
-                        // cancelled via MatrixClient::cancelPendingEvent
-                        cancelled = true;
-                        return;
-                    }
-                    this.updatePendingEventStatus(room, event, EventStatus.SENDING);
-                });
-            })
-            .then(() => {
-                if (cancelled) return {} as ISendEventResponse;
-                let promise: Promise<ISendEventResponse> | null = null;
-                if (this.scheduler) {
-                    // if this returns a promise then the scheduler has control now and will
-                    // resolve/reject when it is done. Internally, the scheduler will invoke
-                    // processFn which is set to this._sendEventHttpRequest so the same code
-                    // path is executed regardless.
-                    promise = this.scheduler.queueEvent(event);
-                    if (promise && this.scheduler.getQueueForEvent(event)!.length > 1) {
-                        // event is processed FIFO so if the length is 2 or more we know
-                        // this event is stuck behind an earlier event.
-                        this.updatePendingEventStatus(room, event, EventStatus.QUEUED);
-                    }
-                }
+            if (cancelled) {
+                // cancelled via MatrixClient::cancelPendingEvent
+                return {} as ISendEventResponse;
+            }
 
-                if (!promise) {
-                    promise = this.sendEventHttpRequest(event);
-                    if (room) {
-                        promise = promise.then((res) => {
-                            room.updatePendingEvent(event, EventStatus.SENT, res["event_id"]);
-                            return res;
-                        });
-                    }
-                }
+            // encryptEventIfNeeded may have updated the status from SENDING to ENCRYPTING. If so, we need
+            // to put it back.
+            if (event.status === EventStatus.ENCRYPTING) {
+                this.updatePendingEventStatus(room, event, EventStatus.SENDING);
+            }
 
-                return promise;
-            })
-            .catch((err) => {
-                this.logger.error("Error sending event", err.stack || err);
-                try {
-                    // set the error on the event before we update the status:
-                    // updating the status emits the event, so the state should be
-                    // consistent at that point.
-                    event.error = err;
-                    this.updatePendingEventStatus(room, event, EventStatus.NOT_SENT);
-                } catch (e) {
-                    this.logger.error("Exception in error handler!", (<Error>e).stack || err);
+            let promise: Promise<ISendEventResponse> | null = null;
+            if (this.scheduler) {
+                // if this returns a promise then the scheduler has control now and will
+                // resolve/reject when it is done. Internally, the scheduler will invoke
+                // processFn which is set to this._sendEventHttpRequest so the same code
+                // path is executed regardless.
+                promise = this.scheduler.queueEvent(event);
+                if (promise && this.scheduler.getQueueForEvent(event)!.length > 1) {
+                    // event is processed FIFO so if the length is 2 or more we know
+                    // this event is stuck behind an earlier event.
+                    this.updatePendingEventStatus(room, event, EventStatus.QUEUED);
                 }
-                if (err instanceof MatrixError) {
-                    err.event = event;
+            }
+
+            if (!promise) {
+                promise = this.sendEventHttpRequest(event);
+                if (room) {
+                    promise = promise.then((res) => {
+                        room.updatePendingEvent(event, EventStatus.SENT, res["event_id"]);
+                        return res;
+                    });
                 }
-                throw err;
-            });
+            }
+
+            return await promise;
+        } catch (err) {
+            this.logger.error("Error sending event", err);
+            try {
+                // set the error on the event before we update the status:
+                // updating the status emits the event, so the state should be
+                // consistent at that point.
+                event.error = <MatrixError>err;
+                this.updatePendingEventStatus(room, event, EventStatus.NOT_SENT);
+            } catch (e) {
+                this.logger.error("Exception in error handler!", e);
+            }
+            if (err instanceof MatrixError) {
+                err.event = event;
+            }
+            throw err;
+        }
     }
 
-    private encryptEventIfNeeded(event: MatrixEvent, room?: Room): Promise<void> | null {
+    private async encryptEventIfNeeded(event: MatrixEvent, room?: Room): Promise<void> {
+        // If the room is unknown, we cannot encrypt for it
+        if (!room) return;
+
+        if (!(await this.shouldEncryptEventForRoom(event, room))) return;
+
+        if (!this.cryptoBackend && this.usingExternalCrypto) {
+            // The client has opted to allow sending messages to encrypted
+            // rooms even if the room is encrypted, and we haven't set up
+            // crypto. This is useful for users of matrix-org/pantalaimon
+            return;
+        }
+
+        if (!this.cryptoBackend) {
+            throw new Error("This room is configured to use encryption, but your client does not support encryption.");
+        }
+
+        this.updatePendingEventStatus(room, event, EventStatus.ENCRYPTING);
+        await this.cryptoBackend.encryptEvent(event, room);
+    }
+
+    /**
+     * Determine whether a given event should be encrypted when we send it to the given room.
+     *
+     * This takes into account event type and room configuration.
+     */
+    private async shouldEncryptEventForRoom(event: MatrixEvent, room: Room): Promise<boolean> {
         if (event.isEncrypted()) {
             // this event has already been encrypted; this happens if the
             // encryption step succeeded, but the send step failed on the first
             // attempt.
-            return null;
-        }
-
-        if (event.isRedaction()) {
-            // Redactions do not support encryption in the spec at this time,
-            // whilst it mostly worked in some clients, it wasn't compliant.
-            return null;
-        }
-
-        if (!room || !this.isRoomEncrypted(event.getRoomId()!)) {
-            return null;
-        }
-
-        if (!this.cryptoBackend && this.usingExternalCrypto) {
-            // The client has opted to allow sending messages to encrypted
-            // rooms even if the room is encrypted, and we haven't setup
-            // crypto. This is useful for users of matrix-org/pantalaimon
-            return null;
+            return false;
         }
 
         if (event.getType() === EventType.Reaction) {
@@ -4731,14 +4874,23 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             // The reaction key / content / emoji value does warrant encrypting, but
             // this will be handled separately by encrypting just this value.
             // See https://github.com/matrix-org/matrix-doc/pull/1849#pullrequestreview-248763642
-            return null;
+            return false;
         }
 
-        if (!this.cryptoBackend) {
-            throw new Error("This room is configured to use encryption, but your client does not support encryption.");
+        if (event.isRedaction()) {
+            // Redactions do not support encryption in the spec at this time.
+            // Whilst it mostly worked in some clients, it wasn't compliant.
+            return false;
         }
 
-        return this.cryptoBackend.encryptEvent(event, room);
+        // If the room has an m.room.encryption event, we should encrypt.
+        if (room.hasEncryptionStateEvent()) return true;
+
+        // If we have a crypto impl, and *it* thinks we should encrypt, then we should.
+        if (await this.cryptoBackend?.isEncryptionEnabledInRoom(room.roomId)) return true;
+
+        // Otherwise, no need to encrypt.
+        return false;
     }
 
     /**
@@ -4753,7 +4905,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         eventType?: EventType | string | null,
     ): EventType | string | null | undefined {
         if (eventType === EventType.Reaction) return eventType;
-        return this.isRoomEncrypted(roomId) ? EventType.RoomMessageEncrypted : eventType;
+        return this.getRoom(roomId)?.hasEncryptionStateEvent() ? EventType.RoomMessageEncrypted : eventType;
     }
 
     protected updatePendingEventStatus(room: Room | null, event: MatrixEvent, newStatus: EventStatus): void {
@@ -5683,6 +5835,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * @param allowDirectLinks - If true, return any non-mxc URLs
      * directly. Fetching such URLs will leak information about the user to
      * anyone they share a room with. If false, will return null for such URLs.
+     * @param allowRedirects - If true, the caller supports the URL being 307 or
+     * 308 redirected to another resource upon request. If false, redirects
+     * are not expected.
      * @returns the avatar URL or null.
      */
     public mxcUrlToHttp(
@@ -5691,8 +5846,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         height?: number,
         resizeMethod?: string,
         allowDirectLinks?: boolean,
+        allowRedirects?: boolean,
     ): string | null {
-        return getHttpUriForMxc(this.baseUrl, mxcUrl, width, height, resizeMethod, allowDirectLinks);
+        return getHttpUriForMxc(this.baseUrl, mxcUrl, width, height, resizeMethod, allowDirectLinks, allowRedirects);
     }
 
     /**

src/content-repo.ts

@@ -28,6 +28,9 @@ import { encodeParams } from "./utils";
  * directly. Fetching such URLs will leak information about the user to
  * anyone they share a room with. If false, will return the emptry string
  * for such URLs.
+ * @param allowRedirects - If true, the caller supports the URL being 307 or
+ * 308 redirected to another resource upon request. If false, redirects
+ * are not expected.
  * @returns The complete URL to the content.
  */
 export function getHttpUriForMxc(
@@ -37,6 +40,7 @@ export function getHttpUriForMxc(
     height?: number,
     resizeMethod?: string,
     allowDirectLinks = false,
+    allowRedirects?: boolean,
 ): string {
     if (typeof mxc !== "string" || !mxc) {
         return "";
@@ -67,6 +71,11 @@ export function getHttpUriForMxc(
         prefix = "/_matrix/media/v3/thumbnail/";
     }
 
+    if (typeof allowRedirects === "boolean") {
+        // We add this after, so we don't convert everything to a thumbnail request.
+        params["allow_redirect"] = JSON.stringify(allowRedirects);
+    }
+
     const fragmentOffset = serverAndMediaId.indexOf("#");
     let fragment = "";
     if (fragmentOffset >= 0) {

src/crypto-api.ts

@@ -53,6 +53,18 @@ export interface CryptoApi {
      */
     getOwnDeviceKeys(): Promise<OwnDeviceKeys>;
 
+    /**
+     * Check if we believe the given room to be encrypted.
+     *
+     * This method returns true if the room has been configured with encryption. The setting is persistent, so that
+     * even if the encryption event is removed from the room state, it still returns true. This helps to guard against
+     * a downgrade attack wherein a server admin attempts to remove encryption.
+     *
+     * @returns `true` if the room with the supplied ID is encrypted. `false` if the room is not encrypted, or is unknown to
+     * us.
+     */
+    isEncryptionEnabledInRoom(roomId: string): Promise<boolean>;
+
     /**
      * Perform any background tasks that can be done before a message is ready to
      * send, in order to speed up sending of the message.
@@ -189,7 +201,7 @@ export interface CryptoApi {
      * Cross-signing a device indicates, to our other devices and to other users, that we have verified that it really
      * belongs to us.
      *
-     * Requires that cross-signing has been set up on this device (normally by calling {@link bootstrapCrossSigning}.
+     * Requires that cross-signing has been set up on this device (normally by calling {@link bootstrapCrossSigning}).
      *
      * *Note*: Do not call this unless you have verified, somehow, that the device is genuine!
      *
@@ -586,9 +598,9 @@ export class DeviceVerificationStatus {
  */
 export interface ImportRoomKeyProgressData {
     stage: string; // TODO: Enum
-    successes: number;
-    failures: number;
-    total: number;
+    successes?: number;
+    failures?: number;
+    total?: number;
 }
 
 /**

src/crypto/OlmDevice.ts

@@ -64,7 +64,7 @@ export interface InboundGroupSessionData {
     room_id: string; // eslint-disable-line camelcase
     /** pickled Olm.InboundGroupSession */
     session: string;
-    keysClaimed: Record<string, string>;
+    keysClaimed?: Record<string, string>;
     /** Devices involved in forwarding this session to us (normally empty). */
     forwardingCurve25519KeyChain: string[];
     /** whether this session is untrusted. */

src/crypto/index.ts

@@ -259,6 +259,15 @@ export enum CryptoEvent {
     WillUpdateDevices = "crypto.willUpdateDevices",
     DevicesUpdated = "crypto.devicesUpdated",
     KeysChanged = "crossSigning.keysChanged",
+
+    /**
+     * Fires when data is being migrated from legacy crypto to rust crypto.
+     *
+     * The payload is a pair `(progress, total)`, where `progress` is the number of steps completed so far, and
+     * `total` is the total number of steps. When migration is complete, a final instance of the event is emitted, with
+     * `progress === total === -1`.
+     */
+    LegacyCryptoStoreMigrationProgress = "crypto.legacyCryptoStoreMigrationProgress",
 }
 
 export type CryptoEventHandlerMap = {
@@ -368,6 +377,8 @@ export type CryptoEventHandlerMap = {
      */
     [CryptoEvent.DevicesUpdated]: (users: string[], initialFetch: boolean) => void;
     [CryptoEvent.UserCrossSigningUpdated]: (userId: string) => void;
+
+    [CryptoEvent.LegacyCryptoStoreMigrationProgress]: (progress: number, total: number) => void;
 };
 
 export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap> implements CryptoBackend {
@@ -4262,6 +4273,13 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
         return this.roomList.isRoomEncrypted(roomId);
     }
 
+    /**
+     * Implementation of {@link CryptoApi#isEncryptionEnabledInRoom}.
+     */
+    public async isEncryptionEnabledInRoom(roomId: string): Promise<boolean> {
+        return this.isRoomEncrypted(roomId);
+    }
+
     /**
      * @returns information about the encryption on the room with the supplied
      * ID, or null if the room is not encrypted or unknown to us.

src/crypto/keybackup.ts

@@ -15,6 +15,8 @@ limitations under the License.
 */
 
 // Export for backward compatibility
+import { ImportRoomKeyProgressData } from "../crypto-api";
+
 export type {
     Curve25519AuthData as ICurve25519AuthData,
     Aes256AuthData as IAes256AuthData,
@@ -41,5 +43,5 @@ export interface IKeyBackupRestoreResult {
 
 export interface IKeyBackupRestoreOpts {
     cacheCompleteCallback?: () => void;
-    progressCallback?: (progress: { stage: string }) => void;
+    progressCallback?: (progress: ImportRoomKeyProgressData) => void;
 }

src/crypto/store/base.ts

@@ -176,13 +176,20 @@ export interface CryptoStore {
         txn: unknown,
     ): void;
 
+    /**
+     * Count the number of Megolm sessions in the database.
+     *
+     * @internal
+     */
+    countEndToEndInboundGroupSessions(): Promise<number>;
+
     /**
      * Get a batch of Megolm sessions from the database.
      *
      * @returns A batch of Megolm Sessions, or `null` if no sessions are left.
      * @internal
      */
-    getEndToEndInboundGroupSessionsBatch(): Promise<ISession[] | null>;
+    getEndToEndInboundGroupSessionsBatch(): Promise<SessionExtended[] | null>;
 
     /**
      * Delete a batch of Megolm sessions from the database.
@@ -223,6 +230,11 @@ export interface ISession {
     sessionData?: InboundGroupSessionData;
 }
 
+/** Extended data on a Megolm session */
+export interface SessionExtended extends ISession {
+    needsBackup: boolean;
+}
+
 /** Data on an Olm session */
 export interface ISessionInfo {
     deviceKey?: string;
@@ -313,6 +325,9 @@ export enum MigrationState {
 
     /** OLM_SESSIONS_MIGRATED, and in addition, we have migrated all the Megolm sessions. */
     MEGOLM_SESSIONS_MIGRATED,
+
+    /** MEGOLM_SESSIONS_MIGRATED, and in addition, we have migrated all the room settings. */
+    ROOM_SETTINGS_MIGRATED,
 }
 
 /**

src/crypto/store/indexeddb-crypto-store-backend.ts

@@ -21,6 +21,7 @@ import {
     IDeviceData,
     IProblem,
     ISession,
+    SessionExtended,
     ISessionInfo,
     IWithheld,
     MigrationState,
@@ -810,34 +811,63 @@ export class Backend implements CryptoStore {
         });
     }
 
+    /**
+     * Count the number of Megolm sessions in the database.
+     *
+     * Implementation of {@link CryptoStore.countEndToEndInboundGroupSessions}.
+     *
+     * @internal
+     */
+    public async countEndToEndInboundGroupSessions(): Promise<number> {
+        let result = 0;
+        await this.doTxn("readonly", [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+            const sessionStore = txn.objectStore(IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS);
+            const countReq = sessionStore.count();
+            countReq.onsuccess = (): void => {
+                result = countReq.result;
+            };
+        });
+        return result;
+    }
+
     /**
      * Fetch a batch of Megolm sessions from the database.
      *
      * Implementation of {@link CryptoStore.getEndToEndInboundGroupSessionsBatch}.
      */
-    public async getEndToEndInboundGroupSessionsBatch(): Promise<null | ISession[]> {
-        const result: ISession[] = [];
-        await this.doTxn("readonly", [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
-            const objectStore = txn.objectStore(IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS);
-            const getReq = objectStore.openCursor();
-            getReq.onsuccess = function (): void {
-                try {
-                    const cursor = getReq.result;
-                    if (cursor) {
-                        result.push({
-                            senderKey: cursor.value.senderCurve25519Key,
-                            sessionId: cursor.value.sessionId,
-                            sessionData: cursor.value.session,
-                        });
-                        if (result.length < SESSION_BATCH_SIZE) {
-                            cursor.continue();
+    public async getEndToEndInboundGroupSessionsBatch(): Promise<null | SessionExtended[]> {
+        const result: SessionExtended[] = [];
+        await this.doTxn(
+            "readonly",
+            [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS, IndexedDBCryptoStore.STORE_BACKUP],
+            (txn) => {
+                const sessionStore = txn.objectStore(IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS);
+                const backupStore = txn.objectStore(IndexedDBCryptoStore.STORE_BACKUP);
+
+                const getReq = sessionStore.openCursor();
+                getReq.onsuccess = function (): void {
+                    try {
+                        const cursor = getReq.result;
+                        if (cursor) {
+                            const backupGetReq = backupStore.get(cursor.key);
+                            backupGetReq.onsuccess = (): void => {
+                                result.push({
+                                    senderKey: cursor.value.senderCurve25519Key,
+                                    sessionId: cursor.value.sessionId,
+                                    sessionData: cursor.value.session,
+                                    needsBackup: backupGetReq.result !== undefined,
+                                });
+                                if (result.length < SESSION_BATCH_SIZE) {
+                                    cursor.continue();
+                                }
+                            };
                         }
+                    } catch (e) {
+                        abortWithException(txn, <Error>e);
                     }
-                } catch (e) {
-                    abortWithException(txn, <Error>e);
-                }
-            };
-        });
+                };
+            },
+        );
 
         if (result.length === 0) {
             // No sessions left.

src/crypto/store/indexeddb-crypto-store.ts

@@ -25,6 +25,7 @@ import {
     IDeviceData,
     IProblem,
     ISession,
+    SessionExtended,
     ISessionInfo,
     IWithheld,
     MigrationState,
@@ -505,6 +506,17 @@ export class IndexedDBCryptoStore implements CryptoStore {
         return this.backend!.filterOutNotifiedErrorDevices(devices);
     }
 
+    /**
+     * Count the number of Megolm sessions in the database.
+     *
+     * Implementation of {@link CryptoStore.countEndToEndInboundGroupSessions}.
+     *
+     * @internal
+     */
+    public countEndToEndInboundGroupSessions(): Promise<number> {
+        return this.backend!.countEndToEndInboundGroupSessions();
+    }
+
     /**
      * Fetch a batch of Olm sessions from the database.
      *
@@ -610,7 +622,7 @@ export class IndexedDBCryptoStore implements CryptoStore {
      *
      * @internal
      */
-    public getEndToEndInboundGroupSessionsBatch(): Promise<ISession[] | null> {
+    public getEndToEndInboundGroupSessionsBatch(): Promise<SessionExtended[] | null> {
         return this.backend!.getEndToEndInboundGroupSessionsBatch();
     }
 

src/crypto/store/localStorage-crypto-store.ts

@@ -21,6 +21,7 @@ import {
     IDeviceData,
     IProblem,
     ISession,
+    SessionExtended,
     ISessionInfo,
     IWithheld,
     MigrationState,
@@ -126,7 +127,11 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore implements Crypto
     public countEndToEndSessions(txn: unknown, func: (count: number) => void): void {
         let count = 0;
         for (let i = 0; i < this.store.length; ++i) {
-            if (this.store.key(i)?.startsWith(keyEndToEndSessions(""))) ++count;
+            const key = this.store.key(i);
+            if (key?.startsWith(keyEndToEndSessions(""))) {
+                const sessions = getJsonItem(this.store, key);
+                count += Object.keys(sessions ?? {}).length;
+            }
         }
         func(count);
     }
@@ -350,6 +355,24 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore implements Crypto
         setJsonItem(this.store, keyEndToEndInboundGroupSessionWithheld(senderCurve25519Key, sessionId), sessionData);
     }
 
+    /**
+     * Count the number of Megolm sessions in the database.
+     *
+     * Implementation of {@link CryptoStore.countEndToEndInboundGroupSessions}.
+     *
+     * @internal
+     */
+    public async countEndToEndInboundGroupSessions(): Promise<number> {
+        let count = 0;
+        for (let i = 0; i < this.store.length; ++i) {
+            const key = this.store.key(i);
+            if (key?.startsWith(KEY_INBOUND_SESSION_PREFIX)) {
+                count += 1;
+            }
+        }
+        return count;
+    }
+
     /**
      * Fetch a batch of Megolm sessions from the database.
      *
@@ -357,20 +380,24 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore implements Crypto
      *
      * @internal
      */
-    public async getEndToEndInboundGroupSessionsBatch(): Promise<ISession[] | null> {
-        const result: ISession[] = [];
+    public async getEndToEndInboundGroupSessionsBatch(): Promise<SessionExtended[] | null> {
+        const sessionsNeedingBackup = getJsonItem<string[]>(this.store, KEY_SESSIONS_NEEDING_BACKUP) || {};
+        const result: SessionExtended[] = [];
         for (let i = 0; i < this.store.length; ++i) {
             const key = this.store.key(i);
             if (key?.startsWith(KEY_INBOUND_SESSION_PREFIX)) {
+                const key2 = key.slice(KEY_INBOUND_SESSION_PREFIX.length);
+
                 // we can't use split, as the components we are trying to split out
                 // might themselves contain '/' characters. We rely on the
                 // senderKey being a (32-byte) curve25519 key, base64-encoded
                 // (hence 43 characters long).
 
                 result.push({
-                    senderKey: key.slice(KEY_INBOUND_SESSION_PREFIX.length, KEY_INBOUND_SESSION_PREFIX.length + 43),
-                    sessionId: key.slice(KEY_INBOUND_SESSION_PREFIX.length + 44),
+                    senderKey: key2.slice(0, 43),
+                    sessionId: key2.slice(44),
                     sessionData: getJsonItem(this.store, key)!,
+                    needsBackup: key2 in sessionsNeedingBackup,
                 });
 
                 if (result.length >= SESSION_BATCH_SIZE) {

src/crypto/store/memory-crypto-store.ts

@@ -21,6 +21,7 @@ import {
     IDeviceData,
     IProblem,
     ISession,
+    SessionExtended,
     ISessionInfo,
     IWithheld,
     MigrationState,
@@ -335,7 +336,11 @@ export class MemoryCryptoStore implements CryptoStore {
     // Olm Sessions
 
     public countEndToEndSessions(txn: unknown, func: (count: number) => void): void {
-        func(Object.keys(this.sessions).length);
+        let count = 0;
+        for (const deviceSessions of Object.values(this.sessions)) {
+            count += Object.keys(deviceSessions).length;
+        }
+        func(count);
     }
 
     public getEndToEndSession(
@@ -527,6 +532,17 @@ export class MemoryCryptoStore implements CryptoStore {
         this.inboundGroupSessionsWithheld[k] = sessionData;
     }
 
+    /**
+     * Count the number of Megolm sessions in the database.
+     *
+     * Implementation of {@link CryptoStore.countEndToEndInboundGroupSessions}.
+     *
+     * @internal
+     */
+    public async countEndToEndInboundGroupSessions(): Promise<number> {
+        return Object.keys(this.inboundGroupSessions).length;
+    }
+
     /**
      * Fetch a batch of Megolm sessions from the database.
      *
@@ -534,13 +550,14 @@ export class MemoryCryptoStore implements CryptoStore {
      *
      * @internal
      */
-    public async getEndToEndInboundGroupSessionsBatch(): Promise<null | ISession[]> {
-        const result: ISession[] = [];
+    public async getEndToEndInboundGroupSessionsBatch(): Promise<null | SessionExtended[]> {
+        const result: SessionExtended[] = [];
         for (const [key, session] of Object.entries(this.inboundGroupSessions)) {
             result.push({
                 senderKey: key.slice(0, 43),
                 sessionId: key.slice(44),
                 sessionData: session,
+                needsBackup: key in this.sessionsNeedingBackup,
             });
             if (result.length >= SESSION_BATCH_SIZE) {
                 return result;

src/matrixrtc/CallMembership.ts

@@ -27,7 +27,8 @@ export interface CallMembershipData {
     scope: CallScope;
     device_id: string;
     created_ts?: number;
-    expires: number;
+    expires?: number;
+    expires_ts?: number;
     foci_active?: Focus[];
     membershipID: string;
 }
@@ -41,7 +42,20 @@ export class CallMembership {
         private parentEvent: MatrixEvent,
         private data: CallMembershipData,
     ) {
-        if (typeof data.expires !== "number") throw new Error("Malformed membership: expires must be numeric");
+        if (!(data.expires || data.expires_ts)) {
+            throw new Error("Malformed membership: expires_ts or expires must be present");
+        }
+        if (data.expires) {
+            if (typeof data.expires !== "number") {
+                throw new Error("Malformed membership: expires must be numeric");
+            }
+        }
+        if (data.expires_ts) {
+            if (typeof data.expires_ts !== "number") {
+                throw new Error("Malformed membership: expires_ts must be numeric");
+            }
+        }
+
         if (typeof data.device_id !== "string") throw new Error("Malformed membership event: device_id must be string");
         if (typeof data.call_id !== "string") throw new Error("Malformed membership event: call_id must be string");
         if (typeof data.scope !== "string") throw new Error("Malformed membership event: scope must be string");
@@ -77,16 +91,27 @@ export class CallMembership {
     }
 
     public getAbsoluteExpiry(): number {
-        return this.createdTs() + this.data.expires;
+        if (this.data.expires) {
+            return this.createdTs() + this.data.expires;
+        } else {
+            // We know it exists because we checked for this in the constructor.
+            return this.data.expires_ts!;
+        }
     }
 
     // gets the expiry time of the event, converted into the device's local time
     public getLocalExpiry(): number {
-        const relativeCreationTime = this.parentEvent.getTs() - this.createdTs();
+        if (this.data.expires) {
+            const relativeCreationTime = this.parentEvent.getTs() - this.createdTs();
 
-        const localCreationTs = this.parentEvent.localTimestamp - relativeCreationTime;
+            const localCreationTs = this.parentEvent.localTimestamp - relativeCreationTime;
 
-        return localCreationTs + this.data.expires;
+            return localCreationTs + this.data.expires;
+        } else {
+            // With expires_ts we cannot convert to local time.
+            // TODO: Check the server timestamp and compute a diff to local time.
+            return this.data.expires_ts!;
+        }
     }
 
     public getMsUntilExpiry(): number {

src/matrixrtc/MatrixRTCSession.ts

@@ -133,7 +133,7 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
         for (const memberEvent of callMemberEvents) {
             const eventMemberships: CallMembershipData[] = memberEvent.getContent()["memberships"];
             if (eventMemberships === undefined) {
-                logger.warn(`Ignoring malformed member event from ${memberEvent.getSender()}: no memberships section`);
+                logger.debug(`Ignoring malformed member event from ${memberEvent.getSender()}: no memberships section`);
                 continue;
             }
             if (!Array.isArray(eventMemberships)) {
@@ -624,6 +624,9 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
         };
 
         if (prevMembership) m.created_ts = prevMembership.createdTs();
+        if (m.created_ts) m.expires_ts = m.created_ts + (m.expires ?? 0);
+        // TODO: Date.now() should be the origin_server_ts (now).
+        else m.expires_ts = Date.now() + (m.expires ?? 0);
 
         return m;
     }

src/models/event-timeline-set.ts

@@ -1003,8 +1003,9 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
 
         if (!shouldLiveInRoom && !shouldLiveInThread) {
             logger.warn(
-                `EventTimelineSet:canContain event encountered which cannot be added to any timeline roomId=${this.room
-                    ?.roomId} eventId=${event.getId()} threadId=${event.threadRootId}`,
+                `EventTimelineSet:canContain event encountered which cannot be added to any timeline roomId=${
+                    this.room?.roomId
+                } eventId=${event.getId()} threadId=${event.threadRootId}`,
             );
         }
 

src/models/event.ts

@@ -350,7 +350,7 @@ export class MatrixEvent extends TypedEventEmitter<MatrixEventEmittedEvents, Mat
     /**
      * most recent error associated with sending the event, if any
      * @privateRemarks
-     * Should be read-only
+     * Should be read-only. May not be a MatrixError.
      */
     public error: MatrixError | null = null;
     /**

src/models/read-receipt.ts

@@ -79,7 +79,7 @@ export abstract class ReadReceipt<
     private receiptCacheByEventId: ReceiptCache = new Map();
 
     public abstract getUnfilteredTimelineSet(): EventTimelineSet;
-    public abstract timeline: MatrixEvent[];
+    public abstract get timeline(): MatrixEvent[];
 
     /**
      * Gets the latest receipt for a given user in the room

src/models/room.ts

@@ -381,13 +381,6 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
      * The room summary.
      */
     public summary: RoomSummary | null = null;
-    /**
-     * The live event timeline for this room, with the oldest event at index 0.
-     *
-     * @deprecated Present for backwards compatibility.
-     *             Use getLiveTimeline().getEvents() instead
-     */
-    public timeline!: MatrixEvent[];
     /**
      * oldState The state of the room at the time of the oldest event in the live timeline.
      *
@@ -793,6 +786,16 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
         return this.getUnfilteredTimelineSet().getLiveTimeline();
     }
 
+    /**
+     * The live event timeline for this room, with the oldest event at index 0.
+     *
+     * @deprecated Present for backwards compatibility.
+     *             Use getLiveTimeline().getEvents() instead
+     */
+    public get timeline(): MatrixEvent[] {
+        return this.getLiveTimeline().getEvents();
+    }
+
     /**
      * Get the timestamp of the last message in the room
      *
@@ -980,7 +983,7 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
         // that this function is only called once (unless loading the members
         // fails), since loadMembersIfNeeded always returns this.membersPromise
         // if set, which will be the result of the first (successful) call.
-        if (rawMembersEvents === null || (this.client.isCryptoEnabled() && this.client.isRoomEncrypted(this.roomId))) {
+        if (rawMembersEvents === null || this.hasEncryptionStateEvent()) {
             fromServer = true;
             rawMembersEvents = await this.loadMembersFromServer();
             logger.log(`LL: got ${rawMembersEvents.length} ` + `members from server for room ${this.roomId}`);
@@ -1221,11 +1224,9 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
         const previousOldState = this.oldState;
         const previousCurrentState = this.currentState;
 
-        // maintain this.timeline as a reference to the live timeline,
-        // and this.oldState and this.currentState as references to the
+        // maintain this.oldState and this.currentState as references to the
         // state at the start and end of that timeline. These are more
         // for backwards-compatibility than anything else.
-        this.timeline = this.getLiveTimeline().getEvents();
         this.oldState = this.getLiveTimeline().getState(EventTimeline.BACKWARDS)!;
         this.currentState = this.getLiveTimeline().getState(EventTimeline.FORWARDS)!;
 
@@ -1275,9 +1276,12 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
      * error will be thrown.
      *
      * @returns the result
+     *
+     * @deprecated Not supported under rust crypto. Instead, call {@link Room.getEncryptionTargetMembers},
+     * {@link CryptoApi.getUserDeviceInfo}, and {@link CryptoApi.getDeviceVerificationStatus}.
      */
     public async hasUnverifiedDevices(): Promise<boolean> {
-        if (!this.client.isRoomEncrypted(this.roomId)) {
+        if (!this.hasEncryptionStateEvent()) {
             return false;
         }
         const e2eMembers = await this.getEncryptionTargetMembers();
@@ -1979,6 +1983,7 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
         }
 
         this.on(ThreadEvent.NewReply, this.onThreadReply);
+        this.on(ThreadEvent.Update, this.onThreadUpdate);
         this.on(ThreadEvent.Delete, this.onThreadDelete);
         this.threadsReady = true;
     }
@@ -2082,6 +2087,10 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
         }
     }
 
+    private onThreadUpdate(thread: Thread): void {
+        this.updateThreadRootEvents(thread, false, false);
+    }
+
     private onThreadReply(thread: Thread): void {
         this.updateThreadRootEvents(thread, false, true);
     }
@@ -2329,7 +2338,9 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
             this.lastThread = thread;
         }
 
-        if (this.threadsReady) {
+        // We need to update the thread root events, but the thread may not be ready yet.
+        // If it isn't, it will fire ThreadEvent.Update when it is and we'll call updateThreadRootEvents then.
+        if (this.threadsReady && thread.initialEventsFetched) {
             this.updateThreadRootEvents(thread, toStartOfTimeline, false);
         }
         this.emit(ThreadEvent.New, thread, toStartOfTimeline);
@@ -2558,7 +2569,7 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
                 .filter((event) => {
                     // Filter out the unencrypted messages if the room is encrypted
                     const isEventEncrypted = event.type === EventType.RoomMessageEncrypted;
-                    const isRoomEncrypted = this.client.isRoomEncrypted(this.roomId);
+                    const isRoomEncrypted = this.hasEncryptionStateEvent();
                     return isEventEncrypted || !isRoomEncrypted;
                 });
 
@@ -3163,7 +3174,7 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
     public maySendMessage(): boolean {
         return (
             this.getMyMembership() === "join" &&
-            (this.client.isRoomEncrypted(this.roomId)
+            (this.hasEncryptionStateEvent()
                 ? this.currentState.maySendEvent(EventType.RoomMessageEncrypted, this.myUserId)
                 : this.currentState.maySendEvent(EventType.RoomMessage, this.myUserId))
         );
@@ -3665,6 +3676,18 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
     public compareEventOrdering(leftEventId: string, rightEventId: string): number | null {
         return compareEventOrdering(this, leftEventId, rightEventId);
     }
+
+    /**
+     * Return true if this room has an `m.room.encryption` state event.
+     *
+     * If this returns `true`, events sent to this room should be encrypted (and `MatrixClient.sendEvent` and friends
+     * will encrypt outgoing events).
+     */
+    public hasEncryptionStateEvent(): boolean {
+        return Boolean(
+            this.getLiveTimeline().getState(EventTimeline.FORWARDS)?.getStateEvents(EventType.RoomEncryption, ""),
+        );
+    }
 }
 
 // a map from current event status to a list of allowed next statuses

src/models/thread.ts

@@ -84,7 +84,6 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
      * A reference to all the events ID at the bottom of the threads
      */
     public readonly timelineSet: EventTimelineSet;
-    public timeline: MatrixEvent[] = [];
 
     private _currentUserParticipated = false;
 
@@ -133,7 +132,13 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
     private readonly pendingEventOrdering: PendingEventOrdering;
     private processRootEventPromise?: Promise<void>;
 
+    /**
+     * Whether or not we need to fetch the initial set of events for the thread. We can
+     * only do this if the server has support for it, so if it doesn't we just pretend
+     * that we've already fetched them.
+     */
     public initialEventsFetched = !Thread.hasServerSideSupport;
+
     /**
      * An array of events to add to the timeline once the thread has been initialised
      * with server suppport.
@@ -317,7 +322,6 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
                 fromCache: false,
                 roomState: this.roomState,
             });
-            this.timeline = this.events;
         }
     }
 
@@ -344,9 +348,6 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
             return;
         }
         this.timelineSet.insertEventIntoTimeline(event, this.liveTimeline, this.roomState);
-
-        // As far as we know, timeline should always be the same as events
-        this.timeline = this.events;
     }
 
     public addEvents(events: MatrixEvent[], toStartOfTimeline: boolean): void {
@@ -363,7 +364,7 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
      * to the start (and not the end) of the timeline.
      * @param emit - whether to emit the Update event if the thread was updated or not.
      */
-    public async addEvent(event: MatrixEvent, toStartOfTimeline: boolean, emit = true): Promise<void> {
+    public addEvent(event: MatrixEvent, toStartOfTimeline: boolean, emit = true): void {
         // Modify this event to point at our room's state, and mark its thread
         // as this.
         this.setEventMetadata(event);
@@ -382,56 +383,7 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
             this.addEventToTimeline(event, false);
             this.fetchEditsWhereNeeded(event);
         } else if (event.isRelation(RelationType.Annotation) || event.isRelation(RelationType.Replace)) {
-            // If this event is not a direct member of the thread, but is a
-            // reference to something that is, then we have two cases:
-
-            if (!this.initialEventsFetched) {
-                // Case 1: we haven't yet fetched events from the server. In
-                // this case, when we do, the events we get back might only be
-                // the first-order ones, so this event (which is second-order -
-                // a reference to something directly in the thread) needs to be
-                // kept so we can replay it when the first-order ones turn up.
-
-                /**
-                 * A thread can be fully discovered via a single sync response
-                 * And when that's the case we still ask the server to do an initialisation
-                 * as it's the safest to ensure we have everything.
-                 * However when we are in that scenario we might loose annotation or edits
-                 *
-                 * This fix keeps a reference to those events and replay them once the thread
-                 * has been initialised properly.
-                 */
-                this.replayEvents?.push(event);
-            } else {
-                // Case 2: this is happening later, and we have a timeline. In
-                // this case, these events might be out-of order.
-                //
-                // Specifically, if the server doesn't support recursion, so we
-                // only get these events through sync, they might be coming
-                // later than the first-order ones, so we insert them based on
-                // timestamp (despite the problems with this documented in
-                // #3325).
-                //
-                // If the server does support recursion, we should have got all
-                // the interspersed events from the server when we fetched the
-                // initial events, so if they are coming via sync they should be
-                // the latest ones, so we can add them as normal.
-                //
-                // (Note that both insertEventIntoTimeline and addEventToTimeline
-                // do nothing if we have seen this event before.)
-
-                const recursionSupport =
-                    this.client.canSupport.get(Feature.RelationsRecursion) ?? ServerSupport.Unsupported;
-
-                if (recursionSupport === ServerSupport.Unsupported) {
-                    this.insertEventIntoTimeline(event);
-                } else {
-                    this.addEventToTimeline(event, toStartOfTimeline);
-                }
-            }
-            // Apply annotations and replace relations to the relations of the timeline only
-            this.timelineSet.relations?.aggregateParentEvent(event);
-            this.timelineSet.relations?.aggregateChildEvent(event, this.timelineSet);
+            this.addRelatedThreadEvent(event, toStartOfTimeline);
             return;
         } else if (this.initialEventsFetched) {
             // If initial events have not been fetched, we are OK to throw away
@@ -468,12 +420,64 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
         }
     }
 
+    private addRelatedThreadEvent(event: MatrixEvent, toStartOfTimeline: boolean): void {
+        // If this event is not a direct member of the thread, but is a
+        // reference to something that is, then we have two cases:
+
+        if (!this.initialEventsFetched) {
+            // Case 1: we haven't yet fetched events from the server. In
+            // this case, when we do, the events we get back might only be
+            // the first-order ones, so this event (which is second-order -
+            // a reference to something directly in the thread) needs to be
+            // kept so we can replay it when the first-order ones turn up.
+
+            /**
+             * A thread can be fully discovered via a single sync response
+             * And when that's the case we still ask the server to do an initialisation
+             * as it's the safest to ensure we have everything.
+             * However when we are in that scenario we might loose annotation or edits
+             *
+             * This fix keeps a reference to those events and replay them once the thread
+             * has been initialised properly.
+             */
+            this.replayEvents?.push(event);
+        } else {
+            // Case 2: this is happening later, and we have a timeline. In
+            // this case, these events might be out-of order.
+            //
+            // Specifically, if the server doesn't support recursion, so we
+            // only get these events through sync, they might be coming
+            // later than the first-order ones, so we insert them based on
+            // timestamp (despite the problems with this documented in
+            // #3325).
+            //
+            // If the server does support recursion, we should have got all
+            // the interspersed events from the server when we fetched the
+            // initial events, so if they are coming via sync they should be
+            // the latest ones, so we can add them as normal.
+            //
+            // (Note that both insertEventIntoTimeline and addEventToTimeline
+            // do nothing if we have seen this event before.)
+
+            const recursionSupport =
+                this.client.canSupport.get(Feature.RelationsRecursion) ?? ServerSupport.Unsupported;
+
+            if (recursionSupport === ServerSupport.Unsupported) {
+                this.insertEventIntoTimeline(event);
+            } else {
+                this.addEventToTimeline(event, toStartOfTimeline);
+            }
+        }
+        // Apply annotations and replace relations to the relations of the timeline only
+        this.timelineSet.relations?.aggregateParentEvent(event);
+        this.timelineSet.relations?.aggregateChildEvent(event, this.timelineSet);
+    }
+
     public async processEvent(event: Optional<MatrixEvent>): Promise<void> {
         if (event) {
             this.setEventMetadata(event);
             await this.fetchEditsWhereNeeded(event);
         }
-        this.timeline = this.events;
     }
 
     /**
@@ -717,6 +721,16 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
         return this.lastPendingEvent ?? this.lastEvent ?? this.lastReply();
     }
 
+    /**
+     * The live event timeline for this thread.
+     * @deprecated Present for backwards compatibility.
+     *             Use this.events instead
+     * @returns The live event timeline for this thread.
+     */
+    public get timeline(): MatrixEvent[] {
+        return this.events;
+    }
+
     public get events(): MatrixEvent[] {
         return this.liveTimeline.getEvents();
     }

src/rendezvous/MSC3906Rendezvous.ts

@@ -17,12 +17,12 @@ limitations under the License.
 import { UnstableValue } from "matrix-events-sdk";
 
 import { RendezvousChannel, RendezvousFailureListener, RendezvousFailureReason, RendezvousIntent } from ".";
-import { ICrossSigningKey, IGetLoginTokenCapability, MatrixClient, GET_LOGIN_TOKEN_CAPABILITY } from "../client";
-import { CrossSigningInfo } from "../crypto/CrossSigning";
-import { DeviceInfo } from "../crypto/deviceinfo";
+import { IGetLoginTokenCapability, MatrixClient, GET_LOGIN_TOKEN_CAPABILITY } from "../client";
 import { buildFeatureSupportMap, Feature, ServerSupport } from "../feature";
 import { logger } from "../logger";
 import { sleep } from "../utils";
+import { CrossSigningKey } from "../crypto-api";
+import { Device } from "../matrix";
 
 enum PayloadType {
     Start = "m.login.start",
@@ -116,7 +116,7 @@ export class MSC3906Rendezvous {
 
         await this.send({ type: PayloadType.Progress, protocols: [LOGIN_TOKEN_PROTOCOL.name] });
 
-        logger.info("Waiting for other device to chose protocol");
+        logger.info("Waiting for other device to choose protocol");
         const { type, protocol, outcome } = await this.receive();
 
         if (type === PayloadType.Finish) {
@@ -178,12 +178,8 @@ export class MSC3906Rendezvous {
         return deviceId;
     }
 
-    private async verifyAndCrossSignDevice(
-        deviceInfo: DeviceInfo,
-    ): Promise<CrossSigningInfo | DeviceInfo | ICrossSigningKey | undefined> {
-        if (!this.client.crypto) {
-            throw new Error("Crypto not available on client");
-        }
+    private async verifyAndCrossSignDevice(deviceInfo: Device): Promise<void> {
+        const crypto = this.client.getCrypto()!;
 
         if (!this.newDeviceId) {
             throw new Error("No new device ID set");
@@ -196,36 +192,32 @@ export class MSC3906Rendezvous {
             );
         }
 
-        const userId = this.client.getUserId();
+        const userId = this.client.getSafeUserId();
 
-        if (!userId) {
-            throw new Error("No user ID set");
-        }
         // mark the device as verified locally + cross sign
         logger.info(`Marking device ${this.newDeviceId} as verified`);
-        const info = await this.client.crypto.setDeviceVerification(userId, this.newDeviceId, true, false, true);
+        await crypto.setDeviceVerified(userId, this.newDeviceId, true);
+        await crypto.crossSignDevice(this.newDeviceId);
 
-        const masterPublicKey = this.client.crypto.crossSigningInfo.getId("master")!;
+        const masterPublicKey = (await crypto.getCrossSigningKeyId(CrossSigningKey.Master)) ?? undefined;
+
+        const ourDeviceId = this.client.getDeviceId()!;
+        const ourDeviceKey = (await crypto.getOwnDeviceKeys()).ed25519;
 
         await this.send({
             type: PayloadType.Finish,
             outcome: Outcome.Verified,
-            verifying_device_id: this.client.getDeviceId()!,
-            verifying_device_key: this.client.getDeviceEd25519Key()!,
+            verifying_device_id: ourDeviceId,
+            verifying_device_key: ourDeviceKey,
             master_key: masterPublicKey,
         });
-
-        return info;
     }
 
     /**
      * Verify the device and cross-sign it.
      * @param timeout - time in milliseconds to wait for device to come online
-     * @returns the new device info if the device was verified
      */
-    public async verifyNewDeviceOnExistingDevice(
-        timeout = 10 * 1000,
-    ): Promise<DeviceInfo | CrossSigningInfo | ICrossSigningKey | undefined> {
+    public async verifyNewDeviceOnExistingDevice(timeout = 10 * 1000): Promise<void> {
         if (!this.newDeviceId) {
             throw new Error("No new device to sign");
         }
@@ -235,31 +227,33 @@ export class MSC3906Rendezvous {
             return undefined;
         }
 
-        if (!this.client.crypto) {
+        const crypto = this.client.getCrypto();
+        if (!crypto) {
             throw new Error("Crypto not available on client");
         }
 
-        const userId = this.client.getUserId();
-
-        if (!userId) {
-            throw new Error("No user ID set");
-        }
-
-        let deviceInfo = this.client.crypto.getStoredDevice(userId, this.newDeviceId);
+        let deviceInfo = await this.getOwnDevice(this.newDeviceId);
 
         if (!deviceInfo) {
             logger.info("Going to wait for new device to be online");
             await sleep(timeout);
-            deviceInfo = this.client.crypto.getStoredDevice(userId, this.newDeviceId);
+            deviceInfo = await this.getOwnDevice(this.newDeviceId);
         }
 
         if (deviceInfo) {
-            return await this.verifyAndCrossSignDevice(deviceInfo);
+            await this.verifyAndCrossSignDevice(deviceInfo);
+            return;
         }
 
         throw new Error("Device not online within timeout");
     }
 
+    private async getOwnDevice(deviceId: string): Promise<Device | undefined> {
+        const userId = this.client.getSafeUserId();
+        const ownDeviceInfo = await this.client.getCrypto()!.getUserDeviceInfo([userId]);
+        return ownDeviceInfo.get(userId)?.get(deviceId);
+    }
+
     public async cancel(reason: RendezvousFailureReason): Promise<void> {
         this.onFailure?.(reason);
         await this.channel.cancel(reason);

src/rust-crypto/KeyClaimManager.ts

@@ -73,7 +73,10 @@ export class KeyClaimManager {
             throw new Error(`Cannot ensure Olm sessions: shutting down`);
         }
         logger.info("Checking for missing Olm sessions");
-        const claimRequest = await this.olmMachine.getMissingSessions(userList);
+        // By passing the userId array to rust we transfer ownership of the items to rust, causing
+        // them to be invalidated on the JS side as soon as the method is called.
+        // As we haven't created the `userList` let's clone the users, to not break the caller from re-using it.
+        const claimRequest = await this.olmMachine.getMissingSessions(userList.map((u) => u.clone()));
         if (claimRequest) {
             logger.info("Making /keys/claim request");
             await this.outgoingRequestProcessor.makeOutgoingRequest(claimRequest);

src/rust-crypto/RoomEncryptor.ts

@@ -88,7 +88,8 @@ export class RoomEncryptor {
      */
     public onCryptoEvent(config: IContent): void {
         if (JSON.stringify(this.encryptionSettings) != JSON.stringify(config)) {
-            this.prefixedLogger.error(`Ignoring m.room.encryption event which requests a change of config`);
+            // This should currently be unreachable, since the Rust SDK will reject any attempts to change config.
+            throw new Error("Cannot reconfigure an active RoomEncryptor");
         }
     }
 
@@ -258,6 +259,7 @@ export class RoomEncryptor {
         await logDuration(this.prefixedLogger, "shareRoomKey", async () => {
             const shareMessages: ToDeviceRequest[] = await this.olmMachine.shareRoomKey(
                 new RoomId(this.room.roomId),
+                // safe to pass without cloning, as it's not reused here (before or after)
                 userList,
                 rustEncryptionSettings,
             );

src/rust-crypto/backup.ts

@@ -29,13 +29,14 @@ import { logger } from "../logger";
 import { ClientPrefix, IHttpOpts, MatrixError, MatrixHttpApi, Method } from "../http-api";
 import { CryptoEvent, IMegolmSessionData } from "../crypto";
 import { TypedEventEmitter } from "../models/typed-event-emitter";
-import { encodeUri, immediate, logDuration } from "../utils";
+import { encodeUri, logDuration } from "../utils";
 import { OutgoingRequestProcessor } from "./OutgoingRequestProcessor";
 import { sleep } from "../utils";
 import { BackupDecryptor } from "../common-crypto/CryptoBackend";
 import { IEncryptedPayload } from "../crypto/aes";
 import { ImportRoomKeyProgressData, ImportRoomKeysOpts } from "../crypto-api";
 import { IKeyBackupInfo } from "../crypto/keybackup";
+import { IKeyBackup } from "../crypto/backup";
 
 /** Authentification of the backup info, depends on algorithm */
 type AuthData = KeyBackupInfo["auth_data"];
@@ -128,7 +129,7 @@ export class RustBackupManager extends TypedEventEmitter<RustBackupCryptoEvents,
     /**
      * Handles a backup secret received event and store it if it matches the current backup version.
      *
-     * @param secret - The secret as received from a `m.secert.send` event for secret `m.megolm_backup.v1`.
+     * @param secret - The secret as received from a `m.secret.send` event for secret `m.megolm_backup.v1`.
      * @returns true if the secret is valid and has been stored, false otherwise.
      */
     public async handleBackupSecretReceived(secret: string): Promise<boolean> {
@@ -141,7 +142,9 @@ export class RustBackupManager extends TypedEventEmitter<RustBackupCryptoEvents,
         if (!backupCheck?.backupInfo?.version || !backupCheck.trustInfo.trusted) {
             // There is no server-side key backup, or the backup is not signed by a trusted cross-signing key or trusted own device.
             // This decryption key is useless to us.
-            logger.warn("Received backup decryption key, but there is no trusted server-side key backup");
+            logger.warn(
+                "handleBackupSecretReceived: Received a backup decryption key, but there is no trusted server-side key backup",
+            );
             return false;
         }
 
@@ -149,7 +152,9 @@ export class RustBackupManager extends TypedEventEmitter<RustBackupCryptoEvents,
             const backupDecryptionKey = RustSdkCryptoJs.BackupDecryptionKey.fromBase64(secret);
             const privateKeyMatches = backupInfoMatchesBackupDecryptionKey(backupCheck.backupInfo, backupDecryptionKey);
             if (!privateKeyMatches) {
-                logger.debug(`onReceiveSecret: backup decryption key does not match current backup version`);
+                logger.warn(
+                    `handleBackupSecretReceived: Private decryption key does not match the public key of the current remote backup.`,
+                );
                 // just ignore the secret
                 return false;
             }
@@ -207,15 +212,18 @@ export class RustBackupManager extends TypedEventEmitter<RustBackupCryptoEvents,
             }
             keysByRoom.get(roomId)!.set(key.session_id, key);
         }
-        await this.olmMachine.importBackedUpRoomKeys(keysByRoom, (progress: BigInt, total: BigInt): void => {
-            const importOpt: ImportRoomKeyProgressData = {
-                total: Number(total),
-                successes: Number(progress),
-                stage: "load_keys",
-                failures: 0,
-            };
-            opts?.progressCallback?.(importOpt);
-        });
+        await this.olmMachine.importBackedUpRoomKeys(
+            keysByRoom,
+            (progress: BigInt, total: BigInt, failures: BigInt): void => {
+                const importOpt: ImportRoomKeyProgressData = {
+                    total: Number(total),
+                    successes: Number(progress),
+                    stage: "load_keys",
+                    failures: Number(failures),
+                };
+                opts?.progressCallback?.(importOpt);
+            },
+        );
     }
 
     private keyBackupCheckInProgress: Promise<KeyBackupCheck | null> | null = null;
@@ -323,7 +331,13 @@ export class RustBackupManager extends TypedEventEmitter<RustBackupCryptoEvents,
         await sleep(delay);
 
         try {
-            let numFailures = 0; // number of consecutive network failures for exponential backoff
+            // number of consecutive network failures for exponential backoff
+            let numFailures = 0;
+            // The number of keys left to back up. (Populated lazily: see more comments below.)
+            let remainingToUploadCount: number | null = null;
+            // To avoid computing the key when only a few keys were added (after a sync for example),
+            // we compute the count only when at least two iterations are needed.
+            let isFirstIteration = true;
 
             while (!this.stopped) {
                 // Get a batch of room keys to upload
@@ -342,6 +356,10 @@ export class RustBackupManager extends TypedEventEmitter<RustBackupCryptoEvents,
 
                 if (!request || this.stopped || !this.activeBackupVersion) {
                     logger.log(`Backup: Ending loop for version ${this.activeBackupVersion}.`);
+                    if (!request) {
+                        // nothing more to upload
+                        this.emit(CryptoEvent.KeyBackupSessionsRemaining, 0);
+                    }
                     return;
                 }
 
@@ -349,12 +367,34 @@ export class RustBackupManager extends TypedEventEmitter<RustBackupCryptoEvents,
                     await this.outgoingRequestProcessor.makeOutgoingRequest(request);
                     numFailures = 0;
                     if (this.stopped) break;
-                    try {
-                        const keyCount = await this.olmMachine.roomKeyCounts();
-                        const remaining = keyCount.total - keyCount.backedUp;
-                        this.emit(CryptoEvent.KeyBackupSessionsRemaining, remaining);
-                    } catch (err) {
-                        logger.error("Backup: Failed to get key counts from rust crypto-sdk", err);
+
+                    // Key count performance (`olmMachine.roomKeyCounts()`) can be pretty bad on some configurations.
+                    // In particular, we detected on some M1 macs that when the object store reaches a threshold, the count
+                    // performance stops growing in O(n) and suddenly becomes very slow (40s, 60s or more).
+                    // For reference, the performance drop occurs around 300-400k keys on the platforms where this issue is observed.
+                    // Even on other configurations, the count can take several seconds.
+                    // This will block other operations on the database, like sending messages.
+                    //
+                    // This is a workaround to avoid calling `olmMachine.roomKeyCounts()` too often, and only when necessary.
+                    // We don't call it on the first loop because there could be only a few keys to upload, and we don't want to wait for the count.
+                    if (!isFirstIteration && remainingToUploadCount === null) {
+                        try {
+                            const keyCount = await this.olmMachine.roomKeyCounts();
+                            remainingToUploadCount = keyCount.total - keyCount.backedUp;
+                        } catch (err) {
+                            logger.error("Backup: Failed to get key counts from rust crypto-sdk", err);
+                        }
+                    }
+
+                    if (remainingToUploadCount !== null) {
+                        this.emit(CryptoEvent.KeyBackupSessionsRemaining, remainingToUploadCount);
+                        const keysCountInBatch = this.keysCountInBatch(request);
+                        // `OlmMachine.roomKeyCounts` is called only once for the current backupKeysLoop. But new
+                        // keys could be added during the current loop (after a sync for example).
+                        // So the count can get out of sync with the real number of remaining keys to upload.
+                        // Depending on the number of new keys imported and the time to complete the loop,
+                        // this could result in multiple events being emitted with a remaining key count of 0.
+                        remainingToUploadCount = Math.max(remainingToUploadCount - keysCountInBatch, 0);
                     }
                 } catch (err) {
                     numFailures++;
@@ -378,7 +418,7 @@ export class RustBackupManager extends TypedEventEmitter<RustBackupCryptoEvents,
                             // wait for that and then continue?
                             const waitTime = err.data.retry_after_ms;
                             if (waitTime > 0) {
-                                sleep(waitTime);
+                                await sleep(waitTime);
                                 continue;
                             } // else go to the normal backoff
                         }
@@ -388,12 +428,29 @@ export class RustBackupManager extends TypedEventEmitter<RustBackupCryptoEvents,
                     // exponential backoff if we have failures
                     await sleep(1000 * Math.pow(2, Math.min(numFailures - 1, 4)));
                 }
+                isFirstIteration = false;
             }
         } finally {
             this.backupKeysLoopRunning = false;
         }
     }
 
+    /**
+     * Utility method to count the number of keys in a backup request, in order to update the remaining keys count.
+     * This should be the chunk size of the backup request for all requests but the last, but we don't have access to it
+     * (it's static in the Rust SDK).
+     * @param batch - The backup request to count the keys from.
+     *
+     * @returns The number of keys in the backup request.
+     */
+    private keysCountInBatch(batch: RustSdkCryptoJs.KeysBackupRequest): number {
+        const parsedBody: IKeyBackup = JSON.parse(batch.body);
+        let count = 0;
+        for (const { sessions } of Object.values(parsedBody.rooms)) {
+            count += Object.keys(sessions).length;
+        }
+        return count;
+    }
     /**
      * Get information about the current key backup from the server
      *
@@ -534,9 +591,6 @@ export class RustBackupDecryptor implements BackupDecryptor {
                 );
                 decrypted.session_id = sessionId;
                 keys.push(decrypted);
-
-                // there might be lots of sessions, so don't hog the event loop
-                await immediate();
             } catch (e) {
                 logger.log("Failed to decrypt megolm session from backup", e, sessionData);
             }

src/rust-crypto/index.ts

@@ -15,12 +15,15 @@ limitations under the License.
 */
 
 import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
+import { StoreHandle } from "@matrix-org/matrix-sdk-crypto-wasm";
 
 import { RustCrypto } from "./rust-crypto";
 import { IHttpOpts, MatrixHttpApi } from "../http-api";
 import { ServerSideSecretStorage } from "../secret-storage";
 import { ICryptoCallbacks } from "../crypto";
 import { Logger } from "../logger";
+import { CryptoStore } from "../crypto/store/base";
+import { migrateFromLegacyCrypto, migrateRoomSettingsFromLegacyCrypto } from "./libolm_migration";
 
 /**
  * Create a new `RustCrypto` implementation
@@ -63,6 +66,19 @@ export async function initRustCrypto(args: {
      * will be unencrypted.
      */
     storePassphrase?: string;
+
+    /** If defined, we will check if any data needs migrating from this store to the rust store. */
+    legacyCryptoStore?: CryptoStore;
+
+    /** The pickle key for `legacyCryptoStore` */
+    legacyPickleKey?: string;
+
+    /**
+     * A callback which will receive progress updates on migration from `legacyCryptoStore`.
+     *
+     * Called with (-1, -1) to mark the end of migration.
+     */
+    legacyMigrationProgressListener?: (progress: number, total: number) => void;
 }): Promise<RustCrypto> {
     const { logger } = args;
 
@@ -73,6 +89,21 @@ export async function initRustCrypto(args: {
     // enable tracing in the rust-sdk
     new RustSdkCryptoJs.Tracing(RustSdkCryptoJs.LoggerLevel.Debug).turnOn();
 
+    logger.debug("Opening Rust CryptoStore");
+    const storeHandle: StoreHandle = await StoreHandle.open(
+        args.storePrefix ?? undefined,
+        (args.storePrefix && args.storePassphrase) ?? undefined,
+    );
+
+    if (args.legacyCryptoStore) {
+        // We have a legacy crypto store, which we may need to migrate from.
+        await migrateFromLegacyCrypto({
+            legacyStore: args.legacyCryptoStore,
+            storeHandle,
+            ...args,
+        });
+    }
+
     const rustCrypto = await initOlmMachine(
         logger,
         args.http,
@@ -80,10 +111,12 @@ export async function initRustCrypto(args: {
         args.deviceId,
         args.secretStorage,
         args.cryptoCallbacks,
-        args.storePrefix,
-        args.storePassphrase,
+        storeHandle,
+        args.legacyCryptoStore,
     );
 
+    storeHandle.free();
+
     logger.debug("Completed rust crypto-sdk setup");
     return rustCrypto;
 }
@@ -95,17 +128,26 @@ async function initOlmMachine(
     deviceId: string,
     secretStorage: ServerSideSecretStorage,
     cryptoCallbacks: ICryptoCallbacks,
-    storePrefix: string | null,
-    storePassphrase: string | undefined,
+    storeHandle: StoreHandle,
+    legacyCryptoStore?: CryptoStore,
 ): Promise<RustCrypto> {
     logger.debug("Init OlmMachine");
-    const olmMachine = await RustSdkCryptoJs.OlmMachine.initialize(
+
+    const olmMachine = await RustSdkCryptoJs.OlmMachine.initFromStore(
         new RustSdkCryptoJs.UserId(userId),
         new RustSdkCryptoJs.DeviceId(deviceId),
-        storePrefix ?? undefined,
-        (storePrefix && storePassphrase) ?? undefined,
+        storeHandle,
     );
 
+    // A final migration step, now that we have an OlmMachine.
+    if (legacyCryptoStore) {
+        await migrateRoomSettingsFromLegacyCrypto({
+            logger,
+            legacyStore: legacyCryptoStore,
+            olmMachine,
+        });
+    }
+
     // Disable room key requests, per https://github.com/vector-im/element-web/issues/26524.
     olmMachine.roomKeyRequestsEnabled = false;
 
@@ -116,6 +158,7 @@ async function initOlmMachine(
     await olmMachine.registerUserIdentityUpdatedCallback((userId: RustSdkCryptoJs.UserId) =>
         rustCrypto.onUserIdentityUpdated(userId),
     );
+    await olmMachine.registerDevicesUpdatedCallback((userIds: string[]) => rustCrypto.onDevicesUpdated(userIds));
 
     // Check if there are any key backup secrets pending processing. There may be multiple secrets to process if several devices have gossiped them.
     // The `registerReceiveSecretCallback` function will only be triggered for new secrets. If the client is restarted before processing them, the secrets will need to be manually handled.

src/rust-crypto/libolm_migration.ts

@@ -0,0 +1,382 @@
+/*
+Copyright 2023-2024 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
+
+import { Logger } from "../logger";
+import { CryptoStore, MigrationState, SecretStorePrivateKeys } from "../crypto/store/base";
+import { IndexedDBCryptoStore } from "../crypto/store/indexeddb-crypto-store";
+import { decryptAES, IEncryptedPayload } from "../crypto/aes";
+import { IHttpOpts, MatrixHttpApi } from "../http-api";
+import { requestKeyBackupVersion } from "./backup";
+import { IRoomEncryption } from "../crypto/RoomList";
+
+/**
+ * Determine if any data needs migrating from the legacy store, and do so.
+ *
+ * This migrates the base account data, and olm and megolm sessions. It does *not* migrate the room list, which should
+ * happen after an `OlmMachine` is created, via {@link migrateRoomSettingsFromLegacyCrypto}.
+ *
+ * @param args - Arguments object.
+ */
+export async function migrateFromLegacyCrypto(args: {
+    /** A `Logger` instance that will be used for debug output. */
+    logger: Logger;
+
+    /**
+     * Low-level HTTP interface: used to make outgoing requests required by the rust SDK.
+     * We expect it to set the access token, etc.
+     */
+    http: MatrixHttpApi<IHttpOpts & { onlyData: true }>;
+
+    /** Store to migrate data from. */
+    legacyStore: CryptoStore;
+
+    /** Pickle key for `legacyStore`. */
+    legacyPickleKey?: string;
+
+    /** Local user's User ID. */
+    userId: string;
+
+    /** Local user's Device ID. */
+    deviceId: string;
+
+    /** Rust crypto store to migrate data into. */
+    storeHandle: RustSdkCryptoJs.StoreHandle;
+
+    /**
+     * A callback which will receive progress updates on migration from `legacyStore`.
+     *
+     * Called with (-1, -1) to mark the end of migration.
+     */
+    legacyMigrationProgressListener?: (progress: number, total: number) => void;
+}): Promise<void> {
+    const { logger, legacyStore } = args;
+
+    // initialise the rust matrix-sdk-crypto-wasm, if it hasn't already been done
+    await RustSdkCryptoJs.initAsync();
+
+    // enable tracing in the rust-sdk
+    new RustSdkCryptoJs.Tracing(RustSdkCryptoJs.LoggerLevel.Debug).turnOn();
+
+    if (!(await legacyStore.containsData())) {
+        // This store was never used. Nothing to migrate.
+        return;
+    }
+
+    await legacyStore.startup();
+    let migrationState = await legacyStore.getMigrationState();
+
+    if (migrationState >= MigrationState.MEGOLM_SESSIONS_MIGRATED) {
+        // All migration is done for now. The room list comes later, once we have an OlmMachine.
+        return;
+    }
+
+    const nOlmSessions = await countOlmSessions(logger, legacyStore);
+    const nMegolmSessions = await countMegolmSessions(logger, legacyStore);
+    const totalSteps = 1 + nOlmSessions + nMegolmSessions;
+    logger.info(
+        `Migrating data from legacy crypto store. ${nOlmSessions} olm sessions and ${nMegolmSessions} megolm sessions to migrate.`,
+    );
+
+    let stepsDone = 0;
+    function onProgress(steps: number): void {
+        stepsDone += steps;
+        args.legacyMigrationProgressListener?.(stepsDone, totalSteps);
+    }
+    onProgress(0);
+
+    const pickleKey = new TextEncoder().encode(args.legacyPickleKey);
+
+    if (migrationState === MigrationState.NOT_STARTED) {
+        logger.info("Migrating data from legacy crypto store. Step 1: base data");
+        await migrateBaseData(args.http, args.userId, args.deviceId, legacyStore, pickleKey, args.storeHandle);
+
+        migrationState = MigrationState.INITIAL_DATA_MIGRATED;
+        await legacyStore.setMigrationState(migrationState);
+    }
+    onProgress(1);
+
+    if (migrationState === MigrationState.INITIAL_DATA_MIGRATED) {
+        logger.info(
+            `Migrating data from legacy crypto store. Step 2: olm sessions (${nOlmSessions} sessions to migrate).`,
+        );
+        await migrateOlmSessions(logger, legacyStore, pickleKey, args.storeHandle, onProgress);
+
+        migrationState = MigrationState.OLM_SESSIONS_MIGRATED;
+        await legacyStore.setMigrationState(migrationState);
+    }
+
+    if (migrationState === MigrationState.OLM_SESSIONS_MIGRATED) {
+        logger.info(
+            `Migrating data from legacy crypto store. Step 3: megolm sessions (${nMegolmSessions} sessions to migrate).`,
+        );
+        await migrateMegolmSessions(logger, legacyStore, pickleKey, args.storeHandle, onProgress);
+
+        migrationState = MigrationState.MEGOLM_SESSIONS_MIGRATED;
+        await legacyStore.setMigrationState(migrationState);
+    }
+
+    // Migration is done.
+    args.legacyMigrationProgressListener?.(-1, -1);
+    logger.info("Migration from legacy crypto store complete");
+}
+
+async function migrateBaseData(
+    http: MatrixHttpApi<IHttpOpts & { onlyData: true }>,
+    userId: string,
+    deviceId: string,
+    legacyStore: CryptoStore,
+    pickleKey: Uint8Array,
+    storeHandle: RustSdkCryptoJs.StoreHandle,
+): Promise<void> {
+    const migrationData = new RustSdkCryptoJs.BaseMigrationData();
+    migrationData.userId = new RustSdkCryptoJs.UserId(userId);
+    migrationData.deviceId = new RustSdkCryptoJs.DeviceId(deviceId);
+
+    await legacyStore.doTxn("readonly", [IndexedDBCryptoStore.STORE_ACCOUNT], (txn) =>
+        legacyStore.getAccount(txn, (a) => {
+            migrationData.pickledAccount = a ?? "";
+        }),
+    );
+
+    const recoveryKey = await getAndDecryptCachedSecretKey(legacyStore, pickleKey, "m.megolm_backup.v1");
+
+    // If we have a backup recovery key, we need to try to figure out which backup version it is for.
+    // All we can really do is ask the server for the most recent version.
+    if (recoveryKey) {
+        const backupInfo = await requestKeyBackupVersion(http);
+        if (backupInfo) {
+            migrationData.backupVersion = backupInfo.version;
+            migrationData.backupRecoveryKey = recoveryKey;
+        }
+    }
+
+    migrationData.privateCrossSigningMasterKey = await getAndDecryptCachedSecretKey(legacyStore, pickleKey, "master");
+    migrationData.privateCrossSigningSelfSigningKey = await getAndDecryptCachedSecretKey(
+        legacyStore,
+        pickleKey,
+        "self_signing",
+    );
+    migrationData.privateCrossSigningUserSigningKey = await getAndDecryptCachedSecretKey(
+        legacyStore,
+        pickleKey,
+        "user_signing",
+    );
+    await RustSdkCryptoJs.Migration.migrateBaseData(migrationData, pickleKey, storeHandle);
+}
+
+async function countOlmSessions(logger: Logger, legacyStore: CryptoStore): Promise<number> {
+    logger.debug("Counting olm sessions to be migrated");
+    let nSessions: number;
+    await legacyStore.doTxn("readonly", [IndexedDBCryptoStore.STORE_SESSIONS], (txn) =>
+        legacyStore.countEndToEndSessions(txn, (n) => (nSessions = n)),
+    );
+    return nSessions!;
+}
+
+async function countMegolmSessions(logger: Logger, legacyStore: CryptoStore): Promise<number> {
+    logger.debug("Counting megolm sessions to be migrated");
+    return await legacyStore.countEndToEndInboundGroupSessions();
+}
+
+async function migrateOlmSessions(
+    logger: Logger,
+    legacyStore: CryptoStore,
+    pickleKey: Uint8Array,
+    storeHandle: RustSdkCryptoJs.StoreHandle,
+    onBatchDone: (batchSize: number) => void,
+): Promise<void> {
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const batch = await legacyStore.getEndToEndSessionsBatch();
+        if (batch === null) return;
+
+        logger.debug(`Migrating batch of ${batch.length} olm sessions`);
+        const migrationData: RustSdkCryptoJs.PickledSession[] = [];
+        for (const session of batch) {
+            const pickledSession = new RustSdkCryptoJs.PickledSession();
+            pickledSession.senderKey = session.deviceKey!;
+            pickledSession.pickle = session.session!;
+            pickledSession.lastUseTime = pickledSession.creationTime = new Date(session.lastReceivedMessageTs!);
+            migrationData.push(pickledSession);
+        }
+
+        await RustSdkCryptoJs.Migration.migrateOlmSessions(migrationData, pickleKey, storeHandle);
+        await legacyStore.deleteEndToEndSessionsBatch(batch);
+        onBatchDone(batch.length);
+    }
+}
+
+async function migrateMegolmSessions(
+    logger: Logger,
+    legacyStore: CryptoStore,
+    pickleKey: Uint8Array,
+    storeHandle: RustSdkCryptoJs.StoreHandle,
+    onBatchDone: (batchSize: number) => void,
+): Promise<void> {
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const batch = await legacyStore.getEndToEndInboundGroupSessionsBatch();
+        if (batch === null) return;
+
+        logger.debug(`Migrating batch of ${batch.length} megolm sessions`);
+        const migrationData: RustSdkCryptoJs.PickledInboundGroupSession[] = [];
+        for (const session of batch) {
+            const sessionData = session.sessionData!;
+
+            const pickledSession = new RustSdkCryptoJs.PickledInboundGroupSession();
+            pickledSession.pickle = sessionData.session;
+            pickledSession.roomId = new RustSdkCryptoJs.RoomId(sessionData.room_id);
+            pickledSession.senderKey = session.senderKey;
+            pickledSession.senderSigningKey = sessionData.keysClaimed?.["ed25519"];
+            pickledSession.backedUp = !session.needsBackup;
+
+            // The Rust SDK `imported` flag is used to indicate the authenticity status of a Megolm
+            // session, which tells us whether we can reliably tell which Olm device is the owner
+            // (creator) of the session.
+            //
+            // If `imported` is true, then we have no cryptographic proof that the session is owned
+            // by the device with the identity key `senderKey`.
+            //
+            // Only Megolm sessions received directly from the owning device via an encrypted
+            // `m.room_key` to-device message should have `imported` flag set to false. Megolm
+            // sessions received by any other currently available means (i.e. from a
+            // `m.forwarded_room_key`, from v1 asymmetric server-side key backup, imported from a
+            // file, etc) should have the `imported` flag set to true.
+            //
+            // Messages encrypted with such Megolm sessions will have a grey shield in the UI
+            // ("Authenticity of this message cannot be guaranteed").
+            //
+            // However, we don't want to bluntly mark all sessions as `imported` during migration
+            // because users will suddenly start seeing all their historic messages decorated with a
+            // grey shield, which would be seen as a non-actionable regression.
+            //
+            // In the legacy crypto stack, the flag encoding similar information was called
+            // `InboundGroupSessionData.untrusted`. The value of this flag was set as follows:
+            //
+            // - For outbound Megolm sessions created by our own device, `untrusted` is `undefined`.
+            // - For Megolm sessions received via a `m.room_key` to-device message, `untrusted` is
+            //   `undefined`.
+            // - For Megolm sessions received via a `m.forwarded_room_key` to-device message,
+            //   `untrusted` is `true`.
+            // - For Megolm sessions imported from a (v1 asymmetric / "legacy") server-side key
+            //   backup, `untrusted` is `true`.
+            // - For Megolm sessions imported from a file, untrusted is `undefined`.
+            //
+            // The main difference between the legacy crypto stack and the Rust crypto stack is that
+            // the Rust stack considers sessions imported from a file as `imported` (not
+            // authenticated). This is because the Megolm session export file format does not
+            // encode this authenticity information.
+            //
+            // Given this migration is only a one-time thing, we make a concession to accept the
+            // loss of information in this case, to avoid degrading UX in a non-actionable way.
+            pickledSession.imported = sessionData.untrusted === true;
+
+            migrationData.push(pickledSession);
+        }
+
+        await RustSdkCryptoJs.Migration.migrateMegolmSessions(migrationData, pickleKey, storeHandle);
+        await legacyStore.deleteEndToEndInboundGroupSessionsBatch(batch);
+        onBatchDone(batch.length);
+    }
+}
+
+/**
+ * Determine if any room settings need migrating from the legacy store, and do so.
+ *
+ * @param args - Arguments object.
+ */
+export async function migrateRoomSettingsFromLegacyCrypto({
+    logger,
+    legacyStore,
+    olmMachine,
+}: {
+    /** A `Logger` instance that will be used for debug output. */
+    logger: Logger;
+
+    /** Store to migrate data from. */
+    legacyStore: CryptoStore;
+
+    /** OlmMachine to store the new data on. */
+    olmMachine: RustSdkCryptoJs.OlmMachine;
+}): Promise<void> {
+    if (!(await legacyStore.containsData())) {
+        // This store was never used. Nothing to migrate.
+        return;
+    }
+
+    const migrationState = await legacyStore.getMigrationState();
+
+    if (migrationState >= MigrationState.ROOM_SETTINGS_MIGRATED) {
+        // We've already migrated the room settings.
+        return;
+    }
+
+    let rooms: Record<string, IRoomEncryption> = {};
+
+    await legacyStore.doTxn("readwrite", [IndexedDBCryptoStore.STORE_ROOMS], (txn) => {
+        legacyStore.getEndToEndRooms(txn, (result) => {
+            rooms = result;
+        });
+    });
+
+    logger.debug(`Migrating ${Object.keys(rooms).length} sets of room settings`);
+    for (const [roomId, legacySettings] of Object.entries(rooms)) {
+        try {
+            const rustSettings = new RustSdkCryptoJs.RoomSettings();
+
+            if (legacySettings.algorithm !== "m.megolm.v1.aes-sha2") {
+                logger.warn(`Room ${roomId}: ignoring room with invalid algorithm ${legacySettings.algorithm}`);
+                continue;
+            }
+            rustSettings.algorithm = RustSdkCryptoJs.EncryptionAlgorithm.MegolmV1AesSha2;
+            rustSettings.sessionRotationPeriodMs = legacySettings.rotation_period_ms;
+            rustSettings.sessionRotationPeriodMessages = legacySettings.rotation_period_msgs;
+            await olmMachine.setRoomSettings(new RustSdkCryptoJs.RoomId(roomId), rustSettings);
+
+            // We don't attempt to clear out the settings from the old store, or record where we've gotten up to,
+            // which means that if the app gets restarted while we're in the middle of this migration, we'll start
+            // again from scratch. So be it. Given that legacy crypto loads the whole room list into memory on startup
+            // anyway, we know it can't be that big.
+        } catch (e) {
+            logger.warn(`Room ${roomId}: ignoring settings ${JSON.stringify(legacySettings)} which caused error ${e}`);
+        }
+    }
+
+    logger.debug(`Completed room settings migration`);
+    await legacyStore.setMigrationState(MigrationState.ROOM_SETTINGS_MIGRATED);
+}
+
+async function getAndDecryptCachedSecretKey(
+    legacyStore: CryptoStore,
+    legacyPickleKey: Uint8Array,
+    name: string,
+): Promise<string | undefined> {
+    let encodedKey: IEncryptedPayload | null = null;
+
+    await legacyStore.doTxn("readonly", "account", (txn) => {
+        legacyStore.getSecretStorePrivateKey(
+            txn,
+            (k) => {
+                encodedKey = k as IEncryptedPayload | null;
+            },
+            name as keyof SecretStorePrivateKeys,
+        );
+    });
+
+    return encodedKey === null ? undefined : await decryptAES(encodedKey, legacyPickleKey, name);
+}

src/rust-crypto/rust-crypto.ts

@@ -311,6 +311,16 @@ export class RustCrypto extends TypedEventEmitter<RustCryptoEvents, RustCryptoEv
         return `Rust SDK ${versions.matrix_sdk_crypto} (${versions.git_sha}), Vodozemac ${versions.vodozemac}`;
     }
 
+    /**
+     * Implementation of {@link CryptoApi#isEncryptionEnabledInRoom}.
+     */
+    public async isEncryptionEnabledInRoom(roomId: string): Promise<boolean> {
+        const roomSettings: RustSdkCryptoJs.RoomSettings | undefined = await this.olmMachine.getRoomSettings(
+            new RustSdkCryptoJs.RoomId(roomId),
+        );
+        return Boolean(roomSettings?.algorithm);
+    }
+
     /**
      * Implementation of {@link CryptoApi#getOwnDeviceKeys}.
      */
@@ -359,7 +369,10 @@ export class RustCrypto extends TypedEventEmitter<RustCryptoEvents, RustCryptoEv
                 /* make sure we have an *up-to-date* idea of the user's cross-signing keys. This is important, because if we
                  * return "false" here, we will end up generating new cross-signing keys and replacing the existing ones.
                  */
-                const request = this.olmMachine.queryKeysForUsers([rustTrackedUser]);
+                const request = this.olmMachine.queryKeysForUsers(
+                    // clone as rust layer will take ownership and it's reused later
+                    [rustTrackedUser.clone()],
+                );
                 await this.outgoingRequestProcessor.makeOutgoingRequest(request);
             }
             const userIdentity = await this.olmMachine.getIdentity(rustTrackedUser);
@@ -1282,7 +1295,27 @@ export class RustCrypto extends TypedEventEmitter<RustCryptoEvents, RustCryptoEv
      */
     public async onCryptoEvent(room: Room, event: MatrixEvent): Promise<void> {
         const config = event.getContent();
+        const settings = new RustSdkCryptoJs.RoomSettings();
 
+        if (config.algorithm === "m.megolm.v1.aes-sha2") {
+            settings.algorithm = RustSdkCryptoJs.EncryptionAlgorithm.MegolmV1AesSha2;
+        } else {
+            // Among other situations, this happens if the crypto state event is redacted.
+            this.logger.warn(`Room ${room.roomId}: ignoring crypto event with invalid algorithm ${config.algorithm}`);
+            return;
+        }
+
+        try {
+            settings.sessionRotationPeriodMs = config.rotation_period_ms;
+            settings.sessionRotationPeriodMessages = config.rotation_period_msgs;
+            await this.olmMachine.setRoomSettings(new RustSdkCryptoJs.RoomId(room.roomId), settings);
+        } catch (e) {
+            this.logger.warn(`Room ${room.roomId}: ignoring crypto event which caused error: ${e}`);
+            return;
+        }
+
+        // If we got this far, the SDK found the event acceptable.
+        // We need to either create or update the active RoomEncryptor.
         const existingEncryptor = this.roomEncryptors[room.roomId];
         if (existingEncryptor) {
             existingEncryptor.onCryptoEvent(config);
@@ -1408,7 +1441,7 @@ export class RustCrypto extends TypedEventEmitter<RustCryptoEvents, RustCryptoEv
      * Callback for `OlmMachine.registerUserIdentityUpdatedCallback`
      *
      * Called by the rust-sdk whenever there is an update to any user's cross-signing status. We re-check their trust
-     * status and emit a `UserTrustStatusChanged` event.
+     * status and emit a `UserTrustStatusChanged` event, as well as a `KeysChanged` if it is our own identity that changed.
      *
      * @param userId - the user with the updated identity
      */
@@ -1419,10 +1452,26 @@ export class RustCrypto extends TypedEventEmitter<RustCryptoEvents, RustCryptoEv
         // If our own user identity has changed, we may now trust the key backup where we did not before.
         // So, re-check the key backup status and enable it if available.
         if (userId.toString() === this.userId) {
+            this.emit(CryptoEvent.KeysChanged, {});
             await this.checkKeyBackupAndEnable();
         }
     }
 
+    /**
+     * Callback for `OlmMachine.registerDevicesUpdatedCallback`
+     *
+     * Called when users' devices have updated. Emits `WillUpdateDevices` and `DevicesUpdated`. In the JavaScript
+     * crypto backend, these events are called at separate times, with `WillUpdateDevices` being emitted just before
+     * the devices are saved, and `DevicesUpdated` being emitted just after. But the OlmMachine only gives us
+     * one event, so we emit both events here.
+     *
+     * @param userIds - an array of user IDs of users whose devices have updated.
+     */
+    public async onDevicesUpdated(userIds: string[]): Promise<void> {
+        this.emit(CryptoEvent.WillUpdateDevices, userIds, false);
+        this.emit(CryptoEvent.DevicesUpdated, userIds, false);
+    }
+
     /**
      * Handles secret received from the rust secret inbox.
      *
@@ -1795,6 +1844,9 @@ function rustEncryptionInfoToJsEncryptionInfo(
 type RustCryptoEvents =
     | CryptoEvent.VerificationRequestReceived
     | CryptoEvent.UserTrustStatusChanged
+    | CryptoEvent.KeysChanged
+    | CryptoEvent.WillUpdateDevices
+    | CryptoEvent.DevicesUpdated
     | RustBackupCryptoEvents;
 
 type RustCryptoEventMap = {
@@ -1809,4 +1861,31 @@ type RustCryptoEventMap = {
     [CryptoEvent.UserTrustStatusChanged]: (userId: string, userTrustLevel: UserVerificationStatus) => void;
 
     [CryptoEvent.KeyBackupDecryptionKeyCached]: (version: string) => void;
+    /**
+     * Fires when the user's cross-signing keys have changed or cross-signing
+     * has been enabled/disabled. The client can use getStoredCrossSigningForUser
+     * with the user ID of the logged in user to check if cross-signing is
+     * enabled on the account. If enabled, it can test whether the current key
+     * is trusted using with checkUserTrust with the user ID of the logged
+     * in user. The checkOwnCrossSigningTrust function may be used to reconcile
+     * the trust in the account key.
+     *
+     * The cross-signing API is currently UNSTABLE and may change without notice.
+     * @experimental
+     */
+    [CryptoEvent.KeysChanged]: (data: {}) => void;
+    /**
+     * Fires whenever the stored devices for a user will be updated
+     * @param users - A list of user IDs that will be updated
+     * @param initialFetch - If true, the store is empty (apart
+     *     from our own device) and is being seeded.
+     */
+    [CryptoEvent.WillUpdateDevices]: (users: string[], initialFetch: boolean) => void;
+    /**
+     * Fires whenever the stored devices for a user have changed
+     * @param users - A list of user IDs that were updated
+     * @param initialFetch - If true, the store was empty (apart
+     *     from our own device) and has been seeded.
+     */
+    [CryptoEvent.DevicesUpdated]: (users: string[], initialFetch: boolean) => void;
 } & RustBackupCryptoEventMap;

src/rust-crypto/verification.ts

@@ -15,9 +15,10 @@ limitations under the License.
 */
 
 import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
-import { Emoji, QrState } from "@matrix-org/matrix-sdk-crypto-wasm";
+import { QrState } from "@matrix-org/matrix-sdk-crypto-wasm";
 
 import {
+    GeneratedSas,
     ShowQrCodeCallbacks,
     ShowSasCallbacks,
     VerificationPhase,
@@ -33,6 +34,7 @@ import { OutgoingRequest, OutgoingRequestProcessor } from "./OutgoingRequestProc
 import { TypedReEmitter } from "../ReEmitter";
 import { MatrixEvent } from "../models/event";
 import { EventType, MsgType } from "../@types/event";
+import { defer, IDeferred } from "../utils";
 
 /**
  * An incoming, or outgoing, request to verify a user or a device via cross-signing.
@@ -75,11 +77,16 @@ export class RustVerificationRequest
         const onChange = async (): Promise<void> => {
             const verification: RustSdkCryptoJs.Qr | RustSdkCryptoJs.Sas | undefined = this.inner.getVerification();
 
-            // If we now have a `Verification` where we lacked one before, or we have transitioned from QR to SAS,
-            // wrap the new rust Verification as a js-sdk Verifier.
+            // Set the _verifier object (wrapping the rust `Verification` as a js-sdk Verifier) if:
+            // - we now have a `Verification` where we lacked one before
+            // - we have transitioned from QR to SAS
+            // - we are verifying with SAS, but we need to replace our verifier with a new one because both parties
+            //   tried to start verification at the same time, and we lost the tie breaking
             if (verification instanceof RustSdkCryptoJs.Sas) {
                 if (this._verifier === undefined || this._verifier instanceof RustQrCodeVerifier) {
                     this.setVerifier(new RustSASVerifier(verification, this, outgoingRequestProcessor));
+                } else if (this._verifier instanceof RustSASVerifier) {
+                    this._verifier.replaceInner(verification);
                 }
             } else if (verification instanceof RustSdkCryptoJs.Qr && this._verifier === undefined) {
                 this.setVerifier(new RustQrCodeVerifier(verification, outgoingRequestProcessor));
@@ -425,7 +432,7 @@ export class RustVerificationRequest
      * this verification.
      */
     public get cancellationCode(): string | null {
-        throw new Error("not implemented");
+        return this.inner.cancelInfo?.cancelCode() ?? null;
     }
 
     /**
@@ -434,7 +441,14 @@ export class RustVerificationRequest
      * Only defined when phase is Cancelled
      */
     public get cancellingUserId(): string | undefined {
-        throw new Error("not implemented");
+        const cancelInfo = this.inner.cancelInfo;
+        if (!cancelInfo) {
+            return undefined;
+        } else if (cancelInfo.cancelledbyUs()) {
+            return this.olmMachine.userId.toString();
+        } else {
+            return this.inner.otherUserId.toString();
+        }
     }
 }
 
@@ -448,46 +462,45 @@ abstract class BaseRustVerifer<InnerType extends RustSdkCryptoJs.Qr | RustSdkCry
     VerifierEvent | VerificationRequestEvent,
     VerifierEventHandlerMap & VerificationRequestEventHandlerMap
 > {
-    /** A promise which completes when the verification completes (or rejects when it is cancelled/fails) */
-    protected readonly completionPromise: Promise<void>;
+    /** A deferred which completes when the verification completes (or rejects when it is cancelled/fails) */
+    protected readonly completionDeferred: IDeferred<void>;
 
     public constructor(
-        protected readonly inner: InnerType,
+        protected inner: InnerType,
         protected readonly outgoingRequestProcessor: OutgoingRequestProcessor,
     ) {
         super();
 
-        this.completionPromise = new Promise<void>((resolve, reject) => {
-            const onChange = async (): Promise<void> => {
-                this.onChange();
-
-                if (this.inner.isDone()) {
-                    resolve(undefined);
-                } else if (this.inner.isCancelled()) {
-                    const cancelInfo = this.inner.cancelInfo()!;
-                    reject(
-                        new Error(
-                            `Verification cancelled by ${
-                                cancelInfo.cancelledbyUs() ? "us" : "them"
-                            } with code ${cancelInfo.cancelCode()}: ${cancelInfo.reason()}`,
-                        ),
-                    );
-                }
-
-                this.emit(VerificationRequestEvent.Change);
-            };
-            inner.registerChangesCallback(onChange);
+        this.completionDeferred = defer();
+        inner.registerChangesCallback(async () => {
+            this.onChange();
         });
         // stop the runtime complaining if nobody catches a failure
-        this.completionPromise.catch(() => null);
+        this.completionDeferred.promise.catch(() => null);
     }
 
     /**
      * Hook which is called when the underlying rust class notifies us that there has been a change.
      *
-     * Can be overridden by subclasses to see if we can notify the application about an update.
+     * Can be overridden by subclasses to see if we can notify the application about an update. The overriding method
+     * must call `super.onChange()`.
      */
-    protected onChange(): void {}
+    protected onChange(): void {
+        if (this.inner.isDone()) {
+            this.completionDeferred.resolve(undefined);
+        } else if (this.inner.isCancelled()) {
+            const cancelInfo = this.inner.cancelInfo()!;
+            this.completionDeferred.reject(
+                new Error(
+                    `Verification cancelled by ${
+                        cancelInfo.cancelledbyUs() ? "us" : "them"
+                    } with code ${cancelInfo.cancelCode()}: ${cancelInfo.reason()}`,
+                ),
+            );
+        }
+
+        this.emit(VerificationRequestEvent.Change);
+    }
 
     /**
      * Returns true if the verification has been cancelled, either by us or the other side.
@@ -557,6 +570,8 @@ export class RustQrCodeVerifier extends BaseRustVerifer<RustSdkCryptoJs.Qr> impl
                 cancel: () => this.cancel(),
             };
         }
+
+        super.onChange();
     }
 
     /**
@@ -572,7 +587,7 @@ export class RustQrCodeVerifier extends BaseRustVerifer<RustSdkCryptoJs.Qr> impl
             this.emit(VerifierEvent.ShowReciprocateQr, this.callbacks);
         }
         // Nothing to do here but wait.
-        await this.completionPromise;
+        await this.completionDeferred.promise;
     }
 
     /**
@@ -649,28 +664,42 @@ export class RustSASVerifier extends BaseRustVerifer<RustSdkCryptoJs.Sas> implem
      *    or times out.
      */
     public async verify(): Promise<void> {
+        await this.sendAccept();
+        await this.completionDeferred.promise;
+    }
+
+    /**
+     * Send the accept or start event, if it hasn't already been sent
+     */
+    private async sendAccept(): Promise<void> {
         const req: undefined | OutgoingRequest = this.inner.accept();
         if (req) {
             await this.outgoingRequestProcessor.makeOutgoingRequest(req);
         }
-        await this.completionPromise;
     }
 
     /** if we can now show the callbacks, do so */
     protected onChange(): void {
+        super.onChange();
+
         if (this.callbacks === null) {
-            const emoji: Array<Emoji> | undefined = this.inner.emoji();
-            const decimal = this.inner.decimals() as [number, number, number] | undefined;
+            const emoji = this.inner.emoji();
+            const decimal = this.inner.decimals();
 
             if (emoji === undefined && decimal === undefined) {
                 return;
             }
 
+            const sas: GeneratedSas = {};
+            if (emoji) {
+                sas.emoji = emoji.map((e) => [e.symbol, e.description]);
+            }
+            if (decimal) {
+                sas.decimal = [decimal[0], decimal[1], decimal[2]];
+            }
+
             this.callbacks = {
-                sas: {
-                    decimal: decimal,
-                    emoji: emoji?.map((e) => [e.symbol, e.description]),
-                },
+                sas,
                 confirm: async (): Promise<void> => {
                     const requests: Array<OutgoingRequest> = await this.inner.confirm();
                     for (const m of requests) {
@@ -704,6 +733,25 @@ export class RustSASVerifier extends BaseRustVerifer<RustSdkCryptoJs.Sas> implem
     public getShowSasCallbacks(): ShowSasCallbacks | null {
         return this.callbacks;
     }
+
+    /**
+     * Replace the inner Rust verifier with a different one.
+     *
+     * @param inner - the new Rust verifier
+     * @internal
+     */
+    public replaceInner(inner: RustSdkCryptoJs.Sas): void {
+        if (this.inner != inner) {
+            this.inner = inner;
+            inner.registerChangesCallback(async () => {
+                this.onChange();
+            });
+            // replaceInner will only get called if we started the verification at the same time as the other side, and we lost
+            // the tie breaker.  So we need to re-accept their verification.
+            this.sendAccept();
+            this.onChange();
+        }
+    }
 }
 
 /** For each specced verification method, the rust-side `VerificationMethod` corresponding to it */

src/sliding-sync-sdk.ts

@@ -615,7 +615,7 @@ export class SlidingSyncSdk {
             }
         }
 
-        const encrypted = this.client.isRoomEncrypted(room.roomId);
+        const encrypted = room.hasEncryptionStateEvent();
         // we do this first so it's correct when any of the events fire
         if (roomData.notification_count != null) {
             room.setUnreadNotificationCount(NotificationCountType.Total, roomData.notification_count);

src/store/indexeddb.ts

@@ -90,10 +90,10 @@ export class IndexedDBStore extends MemoryStore {
      * ```
      * let opts = { indexedDB: window.indexedDB, localStorage: window.localStorage };
      * let store = new IndexedDBStore(opts);
-     * await store.startup(); // load from indexed db
      * let client = sdk.createClient({
      *     store: store,
      * });
+     * await store.startup(); // load from indexed db, must be called after createClient
      * client.startClient();
      * client.on("sync", function(state, prevState, data) {
      *     if (state === "PREPARED") {
@@ -140,7 +140,9 @@ export class IndexedDBStore extends MemoryStore {
                 logger.log(`IndexedDBStore.startup: processing presence events`);
                 userPresenceEvents.forEach(([userId, rawEvent]) => {
                     if (!this.createUser) {
-                        throw new Error("createUser is undefined, it should be set with setUserCreator()!");
+                        throw new Error(
+                            "`IndexedDBStore.startup` must be called after assigning it to the client, not before!",
+                        );
                     }
                     const u = this.createUser(userId);
                     if (rawEvent) {

src/sync.ts

@@ -1760,11 +1760,11 @@ export class SyncApi {
         return events?.find((e) => e.getType() === EventType.RoomEncryption && e.getStateKey() === "");
     }
 
-    // When processing the sync response we cannot rely on MatrixClient::isRoomEncrypted before we actually
+    // When processing the sync response we cannot rely on Room.hasEncryptionStateEvent we actually
     // inject the events into the room object, so we have to inspect the events themselves.
     private isRoomEncrypted(room: Room, stateEventList: MatrixEvent[], timelineEventList?: MatrixEvent[]): boolean {
         return (
-            this.client.isRoomEncrypted(room.roomId) ||
+            room.hasEncryptionStateEvent() ||
             !!this.findEncryptionEvent(stateEventList) ||
             !!this.findEncryptionEvent(timelineEventList)
         );

src/version-support.ts

@@ -15,9 +15,22 @@ limitations under the License.
 */
 
 /**
- * The minimum Matrix specification version the js-sdk supports.
+ * A list of the spec versions which the js-sdk is compatible with.
  *
- * (This means that we require any servers we connect to to declare support for this spec version, so it is important
- * for it not to be too old, as well as not too new.)
+ * In practice, this means: when we connect to a server, it must declare support for one of the versions in this list.
+ *
+ * Note that it does not *necessarily* mean that the js-sdk has good support for all the features in the listed spec
+ * versions; only that we should be able to provide a base level of functionality with a server that offers support for
+ * any of the listed versions.
  */
-export const MINIMUM_MATRIX_VERSION = "v1.5";
+export const SUPPORTED_MATRIX_VERSIONS = ["v1.1", "v1.2", "v1.3", "v1.4", "v1.5", "v1.6", "v1.7", "v1.8", "v1.9"];
+
+/**
+ * The oldest Matrix specification version the js-sdk supports.
+ */
+export const MINIMUM_MATRIX_VERSION = SUPPORTED_MATRIX_VERSIONS[0];
+
+/**
+ * The most recent Matrix specification version the js-sdk supports.
+ */
+export const MAXIMUM_MATRIX_VERSION = SUPPORTED_MATRIX_VERSIONS[SUPPORTED_MATRIX_VERSIONS.length - 1];

src/webrtc/call.ts

@@ -1309,8 +1309,9 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
 
                 const track = stream.getTracks().find((track) => track.kind === "video");
 
-                const sender = this.transceivers.get(getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "video"))
-                    ?.sender;
+                const sender = this.transceivers.get(
+                    getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "video"),
+                )?.sender;
 
                 sender?.replaceTrack(track ?? null);
 
@@ -1326,8 +1327,9 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
             }
         } else {
             const track = this.localUsermediaStream?.getTracks().find((track) => track.kind === "video");
-            const sender = this.transceivers.get(getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "video"))
-                ?.sender;
+            const sender = this.transceivers.get(
+                getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "video"),
+            )?.sender;
             sender?.replaceTrack(track ?? null);
 
             this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream!);

tsconfig.json

@@ -8,12 +8,12 @@
         "noUnusedLocals": true,
         "noEmit": true,
         "declaration": true,
-        "strict": true
+        "strict": true,
     },
     "include": ["./src/**/*.ts", "./spec/**/*.ts"],
     "typedocOptions": {
         "entryPoints": ["src/matrix.ts"],
         "excludeExternals": true,
-        "out": "_docs"
-    }
+        "out": "_docs",
+    },
 }

yarn.lock

@@ -95,19 +95,19 @@
   integrity sha512-uU27kfDRlhfKl+w1U6vp16IuvSLtjAxdArVXPa9BvLkrr7CYIsxH5adpHObeAGY/41+syctUWOZ140a2Rvkgjw==
 
 "@babel/core@^7.0.0", "@babel/core@^7.12.10":
-  version "7.23.6"
-  resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.6.tgz#8be77cd77c55baadcc1eae1c33df90ab6d2151d4"
-  integrity sha512-FxpRyGjrMJXh7X3wGLGhNDCRiwpWEF74sKjTLDJSG5Kyvow3QZaG0Adbqzi9ZrVjTWpsX+2cxWXD71NMg93kdw==
+  version "7.23.7"
+  resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.7.tgz#4d8016e06a14b5f92530a13ed0561730b5c6483f"
+  integrity sha512-+UpDgowcmqe36d4NwqvKsyPMlOLNGMsfMmQ5WGCu+siCe3t3dfe9njrzGfdN4qq+bcNUt0+Vw6haRxBOycs4dw==
   dependencies:
     "@ampproject/remapping" "^2.2.0"
     "@babel/code-frame" "^7.23.5"
     "@babel/generator" "^7.23.6"
     "@babel/helper-compilation-targets" "^7.23.6"
     "@babel/helper-module-transforms" "^7.23.3"
-    "@babel/helpers" "^7.23.6"
+    "@babel/helpers" "^7.23.7"
     "@babel/parser" "^7.23.6"
     "@babel/template" "^7.22.15"
-    "@babel/traverse" "^7.23.6"
+    "@babel/traverse" "^7.23.7"
     "@babel/types" "^7.23.6"
     convert-source-map "^2.0.0"
     debug "^4.1.0"
@@ -234,9 +234,9 @@
     semver "^6.3.1"
 
 "@babel/helper-create-class-features-plugin@^7.22.15":
-  version "7.23.6"
-  resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.23.6.tgz#b04d915ce92ce363666f816a884cdcfc9be04953"
-  integrity sha512-cBXU1vZni/CpGF29iTu4YRbOZt3Wat6zCoMDxRF1MayiEc4URxOj31tT65HUM0CRpMowA3HCJaAOVOUnMf96cw==
+  version "7.23.7"
+  resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.23.7.tgz#b2e6826e0e20d337143655198b79d58fdc9bd43d"
+  integrity sha512-xCoqR/8+BoNnXOY7RVSgv6X+o7pmT5q1d+gGcRlXYkI+9B31glE4jeejhKVpA04O1AtzOt7OSQ6VYKP5FcRl9g==
   dependencies:
     "@babel/helper-annotate-as-pure" "^7.22.5"
     "@babel/helper-environment-visitor" "^7.22.20"
@@ -420,13 +420,13 @@
     "@babel/traverse" "^7.22.15"
     "@babel/types" "^7.22.15"
 
-"@babel/helpers@^7.23.6":
-  version "7.23.6"
-  resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.23.6.tgz#d03af2ee5fb34691eec0cda90f5ecbb4d4da145a"
-  integrity sha512-wCfsbN4nBidDRhpDhvcKlzHWCTlgJYUUdSJfzXb2NuBssDSIjc3xcb+znA7l+zYsFljAcGM0aFkN40cR3lXiGA==
+"@babel/helpers@^7.23.7":
+  version "7.23.8"
+  resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.23.8.tgz#fc6b2d65b16847fd50adddbd4232c76378959e34"
+  integrity sha512-KDqYz4PiOWvDFrdHLPhKtCThtIcKVy6avWD2oG4GEvyQ+XDZwHD4YQd+H2vNMnq2rkdxsDkU82T+Vk8U/WXHRQ==
   dependencies:
     "@babel/template" "^7.22.15"
-    "@babel/traverse" "^7.23.6"
+    "@babel/traverse" "^7.23.7"
     "@babel/types" "^7.23.6"
 
 "@babel/highlight@^7.22.13", "@babel/highlight@^7.23.4":
@@ -469,10 +469,10 @@
     "@babel/helper-skip-transparent-expression-wrappers" "^7.22.5"
     "@babel/plugin-transform-optional-chaining" "^7.23.3"
 
-"@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly@^7.23.3":
-  version "7.23.3"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly/-/plugin-bugfix-v8-static-class-fields-redefine-readonly-7.23.3.tgz#20c60d4639d18f7da8602548512e9d3a4c8d7098"
-  integrity sha512-XaJak1qcityzrX0/IU5nKHb34VaibwP3saKqG6a/tppelgllOH13LUann4ZCIBcVOeE6H18K4Vx9QKkVww3z/w==
+"@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly@^7.23.7":
+  version "7.23.7"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly/-/plugin-bugfix-v8-static-class-fields-redefine-readonly-7.23.7.tgz#516462a95d10a9618f197d39ad291a9b47ae1d7b"
+  integrity sha512-LlRT7HgaifEpQA1ZgLVOIJZZFVPWN5iReq/7/JixwBtwcoeVGDBD53ZV28rrsLYOZs1Y/EHhA8N/Z6aazHR8cw==
   dependencies:
     "@babel/helper-environment-visitor" "^7.22.20"
     "@babel/helper-plugin-utils" "^7.22.5"
@@ -678,10 +678,10 @@
   dependencies:
     "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-async-generator-functions@^7.23.4":
-  version "7.23.4"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-async-generator-functions/-/plugin-transform-async-generator-functions-7.23.4.tgz#93ac8e3531f347fba519b4703f9ff2a75c6ae27a"
-  integrity sha512-efdkfPhHYTtn0G6n2ddrESE91fgXxjlqLsnUtPWnJs4a4mZIbUaK7ffqKIIUKXSHwcDvaCVX6GXkaJJFqtX7jw==
+"@babel/plugin-transform-async-generator-functions@^7.23.7":
+  version "7.23.7"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-async-generator-functions/-/plugin-transform-async-generator-functions-7.23.7.tgz#3aa0b4f2fa3788b5226ef9346cf6d16ec61f99cd"
+  integrity sha512-PdxEpL71bJp1byMG0va5gwQcXHxuEYC/BgI/e88mGTtohbZN28O5Yit0Plkkm/dBzCF/BxmbNcses1RH1T+urA==
   dependencies:
     "@babel/helper-environment-visitor" "^7.22.20"
     "@babel/helper-plugin-utils" "^7.22.5"
@@ -728,16 +728,15 @@
     "@babel/helper-plugin-utils" "^7.22.5"
     "@babel/plugin-syntax-class-static-block" "^7.14.5"
 
-"@babel/plugin-transform-classes@^7.23.5":
-  version "7.23.5"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.23.5.tgz#e7a75f815e0c534cc4c9a39c56636c84fc0d64f2"
-  integrity sha512-jvOTR4nicqYC9yzOHIhXG5emiFEOpappSJAl73SDSEDcybD+Puuze8Tnpb9p9qEyYup24tq891gkaygIFvWDqg==
+"@babel/plugin-transform-classes@^7.23.8":
+  version "7.23.8"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.23.8.tgz#d08ae096c240347badd68cdf1b6d1624a6435d92"
+  integrity sha512-yAYslGsY1bX6Knmg46RjiCiNSwJKv2IUC8qOdYKqMMr0491SXFhcHqOdRDeCRohOOIzwN/90C6mQ9qAKgrP7dg==
   dependencies:
     "@babel/helper-annotate-as-pure" "^7.22.5"
-    "@babel/helper-compilation-targets" "^7.22.15"
+    "@babel/helper-compilation-targets" "^7.23.6"
     "@babel/helper-environment-visitor" "^7.22.20"
     "@babel/helper-function-name" "^7.23.0"
-    "@babel/helper-optimise-call-expression" "^7.22.5"
     "@babel/helper-plugin-utils" "^7.22.5"
     "@babel/helper-replace-supers" "^7.22.20"
     "@babel/helper-split-export-declaration" "^7.22.6"
@@ -1001,15 +1000,15 @@
     "@babel/helper-plugin-utils" "^7.22.5"
 
 "@babel/plugin-transform-runtime@^7.12.10":
-  version "7.23.4"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.23.4.tgz#5132b388580002fc5cb7c84eccfb968acdc231cb"
-  integrity sha512-ITwqpb6V4btwUG0YJR82o2QvmWrLgDnx/p2A3CTPYGaRgULkDiC0DRA2C4jlRB9uXGUEfaSS/IGHfVW+ohzYDw==
+  version "7.23.7"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.23.7.tgz#52bbd20054855beb9deae3bee9ceb05289c343e6"
+  integrity sha512-fa0hnfmiXc9fq/weK34MUV0drz2pOL/vfKWvN7Qw127hiUPabFCUMgAbYWcchRzMJit4o5ARsK/s+5h0249pLw==
   dependencies:
     "@babel/helper-module-imports" "^7.22.15"
     "@babel/helper-plugin-utils" "^7.22.5"
-    babel-plugin-polyfill-corejs2 "^0.4.6"
-    babel-plugin-polyfill-corejs3 "^0.8.5"
-    babel-plugin-polyfill-regenerator "^0.5.3"
+    babel-plugin-polyfill-corejs2 "^0.4.7"
+    babel-plugin-polyfill-corejs3 "^0.8.7"
+    babel-plugin-polyfill-regenerator "^0.5.4"
     semver "^6.3.1"
 
 "@babel/plugin-transform-shorthand-properties@^7.23.3":
@@ -1090,9 +1089,9 @@
     "@babel/helper-plugin-utils" "^7.22.5"
 
 "@babel/preset-env@^7.12.11":
-  version "7.23.6"
-  resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.23.6.tgz#ad0ea799d5a3c07db5b9a172819bbd444092187a"
-  integrity sha512-2XPn/BqKkZCpzYhUUNZ1ssXw7DcXfKQEjv/uXZUXgaebCMYmkEsfZ2yY+vv+xtXv50WmL5SGhyB6/xsWxIvvOQ==
+  version "7.23.8"
+  resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.23.8.tgz#7d6f8171ea7c221ecd28059e65ad37c20e441e3e"
+  integrity sha512-lFlpmkApLkEP6woIKprO6DO60RImpatTQKtz4sUcDjVcK8M8mQ4sZsuxaTMNOZf0sqAq/ReYW1ZBHnOQwKpLWA==
   dependencies:
     "@babel/compat-data" "^7.23.5"
     "@babel/helper-compilation-targets" "^7.23.6"
@@ -1100,7 +1099,7 @@
     "@babel/helper-validator-option" "^7.23.5"
     "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression" "^7.23.3"
     "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining" "^7.23.3"
-    "@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly" "^7.23.3"
+    "@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly" "^7.23.7"
     "@babel/plugin-proposal-private-property-in-object" "7.21.0-placeholder-for-preset-env.2"
     "@babel/plugin-syntax-async-generators" "^7.8.4"
     "@babel/plugin-syntax-class-properties" "^7.12.13"
@@ -1121,13 +1120,13 @@
     "@babel/plugin-syntax-top-level-await" "^7.14.5"
     "@babel/plugin-syntax-unicode-sets-regex" "^7.18.6"
     "@babel/plugin-transform-arrow-functions" "^7.23.3"
-    "@babel/plugin-transform-async-generator-functions" "^7.23.4"
+    "@babel/plugin-transform-async-generator-functions" "^7.23.7"
     "@babel/plugin-transform-async-to-generator" "^7.23.3"
     "@babel/plugin-transform-block-scoped-functions" "^7.23.3"
     "@babel/plugin-transform-block-scoping" "^7.23.4"
     "@babel/plugin-transform-class-properties" "^7.23.3"
     "@babel/plugin-transform-class-static-block" "^7.23.4"
-    "@babel/plugin-transform-classes" "^7.23.5"
+    "@babel/plugin-transform-classes" "^7.23.8"
     "@babel/plugin-transform-computed-properties" "^7.23.3"
     "@babel/plugin-transform-destructuring" "^7.23.3"
     "@babel/plugin-transform-dotall-regex" "^7.23.3"
@@ -1169,9 +1168,9 @@
     "@babel/plugin-transform-unicode-regex" "^7.23.3"
     "@babel/plugin-transform-unicode-sets-regex" "^7.23.3"
     "@babel/preset-modules" "0.1.6-no-external-plugins"
-    babel-plugin-polyfill-corejs2 "^0.4.6"
-    babel-plugin-polyfill-corejs3 "^0.8.5"
-    babel-plugin-polyfill-regenerator "^0.5.3"
+    babel-plugin-polyfill-corejs2 "^0.4.7"
+    babel-plugin-polyfill-corejs3 "^0.8.7"
+    babel-plugin-polyfill-regenerator "^0.5.4"
     core-js-compat "^3.31.0"
     semver "^6.3.1"
 
@@ -1196,14 +1195,14 @@
     "@babel/plugin-transform-typescript" "^7.23.3"
 
 "@babel/register@^7.12.10":
-  version "7.22.15"
-  resolved "https://registry.yarnpkg.com/@babel/register/-/register-7.22.15.tgz#c2c294a361d59f5fa7bcc8b97ef7319c32ecaec7"
-  integrity sha512-V3Q3EqoQdn65RCgTLwauZaTfd1ShhwPmbBv+1dkZV/HpCGMKVyn6oFcRlI7RaKqiDQjX2Qd3AuoEguBgdjIKlg==
+  version "7.23.7"
+  resolved "https://registry.yarnpkg.com/@babel/register/-/register-7.23.7.tgz#485a5e7951939d21304cae4af1719fdb887bc038"
+  integrity sha512-EjJeB6+kvpk+Y5DAkEAmbOBEFkh9OASx0huoEkqYTFxAZHzOAX2Oh5uwAUuL2rUddqfM0SA+KPXV2TbzoZ2kvQ==
   dependencies:
     clone-deep "^4.0.1"
     find-cache-dir "^2.0.0"
     make-dir "^2.1.0"
-    pirates "^4.0.5"
+    pirates "^4.0.6"
     source-map-support "^0.5.16"
 
 "@babel/regjsgen@^0.8.0":
@@ -1212,9 +1211,9 @@
   integrity sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA==
 
 "@babel/runtime@^7.0.0", "@babel/runtime@^7.12.5", "@babel/runtime@^7.8.4":
-  version "7.23.6"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.6.tgz#c05e610dc228855dc92ef1b53d07389ed8ab521d"
-  integrity sha512-zHd0eUrf5GZoOWVCXp6koAKQTfZV07eit6bGPmJgnZdnSAvvZee6zniW2XMF7Cmc4ISOOnPy3QaSiIJGJkVEDQ==
+  version "7.23.8"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.8.tgz#8ee6fe1ac47add7122902f257b8ddf55c898f650"
+  integrity sha512-Y7KbAP984rn1VGMbGqKmBLio9V7y5Je9GvU4rQPCPinCyNfUcToxIXl06d59URp/F3LwinvODxab5N/G6qggkw==
   dependencies:
     regenerator-runtime "^0.14.0"
 
@@ -1243,10 +1242,10 @@
     debug "^4.1.0"
     globals "^11.1.0"
 
-"@babel/traverse@^7.23.6":
-  version "7.23.6"
-  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.6.tgz#b53526a2367a0dd6edc423637f3d2d0f2521abc5"
-  integrity sha512-czastdK1e8YByZqezMPFiZ8ahwVMh/ESl9vPgvgdB9AmFMGP5jfpFax74AQgl5zj4XHzqeYAg2l8PuUeRS1MgQ==
+"@babel/traverse@^7.23.7":
+  version "7.23.7"
+  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.7.tgz#9a7bf285c928cb99b5ead19c3b1ce5b310c9c305"
+  integrity sha512-tY3mM8rH9jM0YHFGyfC0/xf+SB5eKUu7HPj7/k3fpi9dAlsMc5YbQvDi0Sh2QTPXqMhyaAtzAr807TIyfQrmyg==
   dependencies:
     "@babel/code-frame" "^7.23.5"
     "@babel/generator" "^7.23.6"
@@ -1357,10 +1356,10 @@
     minimatch "^3.1.2"
     strip-json-comments "^3.1.1"
 
-"@eslint/js@8.55.0":
-  version "8.55.0"
-  resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.55.0.tgz#b721d52060f369aa259cf97392403cb9ce892ec6"
-  integrity sha512-qQfo2mxH5yVom1kacMtZZJFVdW+E70mqHMJvVg6WTLo+VBuQJ4TojZlfWBjK0ve5BdEeNAVxOsl/nvNMpJOaJA==
+"@eslint/js@8.56.0":
+  version "8.56.0"
+  resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.56.0.tgz#ef20350fec605a7f7035a01764731b2de0f3782b"
+  integrity sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A==
 
 "@humanwhocodes/config-array@^0.11.13":
   version "0.11.13"
@@ -1675,10 +1674,10 @@
     "@jridgewell/resolve-uri" "^3.1.0"
     "@jridgewell/sourcemap-codec" "^1.4.14"
 
-"@matrix-org/matrix-sdk-crypto-wasm@^3.5.0":
-  version "3.5.0"
-  resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-wasm/-/matrix-sdk-crypto-wasm-3.5.0.tgz#997d63ae12304142513fe93c5e0872ff10ca30b4"
-  integrity sha512-7as0jJTje+rFu9AF8LEO0tmhtHcou2YQnZOtpiP+lS5rDfIPv5CL8/eb45fzDnbQybt9Jm5zdjBdiLBEaUg2dQ==
+"@matrix-org/matrix-sdk-crypto-wasm@^4.3.0":
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-wasm/-/matrix-sdk-crypto-wasm-4.3.0.tgz#ef08e7eafae6e9e85658c14a41f0d74a48c03f4a"
+  integrity sha512-05+NO78pXda/MTxi05NJwBbbAsOmU6WywBOcAk9GarPzgvrj4GvOuYTY6VR5PD7Gzb3AI+vNx/Ho4V0GFKPW/w==
 
 "@matrix-org/olm@3.2.15":
   version "3.2.15"
@@ -2001,16 +2000,11 @@
     "@types/tough-cookie" "*"
     parse5 "^7.0.0"
 
-"@types/json-schema@^7.0.12":
+"@types/json-schema@^7.0.12", "@types/json-schema@^7.0.9":
   version "7.0.15"
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841"
   integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==
 
-"@types/json-schema@^7.0.9":
-  version "7.0.14"
-  resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.14.tgz#74a97a5573980802f32c8e47b663530ab3b6b7d1"
-  integrity sha512-U3PUjAudAdJBeC2pgN8uTIKgxrb4nlDF3SF0++EldXQvQBGkpFZMSnwQiIoDU77tv45VgNkl/L4ouD+rEomujw==
-
 "@types/json5@^0.0.29":
   version "0.0.29"
   resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
@@ -2029,9 +2023,9 @@
     undici-types "~5.26.4"
 
 "@types/node@18":
-  version "18.19.3"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.3.tgz#e4723c4cb385641d61b983f6fe0b716abd5f8fc0"
-  integrity sha512-k5fggr14DwAytoA/t8rPrIz++lXK7/DqckthCmoZOKNsEbJkId4Z//BqgApXBUGrGddrigYa1oqheo/7YmW4rg==
+  version "18.19.8"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.8.tgz#c1e42b165e5a526caf1f010747e0522cb2c9c36a"
+  integrity sha512-g1pZtPhsvGVTwmeVoexWZLTQaOvXwoSq//pTL0DHeNzUDrFnir4fgETdhjhIxjVnN+hKOuh98+E1eMLnUXstFg==
   dependencies:
     undici-types "~5.26.4"
 
@@ -2050,12 +2044,7 @@
   resolved "https://registry.yarnpkg.com/@types/sdp-transform/-/sdp-transform-2.4.9.tgz#26ef39f487a6909b0512f580b80920a366b27f52"
   integrity sha512-bVr+/OoZZy7wrHlNcEAAa6PAgKA4BoXPYVN2EijMC5WnGgQ4ZEuixmKnVs2roiAvr7RhIFVH17QD27cojgIZCg==
 
-"@types/semver@^7.3.12":
-  version "7.5.5"
-  resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.5.tgz#deed5ab7019756c9c90ea86139106b0346223f35"
-  integrity sha512-+d+WYC1BxJ6yVOgUgzK8gWvp5qF8ssV5r4nsDcZWKRWcDQLQ619tvWAxJQYGgBrO1MnLJC7a5GtiYsAoQ47dJg==
-
-"@types/semver@^7.5.0":
+"@types/semver@^7.3.12", "@types/semver@^7.5.0":
   version "7.5.6"
   resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.6.tgz#c65b2bfce1bec346582c07724e3f8c1017a20339"
   integrity sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==
@@ -2093,15 +2082,15 @@
     "@types/yargs-parser" "*"
 
 "@typescript-eslint/eslint-plugin@^6.0.0":
-  version "6.17.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.17.0.tgz#dfc38f790704ba8a54a1277c51efdb489f6ecf9f"
-  integrity sha512-Vih/4xLXmY7V490dGwBQJTpIZxH4ZFH6eCVmQ4RFkB+wmaCTDAx4dtgoWwMNGKLkqRY1L6rPqzEbjorRnDo4rQ==
+  version "6.19.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.19.1.tgz#bb0676af940bc23bf299ca58dbdc6589c2548c2e"
+  integrity sha512-roQScUGFruWod9CEyoV5KlCYrubC/fvG8/1zXuT0WTcxX87GnMMmnksMwSg99lo1xiKrBzw2icsJPMAw1OtKxg==
   dependencies:
     "@eslint-community/regexpp" "^4.5.1"
-    "@typescript-eslint/scope-manager" "6.17.0"
-    "@typescript-eslint/type-utils" "6.17.0"
-    "@typescript-eslint/utils" "6.17.0"
-    "@typescript-eslint/visitor-keys" "6.17.0"
+    "@typescript-eslint/scope-manager" "6.19.1"
+    "@typescript-eslint/type-utils" "6.19.1"
+    "@typescript-eslint/utils" "6.19.1"
+    "@typescript-eslint/visitor-keys" "6.19.1"
     debug "^4.3.4"
     graphemer "^1.4.0"
     ignore "^5.2.4"
@@ -2110,14 +2099,14 @@
     ts-api-utils "^1.0.1"
 
 "@typescript-eslint/parser@^6.0.0":
-  version "6.17.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.17.0.tgz#8cd7a0599888ca6056082225b2fdf9a635bf32a1"
-  integrity sha512-C4bBaX2orvhK+LlwrY8oWGmSl4WolCfYm513gEccdWZj0CwGadbIADb0FtVEcI+WzUyjyoBj2JRP8g25E6IB8A==
+  version "6.19.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.19.1.tgz#68a87bb21afaf0b1689e9cdce0e6e75bc91ada78"
+  integrity sha512-WEfX22ziAh6pRE9jnbkkLGp/4RhTpffr2ZK5bJ18M8mIfA8A+k97U9ZyaXCEJRlmMHh7R9MJZWXp/r73DzINVQ==
   dependencies:
-    "@typescript-eslint/scope-manager" "6.17.0"
-    "@typescript-eslint/types" "6.17.0"
-    "@typescript-eslint/typescript-estree" "6.17.0"
-    "@typescript-eslint/visitor-keys" "6.17.0"
+    "@typescript-eslint/scope-manager" "6.19.1"
+    "@typescript-eslint/types" "6.19.1"
+    "@typescript-eslint/typescript-estree" "6.19.1"
+    "@typescript-eslint/visitor-keys" "6.19.1"
     debug "^4.3.4"
 
 "@typescript-eslint/scope-manager@5.62.0":
@@ -2128,21 +2117,21 @@
     "@typescript-eslint/types" "5.62.0"
     "@typescript-eslint/visitor-keys" "5.62.0"
 
-"@typescript-eslint/scope-manager@6.17.0":
-  version "6.17.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.17.0.tgz#70e6c1334d0d76562dfa61aed9009c140a7601b4"
-  integrity sha512-RX7a8lwgOi7am0k17NUO0+ZmMOX4PpjLtLRgLmT1d3lBYdWH4ssBUbwdmc5pdRX8rXon8v9x8vaoOSpkHfcXGA==
+"@typescript-eslint/scope-manager@6.19.1":
+  version "6.19.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.19.1.tgz#2f527ee30703a6169a52b31d42a1103d80acd51b"
+  integrity sha512-4CdXYjKf6/6aKNMSly/BP4iCSOpvMmqtDzRtqFyyAae3z5kkqEjKndR5vDHL8rSuMIIWP8u4Mw4VxLyxZW6D5w==
   dependencies:
-    "@typescript-eslint/types" "6.17.0"
-    "@typescript-eslint/visitor-keys" "6.17.0"
+    "@typescript-eslint/types" "6.19.1"
+    "@typescript-eslint/visitor-keys" "6.19.1"
 
-"@typescript-eslint/type-utils@6.17.0":
-  version "6.17.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.17.0.tgz#5febad3f523e393006614cbda28b826925b728d5"
-  integrity sha512-hDXcWmnbtn4P2B37ka3nil3yi3VCQO2QEB9gBiHJmQp5wmyQWqnjA85+ZcE8c4FqnaB6lBwMrPkgd4aBYz3iNg==
+"@typescript-eslint/type-utils@6.19.1":
+  version "6.19.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.19.1.tgz#6a130e3afe605a4898e043fa9f72e96309b54935"
+  integrity sha512-0vdyld3ecfxJuddDjACUvlAeYNrHP/pDeQk2pWBR2ESeEzQhg52DF53AbI9QCBkYE23lgkhLCZNkHn2hEXXYIg==
   dependencies:
-    "@typescript-eslint/typescript-estree" "6.17.0"
-    "@typescript-eslint/utils" "6.17.0"
+    "@typescript-eslint/typescript-estree" "6.19.1"
+    "@typescript-eslint/utils" "6.19.1"
     debug "^4.3.4"
     ts-api-utils "^1.0.1"
 
@@ -2151,10 +2140,10 @@
   resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.62.0.tgz#258607e60effa309f067608931c3df6fed41fd2f"
   integrity sha512-87NVngcbVXUahrRTqIK27gD2t5Cu1yuCXxbLcFtCzZGlfyVWWh8mLHkoxzjsB6DDNnvdL+fW8MiwPEJyGJQDgQ==
 
-"@typescript-eslint/types@6.17.0":
-  version "6.17.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.17.0.tgz#844a92eb7c527110bf9a7d177e3f22bd5a2f40cb"
-  integrity sha512-qRKs9tvc3a4RBcL/9PXtKSehI/q8wuU9xYJxe97WFxnzH8NWWtcW3ffNS+EWg8uPvIerhjsEZ+rHtDqOCiH57A==
+"@typescript-eslint/types@6.19.1":
+  version "6.19.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.19.1.tgz#2d4c9d492a63ede15e7ba7d129bdf7714b77f771"
+  integrity sha512-6+bk6FEtBhvfYvpHsDgAL3uo4BfvnTnoge5LrrCj2eJN8g3IJdLTD4B/jK3Q6vo4Ql/Hoip9I8aB6fF+6RfDqg==
 
 "@typescript-eslint/typescript-estree@5.62.0":
   version "5.62.0"
@@ -2169,13 +2158,13 @@
     semver "^7.3.7"
     tsutils "^3.21.0"
 
-"@typescript-eslint/typescript-estree@6.17.0":
-  version "6.17.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.17.0.tgz#b913d19886c52d8dc3db856903a36c6c64fd62aa"
-  integrity sha512-gVQe+SLdNPfjlJn5VNGhlOhrXz4cajwFd5kAgWtZ9dCZf4XJf8xmgCTLIqec7aha3JwgLI2CK6GY1043FRxZwg==
+"@typescript-eslint/typescript-estree@6.19.1":
+  version "6.19.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.19.1.tgz#796d88d88882f12e85bb33d6d82d39e1aea54ed1"
+  integrity sha512-aFdAxuhzBFRWhy+H20nYu19+Km+gFfwNO4TEqyszkMcgBDYQjmPJ61erHxuT2ESJXhlhrO7I5EFIlZ+qGR8oVA==
   dependencies:
-    "@typescript-eslint/types" "6.17.0"
-    "@typescript-eslint/visitor-keys" "6.17.0"
+    "@typescript-eslint/types" "6.19.1"
+    "@typescript-eslint/visitor-keys" "6.19.1"
     debug "^4.3.4"
     globby "^11.1.0"
     is-glob "^4.0.3"
@@ -2183,17 +2172,17 @@
     semver "^7.5.4"
     ts-api-utils "^1.0.1"
 
-"@typescript-eslint/utils@6.17.0":
-  version "6.17.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.17.0.tgz#f2b16d4c9984474656c420438cdede7eccd4079e"
-  integrity sha512-LofsSPjN/ITNkzV47hxas2JCsNCEnGhVvocfyOcLzT9c/tSZE7SfhS/iWtzP1lKNOEfLhRTZz6xqI8N2RzweSQ==
+"@typescript-eslint/utils@6.19.1":
+  version "6.19.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.19.1.tgz#df93497f9cfddde2bcc2a591da80536e68acd151"
+  integrity sha512-JvjfEZuP5WoMqwh9SPAPDSHSg9FBHHGhjPugSRxu5jMfjvBpq5/sGTD+9M9aQ5sh6iJ8AY/Kk/oUYVEMAPwi7w==
   dependencies:
     "@eslint-community/eslint-utils" "^4.4.0"
     "@types/json-schema" "^7.0.12"
     "@types/semver" "^7.5.0"
-    "@typescript-eslint/scope-manager" "6.17.0"
-    "@typescript-eslint/types" "6.17.0"
-    "@typescript-eslint/typescript-estree" "6.17.0"
+    "@typescript-eslint/scope-manager" "6.19.1"
+    "@typescript-eslint/types" "6.19.1"
+    "@typescript-eslint/typescript-estree" "6.19.1"
     semver "^7.5.4"
 
 "@typescript-eslint/utils@^5.10.0":
@@ -2218,12 +2207,12 @@
     "@typescript-eslint/types" "5.62.0"
     eslint-visitor-keys "^3.3.0"
 
-"@typescript-eslint/visitor-keys@6.17.0":
-  version "6.17.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.17.0.tgz#3ed043709c39b43ec1e58694f329e0b0430c26b6"
-  integrity sha512-H6VwB/k3IuIeQOyYczyyKN8wH6ed8EwliaYHLxOIhyF0dYEIsN8+Bk3GE19qafeMKyZJJHP8+O1HiFhFLUNKSg==
+"@typescript-eslint/visitor-keys@6.19.1":
+  version "6.19.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.19.1.tgz#2164073ed4fc34a5ff3b5e25bb5a442100454c4c"
+  integrity sha512-gkdtIO+xSO/SmI0W68DBg4u1KElmIUo3vXzgHyGPs6cxgB0sa3TlptRAAE0hUY1hM6FcDKEv7aIwiTGm76cXfQ==
   dependencies:
-    "@typescript-eslint/types" "6.17.0"
+    "@typescript-eslint/types" "6.19.1"
     eslint-visitor-keys "^3.4.1"
 
 "@ungap/structured-clone@^1.2.0":
@@ -2286,10 +2275,10 @@ ajv@^6.12.4, ajv@~6.12.6:
     json-schema-traverse "^0.4.1"
     uri-js "^4.2.2"
 
-allchange@^1.0.6:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/allchange/-/allchange-1.1.0.tgz#f8fa129e4b40c0b0a2c072c530f2324c6590e208"
-  integrity sha512-brDWf2feuL3FRyivSyC6AKOgpX+bYgs1Z7+ZmLti6PnBdZgIjRSnKvlc68N8+1UX2rCISx2I+XuUvE3/GJNG2A==
+allchange@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/allchange/-/allchange-1.3.0.tgz#0d38e76e069eacd0279fb9a171770426d8e57d37"
+  integrity sha512-orTQYJQzY98ZNvh9VFpBpxLry9obXvDOYuQZXDnTL/YJL3sphgr93norJrR8Qz8mNlJ3yEm1YS+aEEbC3/3Wjg==
   dependencies:
     "@actions/core" "^1.4.0"
     "@actions/github" "^5.0.0"
@@ -2298,7 +2287,7 @@ allchange@^1.0.6:
     js-yaml "^4.1.0"
     loglevel "^1.7.1"
     semver "^7.3.5"
-    yargs "^17.0.1"
+    yargs "^17.5.1"
 
 another-json@^0.2.0:
   version "0.2.0"
@@ -2505,7 +2494,7 @@ babel-plugin-jest-hoist@^29.6.3:
     "@types/babel__core" "^7.1.14"
     "@types/babel__traverse" "^7.0.6"
 
-babel-plugin-polyfill-corejs2@^0.4.6:
+babel-plugin-polyfill-corejs2@^0.4.7:
   version "0.4.7"
   resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.4.7.tgz#679d1b94bf3360f7682e11f2cb2708828a24fe8c"
   integrity sha512-LidDk/tEGDfuHW2DWh/Hgo4rmnw3cduK6ZkOI1NPFceSK3n/yAGeOsNT7FLnSGHkXj3RHGSEVkN3FsCTY6w2CQ==
@@ -2514,7 +2503,7 @@ babel-plugin-polyfill-corejs2@^0.4.6:
     "@babel/helper-define-polyfill-provider" "^0.4.4"
     semver "^6.3.1"
 
-babel-plugin-polyfill-corejs3@^0.8.5:
+babel-plugin-polyfill-corejs3@^0.8.7:
   version "0.8.7"
   resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.8.7.tgz#941855aa7fdaac06ed24c730a93450d2b2b76d04"
   integrity sha512-KyDvZYxAzkC0Aj2dAPyDzi2Ym15e5JKZSK+maI7NAwSqofvuFglbSsxE7wUOvTg9oFVnHMzVzBKcqEb4PJgtOA==
@@ -2522,7 +2511,7 @@ babel-plugin-polyfill-corejs3@^0.8.5:
     "@babel/helper-define-polyfill-provider" "^0.4.4"
     core-js-compat "^3.33.1"
 
-babel-plugin-polyfill-regenerator@^0.5.3:
+babel-plugin-polyfill-regenerator@^0.5.4:
   version "0.5.4"
   resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.5.4.tgz#c6fc8eab610d3a11eb475391e52584bacfc020f4"
   integrity sha512-S/x2iOCvDaCASLYsOOgWOq4bCfKYVqvO/uxjkaYyZ3rVsVE3CeAI/c84NpyuBBymEgNvHgjEot3a9/Z/kXvqsg==
@@ -2663,9 +2652,9 @@ camelcase@^6.2.0:
   integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==
 
 caniuse-lite@^1.0.30001565:
-  version "1.0.30001568"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001568.tgz#53fa9297273c9a977a560663f48cbea1767518b7"
-  integrity sha512-vSUkH84HontZJ88MiNrOau1EBrCqEQYgkC5gIySiDlpsm8sGVrhU7Kx4V6h0tnqaHzIHZv08HlJIwPbL4XL9+A==
+  version "1.0.30001576"
+  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001576.tgz#893be772cf8ee6056d6c1e2d07df365b9ec0a5c4"
+  integrity sha512-ff5BdakGe2P3SQsMsiqmt1Lc8221NR1VzHj5jXN5vBny9A6fpze94HiVV/n7XRosOlsShJcvMv5mdnpjOGCEgg==
 
 chalk@5.2.0:
   version "5.2.0"
@@ -2719,10 +2708,10 @@ ci-info@^3.2.0:
   resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.9.0.tgz#4279a62028a7b1f262f3473fc9605f5e218c59b4"
   integrity sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==
 
-ci-info@^3.8.0:
-  version "3.8.0"
-  resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.8.0.tgz#81408265a5380c929f0bc665d62256628ce9ef91"
-  integrity sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==
+ci-info@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-4.0.0.tgz#65466f8b280fc019b9f50a5388115d17a63a44f2"
+  integrity sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==
 
 cjs-module-lexer@^1.0.0:
   version "1.2.3"
@@ -2874,10 +2863,10 @@ convert-source-map@^2.0.0:
   resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a"
   integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==
 
-core-js-compat@^3.31.0, core-js-compat@^3.33.1:
-  version "3.34.0"
-  resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.34.0.tgz#61a4931a13c52f8f08d924522bba65f8c94a5f17"
-  integrity sha512-4ZIyeNbW/Cn1wkMMDy+mvrRUxrwFNjKwbhCfQpDd+eLgYipDqp8oGFGtLmhh18EDPKA0g3VUBYOxQGGwvWLVpA==
+core-js-compat@^3.31.0, core-js-compat@^3.33.1, core-js-compat@^3.34.0:
+  version "3.35.0"
+  resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.35.0.tgz#c149a3d1ab51e743bc1da61e39cb51f461a41873"
+  integrity sha512-5blwFAddknKeNgsjBzilkdQ0+YK8L1PfqPYq40NOYMYFSS38qj+hpTcLLWwpIwA2A5bje/x5jmVn2tzUMg9IVw==
   dependencies:
     browserslist "^4.22.2"
 
@@ -2995,7 +2984,7 @@ define-data-property@^1.0.1, define-data-property@^1.1.1:
     gopd "^1.0.1"
     has-property-descriptors "^1.0.0"
 
-define-properties@^1.1.3, define-properties@^1.1.4, define-properties@^1.2.0:
+define-properties@^1.1.3, define-properties@^1.2.0, define-properties@^1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.2.1.tgz#10781cc616eb951a80a034bafcaa7377f6af2b6c"
   integrity sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==
@@ -3073,9 +3062,9 @@ eastasianwidth@^0.2.0:
   integrity sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==
 
 electron-to-chromium@^1.4.601:
-  version "1.4.610"
-  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.610.tgz#e17b22203f4aa2e1ed77759c720546d95a51186f"
-  integrity sha512-mqi2oL1mfeHYtOdCxbPQYV/PL7YrQlxbvFEZ0Ee8GbDdShimqt2/S6z2RWqysuvlwdOrQdqvE0KZrBTipAeJzg==
+  version "1.4.625"
+  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.625.tgz#a9a1d18ee911f9074a9c42d9e84b1c79b29f4059"
+  integrity sha512-DENMhh3MFgaPDoXWrVIqSPInQoLImywfCwrSmVl3cf9QHzoZSiutHwGaB/Ql3VkqcQV30rzgdM+BjKqBAJxo5Q==
 
 emittery@^0.13.1:
   version "0.13.1"
@@ -3294,9 +3283,9 @@ eslint-module-utils@^2.7.4, eslint-module-utils@^2.8.0:
     debug "^3.2.7"
 
 eslint-plugin-import@^2.26.0:
-  version "2.29.0"
-  resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.29.0.tgz#8133232e4329ee344f2f612885ac3073b0b7e155"
-  integrity sha512-QPOO5NO6Odv5lpoTkddtutccQjysJuFxoPS7fAHO+9m9udNHvTCPSAMW9zGAYj8lAIdr40I8yPCdUYrncXtrwg==
+  version "2.29.1"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.29.1.tgz#d45b37b5ef5901d639c15270d74d46d161150643"
+  integrity sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw==
   dependencies:
     array-includes "^3.1.7"
     array.prototype.findlastindex "^1.2.3"
@@ -3314,19 +3303,19 @@ eslint-plugin-import@^2.26.0:
     object.groupby "^1.0.1"
     object.values "^1.1.7"
     semver "^6.3.1"
-    tsconfig-paths "^3.14.2"
+    tsconfig-paths "^3.15.0"
 
 eslint-plugin-jest@^27.1.6:
-  version "27.6.0"
-  resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-27.6.0.tgz#e5c0cf735b3c8cad0ef9db5b565b2fc99f5e55ed"
-  integrity sha512-MTlusnnDMChbElsszJvrwD1dN3x6nZl//s4JD23BxB6MgR66TZlL064su24xEIS3VACfAoHV1vgyMgPw8nkdng==
+  version "27.6.3"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-27.6.3.tgz#8acb8b1e45597fe1f4d4cf25163d90119efc12be"
+  integrity sha512-+YsJFVH6R+tOiO3gCJon5oqn4KWc+mDq2leudk8mrp8RFubLOo9CVyi3cib4L7XMpxExmkmBZQTPDYVBzgpgOA==
   dependencies:
     "@typescript-eslint/utils" "^5.10.0"
 
-eslint-plugin-jsdoc@^46.0.0:
-  version "46.9.0"
-  resolved "https://registry.yarnpkg.com/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-46.9.0.tgz#9887569dbeef0a008a2770bfc5d0f7fc39f21f2b"
-  integrity sha512-UQuEtbqLNkPf5Nr/6PPRCtr9xypXY+g8y/Q7gPa0YK7eDhh0y2lWprXRnaYbW7ACgIUvpDKy9X2bZqxtGzBG9Q==
+eslint-plugin-jsdoc@^48.0.0:
+  version "48.0.2"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-48.0.2.tgz#b843107e8caec57b8b463f04797be3b30d6a52da"
+  integrity sha512-CBFl5Jc7+jlV36RwDm+PQ8Uw5r28pn2/uW/OaB+Gw5bFwn4Py/1eYMZ3hGf9S4meUFZ/sRvS+hVif2mRAp6WqQ==
   dependencies:
     "@es-joy/jsdoccomment" "~0.41.0"
     are-docs-informative "^0.0.2"
@@ -3336,7 +3325,7 @@ eslint-plugin-jsdoc@^46.0.0:
     esquery "^1.5.0"
     is-builtin-module "^3.2.1"
     semver "^7.5.4"
-    spdx-expression-parse "^3.0.1"
+    spdx-expression-parse "^4.0.0"
 
 eslint-plugin-matrix-org@^1.0.0:
   version "1.2.1"
@@ -3351,15 +3340,17 @@ eslint-plugin-tsdoc@^0.2.17:
     "@microsoft/tsdoc" "0.14.2"
     "@microsoft/tsdoc-config" "0.16.2"
 
-eslint-plugin-unicorn@^49.0.0:
-  version "49.0.0"
-  resolved "https://registry.yarnpkg.com/eslint-plugin-unicorn/-/eslint-plugin-unicorn-49.0.0.tgz#4449ea954d7e1455eec8518f9417d7021b245fa8"
-  integrity sha512-0fHEa/8Pih5cmzFW5L7xMEfUTvI9WKeQtjmKpTUmY+BiFCDxkxrTdnURJOHKykhtwIeyYsxnecbGvDCml++z4Q==
+eslint-plugin-unicorn@^50.0.0:
+  version "50.0.1"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-unicorn/-/eslint-plugin-unicorn-50.0.1.tgz#e539cdb02dfd893c603536264c4ed9505b70e3bf"
+  integrity sha512-KxenCZxqSYW0GWHH18okDlOQcpezcitm5aOSz6EnobyJ6BIByiPDviQRjJIUAjG/tMN11958MxaQ+qCoU6lfDA==
   dependencies:
     "@babel/helper-validator-identifier" "^7.22.20"
     "@eslint-community/eslint-utils" "^4.4.0"
-    ci-info "^3.8.0"
+    "@eslint/eslintrc" "^2.1.4"
+    ci-info "^4.0.0"
     clean-regexp "^1.0.0"
+    core-js-compat "^3.34.0"
     esquery "^1.5.0"
     indent-string "^4.0.0"
     is-builtin-module "^3.2.1"
@@ -3397,25 +3388,20 @@ eslint-visitor-keys@^2.1.0:
   resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz#f65328259305927392c938ed44eb0a5c9b2bd303"
   integrity sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==
 
-eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.3:
+eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4.3:
   version "3.4.3"
   resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800"
   integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==
 
-eslint-visitor-keys@^3.4.1:
-  version "3.4.2"
-  resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.2.tgz#8c2095440eca8c933bedcadf16fefa44dbe9ba5f"
-  integrity sha512-8drBzUEyZ2llkpCA67iYrgEssKDUu68V8ChqqOfFupIaG/LCVPUT+CoGJpT77zJprs4T/W7p07LP7zAIMuweVw==
-
-eslint@8.55.0:
-  version "8.55.0"
-  resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.55.0.tgz#078cb7b847d66f2c254ea1794fa395bf8e7e03f8"
-  integrity sha512-iyUUAM0PCKj5QpwGfmCAG9XXbZCWsqP/eWAWrG/W0umvjuLRBECwSFdt+rCntju0xEH7teIABPwXpahftIaTdA==
+eslint@8.56.0:
+  version "8.56.0"
+  resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.56.0.tgz#4957ce8da409dc0809f99ab07a1b94832ab74b15"
+  integrity sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ==
   dependencies:
     "@eslint-community/eslint-utils" "^4.2.0"
     "@eslint-community/regexpp" "^4.6.1"
     "@eslint/eslintrc" "^2.1.4"
-    "@eslint/js" "8.55.0"
+    "@eslint/js" "8.56.0"
     "@humanwhocodes/config-array" "^0.11.13"
     "@humanwhocodes/module-importer" "^1.0.1"
     "@nodelib/fs.walk" "^1.2.8"
@@ -3586,7 +3572,18 @@ fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3:
   resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525"
   integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==
 
-fast-glob@^3.2.9, fast-glob@^3.3.1:
+fast-glob@^3.2.9:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.2.tgz#a904501e57cfdd2ffcded45e99a54fef55e46129"
+  integrity sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==
+  dependencies:
+    "@nodelib/fs.stat" "^2.0.2"
+    "@nodelib/fs.walk" "^1.2.3"
+    glob-parent "^5.1.2"
+    merge2 "^1.3.0"
+    micromatch "^4.0.4"
+
+fast-glob@^3.3.1:
   version "3.3.1"
   resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.1.tgz#784b4e897340f3dbbef17413b3f11acf03c874c4"
   integrity sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg==
@@ -3608,9 +3605,9 @@ fast-levenshtein@^2.0.6:
   integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==
 
 fastq@^1.6.0:
-  version "1.15.0"
-  resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.15.0.tgz#d04d07c6a2a68fe4599fea8d2e103a937fae6b3a"
-  integrity sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==
+  version "1.16.0"
+  resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.16.0.tgz#83b9a9375692db77a822df081edb6a9cf6839320"
+  integrity sha512-ifCoaXsDrsdkWTtiNJX5uzHDsrck5TzfKKDcuFFTIrrc/BS076qgEIfoIy1VeZqViznfKiysPYTh/QeHtnIsYA==
   dependencies:
     reusify "^1.0.4"
 
@@ -4023,12 +4020,7 @@ iconv-lite@0.6.3:
   dependencies:
     safer-buffer ">= 2.1.2 < 3.0.0"
 
-ignore@^5.2.0:
-  version "5.2.4"
-  resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324"
-  integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==
-
-ignore@^5.2.4:
+ignore@^5.2.0, ignore@^5.2.4:
   version "5.3.0"
   resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.0.tgz#67418ae40d34d6999c95ff56016759c718c82f78"
   integrity sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==
@@ -5287,12 +5279,12 @@ object-keys@^1.1.1:
   integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==
 
 object.assign@^4.1.4:
-  version "4.1.4"
-  resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.4.tgz#9673c7c7c351ab8c4d0b516f4343ebf4dfb7799f"
-  integrity sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ==
+  version "4.1.5"
+  resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.5.tgz#3a833f9ab7fdb80fc9e8d2300c803d216d8fdbb0"
+  integrity sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==
   dependencies:
-    call-bind "^1.0.2"
-    define-properties "^1.1.4"
+    call-bind "^1.0.5"
+    define-properties "^1.2.1"
     has-symbols "^1.0.3"
     object-keys "^1.1.1"
 
@@ -5505,7 +5497,7 @@ pify@^4.0.1:
   resolved "https://registry.yarnpkg.com/pify/-/pify-4.0.1.tgz#4b2cd25c50d598735c50292224fd8c6df41e3231"
   integrity sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==
 
-pirates@^4.0.4, pirates@^4.0.5:
+pirates@^4.0.4, pirates@^4.0.6:
   version "4.0.6"
   resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.6.tgz#3018ae32ecfcff6c29ba2267cbf21166ac1f36b9"
   integrity sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==
@@ -5534,10 +5526,10 @@ prelude-ls@^1.2.1:
   resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396"
   integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==
 
-prettier@3.1.1:
-  version "3.1.1"
-  resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.1.1.tgz#6ba9f23165d690b6cbdaa88cb0807278f7019848"
-  integrity sha512-22UbSzg8luF4UuZtzgiUOfcGM8s4tjBv6dJRT7j275NXsy2jb4aJa4NNveul5x4eqlF1wuhuR2RElK71RvmVaw==
+prettier@3.2.4:
+  version "3.2.4"
+  resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.2.4.tgz#4723cadeac2ce7c9227de758e5ff9b14e075f283"
+  integrity sha512-FWu1oLHKCrtpO1ypU6J0SbK2d9Ckwysq6bHj/uaCP26DxrPpppCLQRGVuqAxSTvhF00AcvDRyYrLNW7ocBhFFQ==
 
 pretty-format@^28.1.3:
   version "28.1.3"
@@ -5640,9 +5632,9 @@ regenerate@^1.4.2:
   integrity sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==
 
 regenerator-runtime@^0.14.0:
-  version "0.14.0"
-  resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz#5e19d68eb12d486f797e15a3c6a918f7cec5eb45"
-  integrity sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA==
+  version "0.14.1"
+  resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#356ade10263f685dda125100cd862c1db895327f"
+  integrity sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==
 
 regenerator-transform@^0.15.2:
   version "0.15.2"
@@ -5814,12 +5806,12 @@ safe-buffer@^5.0.1:
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
 
 safe-regex-test@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/safe-regex-test/-/safe-regex-test-1.0.0.tgz#793b874d524eb3640d1873aad03596db2d4f2295"
-  integrity sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/safe-regex-test/-/safe-regex-test-1.0.1.tgz#207369b445fd007e534864635b28b2ae7b105783"
+  integrity sha512-Y5NejJTTliTyY4H7sipGqY+RX5P87i3F7c4Rcepy72nq+mNLhIsD0W4c7kEmduMDQCSqtPsXPlSTsFhh2LQv+g==
   dependencies:
-    call-bind "^1.0.2"
-    get-intrinsic "^1.1.3"
+    call-bind "^1.0.5"
+    get-intrinsic "^1.2.2"
     is-regex "^1.1.4"
 
 "safer-buffer@>= 2.1.2 < 3.0.0":
@@ -5988,7 +5980,7 @@ spdx-exceptions@^2.1.0:
   resolved "https://registry.yarnpkg.com/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz#3f28ce1a77a00372683eade4a433183527a2163d"
   integrity sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==
 
-spdx-expression-parse@^3.0.0, spdx-expression-parse@^3.0.1:
+spdx-expression-parse@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz#cf70f50482eefdc98e3ce0a6833e4a53ceeba679"
   integrity sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==
@@ -5996,6 +5988,14 @@ spdx-expression-parse@^3.0.0, spdx-expression-parse@^3.0.1:
     spdx-exceptions "^2.1.0"
     spdx-license-ids "^3.0.0"
 
+spdx-expression-parse@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/spdx-expression-parse/-/spdx-expression-parse-4.0.0.tgz#a23af9f3132115465dac215c099303e4ceac5794"
+  integrity sha512-Clya5JIij/7C6bRR22+tnGXbc4VKlibKSVj2iHvVeX5iMW7s1SIQlqu699JkODJJIhh/pUu8L0/VLh8xflD+LQ==
+  dependencies:
+    spdx-exceptions "^2.1.0"
+    spdx-license-ids "^3.0.0"
+
 spdx-license-ids@^3.0.0:
   version "3.0.16"
   resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.16.tgz#a14f64e0954f6e25cc6587bd4f392522db0d998f"
@@ -6254,10 +6254,10 @@ ts-node@^10.9.1:
     v8-compile-cache-lib "^3.0.1"
     yn "3.1.1"
 
-tsconfig-paths@^3.14.2:
-  version "3.14.2"
-  resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz#6e32f1f79412decd261f92d633a9dc1cfa99f088"
-  integrity sha512-o/9iXgCYc5L/JxCHPe3Hvh8Q/2xm5Z+p18PESBU6Ff33695QnCHBEjcytY2q19ua7Mbl/DavtBOLq+oG0RCL+g==
+tsconfig-paths@^3.15.0:
+  version "3.15.0"
+  resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz#5299ec605e55b1abb23ec939ef15edaf483070d4"
+  integrity sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg==
   dependencies:
     "@types/json5" "^0.0.29"
     json5 "^1.0.2"
@@ -6719,7 +6719,7 @@ yargs-parser@^21.1.1:
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35"
   integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==
 
-yargs@^17.0.1, yargs@^17.3.1, yargs@^17.5.1:
+yargs@^17.3.1, yargs@^17.5.1:
   version "17.7.2"
   resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269"
   integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==