v30.3.0-rc.0

* Comment explaining the purpose of RoomList

* Fix incorrect return type declaration on RoomList.getRoomEncryption

* Move RoomList out of MatrixClient, into legacy Crypto

* Initialise RoomList inside Crypto.init to allow us to await it

.github/actions/sign-release-tarball/action.yml

@@ -0,0 +1,28 @@
+name: Sign Release Tarball
+description: Generates signature for release tarball and uploads it as a release asset
+inputs:
+    gpg-fingerprint:
+        description: Fingerprint of the GPG key to use for signing the tarball.
+        required: true
+    upload-url:
+        description: GitHub release upload URL to upload the signature file to.
+        required: true
+runs:
+    using: composite
+    steps:
+        - name: Generate tarball signature
+          shell: bash
+          run: |
+              git -c tar.tar.gz.command='gzip -cn' archive --format=tar.gz --prefix="${REPO#*/}-${VERSION#v}/" -o "/tmp/${VERSION}.tar.gz" "${VERSION}"
+              gpg -u "$GPG_FINGERPRINT" --armor --output "${VERSION}.tar.gz.asc" --detach-sig "/tmp/${VERSION}.tar.gz"
+              rm "/tmp/${VERSION}.tar.gz"
+          env:
+              GPG_FINGERPRINT: ${{ inputs.gpg-fingerprint }}
+              REPO: ${{ github.repository }}
+
+        - name: Upload tarball signature
+          if: ${{ inputs.upload-url }}
+          uses: shogo82148/actions-upload-release-asset@dccd6d23e64fd6a746dce6814c0bde0a04886085 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ env.VERSION }}.tar.gz.asc

.github/actions/upload-release-assets/action.yml

@@ -0,0 +1,41 @@
+name: Upload release assets
+description: Uploads assets to an existing release and optionally signs them
+inputs:
+    gpg-fingerprint:
+        description: Fingerprint of the GPG key to use for signing the assets, if any.
+        required: false
+    upload-url:
+        description: GitHub release upload URL to upload the assets to.
+        required: true
+    asset-path:
+        description: |
+            The path to the asset you want to upload, if any. You can use glob patterns here.
+            Will be GPG signed and an `.asc` file included in the release artifacts if `gpg-fingerprint` is set.
+        required: true
+runs:
+    using: composite
+    steps:
+        - name: Sign assets
+          if: inputs.gpg-fingerprint
+          shell: bash
+          run: |
+              for FILE in $ASSET_PATH
+              do
+                  gpg -u "$GPG_FINGERPRINT" --armor --output "$FILE".asc --detach-sig "$FILE"
+              done
+          env:
+              GPG_FINGERPRINT: ${{ inputs.gpg-fingerprint }}
+              ASSET_PATH: ${{ inputs.asset-path }}
+
+        - name: Upload asset signatures
+          if: inputs.gpg-fingerprint
+          uses: shogo82148/actions-upload-release-asset@dccd6d23e64fd6a746dce6814c0bde0a04886085 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ inputs.asset-path }}.asc
+
+        - name: Upload assets
+          uses: shogo82148/actions-upload-release-asset@dccd6d23e64fd6a746dce6814c0bde0a04886085 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ inputs.asset-path }}

.github/release-drafter.yml

@@ -0,0 +1,31 @@
+name-template: "v$RESOLVED_VERSION"
+tag-template: "v$RESOLVED_VERSION"
+change-template: "* $TITLE ([#$NUMBER]($URL)). Contributed by @$AUTHOR."
+categories:
+    - title: "🚨 BREAKING CHANGES"
+      label: "X-Breaking-Change"
+    - title: "🦖 Deprecations"
+      label: "T-Deprecation"
+    - title: "✨ Features"
+      label: "T-Enhancement"
+    - title: "🐛 Bug Fixes"
+      label: "T-Defect"
+    - title: "🧰 Maintenance"
+      label: "Dependencies"
+      collapse-after: 5
+change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
+version-resolver:
+    major:
+        labels:
+            - "X-Breaking-Change"
+    default: minor
+exclude-labels:
+    - "T-Task"
+    - "X-Reverted"
+exclude-contributors:
+    - "RiotRobot"
+template: |
+    $CHANGES
+prerelease: true
+prerelease-identifier: rc
+include-pre-releases: false

.github/workflows/cypress.yml

@@ -1,7 +1,7 @@
 # Triggers after the "Downstream artifacts" build has finished, to run the
-# cypress tests (with access to repo secrets)
+# matrix-react-sdk playwright & cypress tests (with access to repo secrets)
 
-name: matrix-react-sdk Cypress End to End Tests
+name: matrix-react-sdk End to End Tests
 on:
     workflow_run:
         workflows: ["Build downstream artifacts"]
@@ -17,10 +17,10 @@ jobs:
         name: Cypress
 
         # We only want to run the cypress tests on merge queue to prevent regressions
-        # from creeping in. They take a long time to run and consume 4 concurrent runners.
+        # from creeping in. They take a long time to run and consume multiple concurrent runners.
         if: github.event.workflow_run.event == 'merge_group'
 
-        uses: matrix-org/matrix-react-sdk/.github/workflows/cypress.yaml@66854039a33ed6cfe1fc635ff2daa8bb261c0b56
+        uses: matrix-org/matrix-react-sdk/.github/workflows/cypress.yaml@develop
         permissions:
             actions: read
             issues: read
@@ -28,12 +28,27 @@ jobs:
             pull-requests: read
         secrets:
             # secrets are not automatically shared with called workflows, so share the cypress dashboard key, and the Kiwi login details
-            CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+            KNAPSACK_PRO_TEST_SUITE_TOKEN_CYPRESS_RUST: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_CYPRESS_RUST}}
+            KNAPSACK_PRO_TEST_SUITE_TOKEN_CYPRESS_LEGACY: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_CYPRESS_LEGACY}}
             TCMS_USERNAME: ${{ secrets.TCMS_USERNAME }}
             TCMS_PASSWORD: ${{ secrets.TCMS_PASSWORD }}
         with:
             react-sdk-repository: matrix-org/matrix-react-sdk
-            rust-crypto: true
+
+    playwright:
+        name: Playwright
+        # We only want to run the playwright tests on merge queue to prevent regressions
+        # from creeping in. They take a long time to run and consume multiple concurrent runners.
+        if: github.event.workflow_run.event == 'merge_group'
+        uses: matrix-org/matrix-react-sdk/.github/workflows/end-to-end-tests.yaml@develop
+        permissions:
+            actions: read
+            issues: read
+            statuses: write
+            pull-requests: read
+            deployments: write
+        with:
+            react-sdk-repository: matrix-org/matrix-react-sdk
 
     # We want to make the cypress tests a required check for the merge queue.
     #
@@ -50,10 +65,26 @@ jobs:
             statuses: write
         runs-on: ubuntu-latest
         steps:
-            - uses: Sibz/github-status-action@650dd1a882a76dbbbc4576fb5974b8d22f29847f # v1.1.6
+            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               with:
                   authToken: "${{ secrets.GITHUB_TOKEN }}"
                   state: success
                   description: Cypress skipped
+
+                  # Keep in step with the `context` that is updated by `Sibz/github-status-action`
+                  # in matrix-org/matrix-react-sdk/.github/workflows/cypress.yaml.
                   context: "${{ github.workflow }} / cypress"
+
+                  sha: "${{ github.event.workflow_run.head_sha }}"
+
+            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
+              with:
+                  authToken: "${{ secrets.GITHUB_TOKEN }}"
+                  state: success
+                  description: Playwright skipped
+
+                  # Keep in step with the `context` that is updated by `Sibz/github-status-action`
+                  # in matrix-org/matrix-react-sdk/.github/workflows/end-to-end-tests.yaml.
+                  context: "${{ github.workflow }} / end-to-end-tests"
+
                   sha: "${{ github.event.workflow_run.head_sha }}"

.github/workflows/docs-pr-netlify.yaml

@@ -22,7 +22,7 @@ jobs:
                   path: docs
 
             - name: 📤 Deploy to Netlify
-              uses: matrix-org/netlify-pr-preview@v2
+              uses: matrix-org/netlify-pr-preview@v3
               with:
                   path: docs
                   owner: ${{ github.event.workflow_run.head_repository.owner.login }}
@@ -32,3 +32,4 @@ jobs:
                   site_id: ${{ secrets.NETLIFY_SITE_ID }}
                   desc: Documentation preview
                   deployment_env: PR Documentation Preview
+        environment: PR Documentation Preview

.github/workflows/downstream-artifacts.yml

@@ -20,7 +20,7 @@ concurrency:
 jobs:
     build-element-web:
         name: Build element-web
-        uses: matrix-org/matrix-react-sdk/.github/workflows/element-web.yaml@v3.80.1
+        uses: matrix-org/matrix-react-sdk/.github/workflows/element-web.yaml@v3.85.0
         with:
             matrix-js-sdk-sha: ${{ github.sha }}
             react-sdk-repository: matrix-org/matrix-react-sdk

.github/workflows/pull_request.yaml

@@ -27,7 +27,7 @@ jobs:
             pull-requests: read
         steps:
             - name: Add notice
-              uses: actions/github-script@v6
+              uses: actions/github-script@v7
               if: contains(github.event.pull_request.labels.*.name, 'X-Blocked')
               with:
                   script: |
@@ -39,7 +39,7 @@ jobs:
         if: github.event.action == 'opened'
         steps:
             - name: Check membership
-              uses: tspascoal/get-user-teams-membership@37c08f7b52a72ca95d12af2e7ab2553ca9adf13b # v2
+              uses: tspascoal/get-user-teams-membership@ba78054988f58bea69b7c6136d563236f8ed2fc0 # v3
               id: teams
               with:
                   username: ${{ github.event.pull_request.user.login }}
@@ -49,7 +49,7 @@ jobs:
 
             - name: Add label
               if: ${{ steps.teams.outputs.isTeamMember == 'false' }}
-              uses: actions/github-script@v6
+              uses: actions/github-script@v7
               with:
                   script: |
                       github.rest.issues.addLabels({
@@ -68,7 +68,7 @@ jobs:
             github.event.pull_request.head.repo.full_name != github.repository
         steps:
             - name: Close pull request
-              uses: actions/github-script@v6
+              uses: actions/github-script@v7
               with:
                   script: |
                       github.rest.issues.createComment({

.github/workflows/release-drafter.yml

@@ -0,0 +1,21 @@
+name: Release Drafter
+on:
+    push:
+        branches: [staging]
+    workflow_dispatch:
+        inputs:
+            previous-version:
+                description: What release to use as a base for release note purposes
+                required: false
+                type: string
+concurrency: ${{ github.workflow }}
+jobs:
+    draft:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: release-drafter/release-drafter@e64b19c4c46173209ed9f2e5a2f4ca7de89a0e86 # v5
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                  disable-autolabeler: true
+                  previous-version: ${{ inputs.previous-version }}

.github/workflows/release-gitflow.yml

@@ -0,0 +1,85 @@
+# Gitflow merge-back master->develop
+name: Merge master -> develop
+on:
+    push:
+        branches: [master]
+    workflow_call:
+        secrets:
+            ELEMENT_BOT_TOKEN:
+                required: true
+        inputs:
+            dependencies:
+                description: List of dependencies to reset.
+                type: string
+                required: false
+concurrency: ${{ github.workflow }}
+jobs:
+    merge:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  fetch-depth: 0
+
+            - name: Get actions scripts
+              uses: actions/checkout@v4
+              with:
+                  repository: matrix-org/matrix-js-sdk
+                  persist-credentials: false
+                  path: .action-repo
+                  sparse-checkout: |
+                      scripts/release
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+
+            - name: Install Deps
+              run: "yarn install --frozen-lockfile"
+
+            - name: Set up git
+              run: |
+                  git config --global user.email "releases@riot.im"
+                  git config --global user.name "RiotRobot"
+
+            - name: Merge to develop
+              run: |
+                  git checkout develop
+                  git merge -X ours master
+
+            - name: Run post-merge-master script to revert package.json fields
+              run: ./.action-repo/scripts/release/post-merge-master.sh
+
+            - name: Reset dependencies
+              if: inputs.dependencies
+              run: |
+                  while IFS= read -r PACKAGE; do
+                      [ -z "$PACKAGE" ] && continue
+
+                      CURRENT_VERSION=$(cat package.json | jq -r .dependencies[\"$PACKAGE\"])
+                      echo "Current $PACKAGE version is $CURRENT_VERSION"
+
+                      if [ "$CURRENT_VERSION" == "null" ]
+                      then
+                          echo "Unable to find $PACKAGE in package.json"
+                          exit 1
+                      fi
+
+                      if [ "$CURRENT_VERSION" == "develop" ]
+                      then
+                          echo "Not updating dependency $PACKAGE"
+                          continue
+                      fi
+
+                      echo "Resetting $1 to develop branch..."
+                      yarn add "github:matrix-org/$PACKAGE#develop"
+                      git add -u
+                      git commit -m "Reset $PACKAGE back to develop branch"
+                  done <<< "$DEPENDENCIES"
+              env:
+                  DEPENDENCIES: ${{ inputs.dependencies }}
+                  FINAL: ${{ inputs.final }}
+
+            - name: Push changes
+              run: git push origin develop

.github/workflows/release-make.yml

@@ -0,0 +1,353 @@
+name: Release Make
+on:
+    workflow_call:
+        secrets:
+            ELEMENT_BOT_TOKEN:
+                required: true
+            NPM_TOKEN:
+                required: false
+            GPG_PASSPHRASE:
+                required: false
+            GPG_PRIVATE_KEY:
+                required: false
+        inputs:
+            final:
+                description: Make final release
+                required: true
+                default: false
+                type: boolean
+            npm:
+                description: Publish to npm
+                type: boolean
+                default: false
+            dependencies:
+                description: |
+                    List of dependencies to update in `npm-dep=version` format.
+                    `version` can be `"current"` to leave it at the current version.
+                type: string
+                required: false
+            include-changes:
+                description: Project to include changelog entries from in this release.
+                type: string
+                required: false
+            gpg-fingerprint:
+                description: Fingerprint of the GPG key to use for signing the git tag and assets, if any.
+                type: string
+                required: false
+            asset-path:
+                description: |
+                    The path to the asset you want to upload, if any. You can use glob patterns here.
+                    Will be GPG signed and an `.asc` file included in the release artifacts if `gpg-fingerprint` is set.
+                type: string
+                required: false
+            expected-asset-count:
+                description: The number of expected assets, including signatures, excluding generated zip & tarball.
+                type: number
+                required: false
+jobs:
+    release:
+        name: Release
+        runs-on: ubuntu-latest
+        environment: Release
+        steps:
+            - name: Load GPG key
+              id: gpg
+              if: inputs.gpg-fingerprint
+              uses: crazy-max/ghaction-import-gpg@82a020f1f7f605c65dd2449b392a52c3fcfef7ef # v6
+              with:
+                  gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+                  passphrase: ${{ secrets.GPG_PASSPHRASE }}
+                  fingerprint: ${{ inputs.gpg-fingerprint }}
+
+            - name: Get draft release
+              id: release
+              uses: cardinalby/git-get-release-action@cedef2faf69cb7c55b285bad07688d04430b7ada # v1
+              env:
+                  GITHUB_TOKEN: ${{ github.token }}
+              with:
+                  draft: true
+                  latest: true
+
+            - uses: actions/checkout@v4
+              with:
+                  ref: staging
+                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  fetch-depth: 0
+
+            - name: Get actions scripts
+              uses: actions/checkout@v4
+              with:
+                  repository: matrix-org/matrix-js-sdk
+                  persist-credentials: false
+                  path: .action-repo
+                  sparse-checkout: |
+                      .github/actions
+                      scripts/release
+
+            - name: Prepare variables
+              id: prepare
+              run: |
+                  echo "VERSION=$VERSION" >> $GITHUB_ENV
+                  {
+                      echo "RELEASE_NOTES<<EOF"
+                      echo "$BODY"
+                      echo "EOF"
+                  } >> $GITHUB_ENV
+
+                  HAS_DIST=0
+                  jq -e .scripts.dist package.json >/dev/null 2>&1 && HAS_DIST=1
+                  echo "has-dist-script=$HAS_DIST" >> $GITHUB_OUTPUT
+              env:
+                  BODY: ${{ steps.release.outputs.body }}
+                  VERSION: ${{ steps.release.outputs.tag_name }}
+
+            - name: Finalise version
+              if: inputs.final
+              run: echo "VERSION=$(echo $VERSION | cut -d- -f1)" >> $GITHUB_ENV
+
+            - name: Check version number not in use
+              uses: actions/github-script@v7
+              with:
+                  script: |
+                      const { VERSION } = process.env;
+                      github.rest.repos.getReleaseByTag({
+                          owner: context.repo.owner,
+                          repo: context.repo.repo,
+                          tag: VERSION,
+                      }).then(() => {
+                          core.setFailed(`Version ${VERSION} already exists`);
+                      }).catch(() => {
+                        // This is fine, we expect there to not be any release with this version yet
+                      });
+
+            - name: Set up git
+              run: |
+                  git config --global user.email "releases@riot.im"
+                  git config --global user.name "RiotRobot"
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+
+            - name: Install dependencies
+              run: "yarn install --frozen-lockfile"
+
+            - name: Update dependencies
+              id: update-dependencies
+              if: inputs.dependencies
+              run: |
+                  UPDATED=()
+                  while IFS= read -r DEPENDENCY; do
+                      [ -z "$DEPENDENCY" ] && continue
+                      IFS="=" read -r PACKAGE UPDATE_VERSION <<< "$DEPENDENCY"
+
+                      CURRENT_VERSION=$(cat package.json | jq -r .dependencies[\"$PACKAGE\"])
+                      echo "Current $PACKAGE version is $CURRENT_VERSION"
+
+                      if [ "$CURRENT_VERSION" == "null" ]
+                      then
+                          echo "Unable to find $PACKAGE in package.json"
+                          exit 1
+                      fi
+
+                      if [ "$UPDATE_VERSION" == "current" ] || [ "$UPDATE_VERSION" == "$CURRENT_VERSION" ]
+                      then
+                          echo "Not updating dependency $PACKAGE"
+                          continue
+                      fi
+
+                      echo "Upgrading $PACKAGE to $UPDATE_VERSION..."
+                      yarn upgrade "$PACKAGE@$UPDATE_VERSION" --exact
+                      git add -u
+                      git commit -m "Upgrade $PACKAGE to $UPDATE_VERSION"
+                      UPDATED+=("$PACKAGE")
+                  done <<< "$DEPENDENCIES"
+
+                  JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${UPDATED[@]}")
+                  echo "updated=$JSON" >> $GITHUB_OUTPUT
+              env:
+                  DEPENDENCIES: ${{ inputs.dependencies }}
+
+            - name: Prevent develop dependencies
+              if: inputs.dependencies
+              run: |
+                  ret=0
+                  cat package.json | jq '.dependencies[]' | grep -q '#develop' || ret=$?
+                  if [ "$ret" -eq 0 ]; then
+                      echo "package.json contains develop dependencies. Refusing to release."
+                      exit
+                  fi
+
+            - name: Bump package.json version
+              run: yarn version --no-git-tag-version --new-version "${VERSION#v}"
+
+            - name: Ingest upstream changes
+              if: |
+                  inputs.include-changes &&
+                  (!inputs.dependencies || contains(fromJSON(steps.update-dependencies.outputs.updated), inputs.include-changes))
+              uses: actions/github-script@v7
+              env:
+                  RELEASE_ID: ${{ steps.release.outputs.id }}
+                  DEPENDENCY: ${{ inputs.include-changes }}
+              with:
+                  retries: 3
+                  script: |
+                      const { RELEASE_ID: releaseId, DEPENDENCY, VERSION } = process.env;
+                      const { owner, repo } = context.repo;
+                      const script = require("./.action-repo/scripts/release/merge-release-notes.js");
+                      const notes = await script({
+                          github,
+                          releaseId,
+                          dependencies: [DEPENDENCY.replace("$VERSION", VERSION)],
+                      });
+                      core.exportVariable("RELEASE_NOTES", notes);
+
+            - name: Add to CHANGELOG.md
+              if: inputs.final
+              run: |
+                  mv CHANGELOG.md CHANGELOG.md.old
+                  HEADER="Changes in [${VERSION#v}](https://github.com/${{ github.repository }}/releases/tag/$VERSION) ($(date '+%Y-%m-%d'))"
+
+                  {
+                      echo "$HEADER"
+                      printf '=%.0s' $(seq ${#HEADER})
+                      echo ""
+                      echo "$RELEASE_NOTES"
+                      echo ""
+                  } > CHANGELOG.md
+
+                  cat CHANGELOG.md.old >> CHANGELOG.md
+                  rm CHANGELOG.md.old
+                  git add CHANGELOG.md
+
+            - name: Run pre-release script to update package.json fields
+              run: |
+                  ./.action-repo/scripts/release/pre-release.sh
+                  git add package.json
+
+            - name: Commit changes
+              run: git commit -m "$VERSION"
+
+            - name: Build assets
+              if: steps.prepare.outputs.has-dist-script == '1'
+              run: DIST_VERSION="$VERSION" yarn dist
+
+            - name: Upload release assets & signatures
+              if: inputs.asset-path
+              uses: ./.action-repo/.github/actions/upload-release-assets
+              with:
+                  gpg-fingerprint: ${{ inputs.gpg-fingerprint }}
+                  upload-url: ${{ steps.release.outputs.upload_url }}
+                  asset-path: ${{ inputs.asset-path }}
+
+            - name: Create signed tag
+              if: inputs.gpg-fingerprint
+              run: |
+                  GIT_COMMITTER_EMAIL="$SIGNING_ID" GPG_TTY=$(tty) git tag -u "$SIGNING_ID" -m "Release $VERSION" "$VERSION"
+              env:
+                  SIGNING_ID: ${{ steps.gpg.outputs.email }}
+
+            - name: Generate & upload tarball signature
+              if: inputs.gpg-fingerprint
+              uses: ./.action-repo/.github/actions/sign-release-tarball
+              with:
+                  gpg-fingerprint: ${{ inputs.gpg-fingerprint }}
+                  upload-url: ${{ steps.release.outputs.upload_url }}
+
+            # We defer pushing changes until after the release assets are built,
+            # signed & uploaded to improve the atomicity of this action.
+            - name: Push changes to staging
+              run: |
+                  git push origin staging $TAG
+                  git reset --hard
+              env:
+                  TAG: ${{ inputs.gpg-fingerprint && env.VERSION || '' }}
+
+            - name: Validate tarball signature
+              if: inputs.gpg-fingerprint
+              run: |
+                  wget https://github.com/$GITHUB_REPOSITORY/archive/refs/tags/$VERSION.tar.gz
+                  gpg --verify "$VERSION.tar.gz.asc" "$VERSION.tar.gz"
+
+            - name: Validate release has expected assets
+              if: inputs.expected-asset-count
+              uses: actions/github-script@v7
+              env:
+                  RELEASE_ID: ${{ steps.release.outputs.id }}
+                  EXPECTED_ASSET_COUNT: ${{ inputs.expected-asset-count }}
+              with:
+                  retries: 3
+                  script: |
+                      const { RELEASE_ID: release_id, EXPECTED_ASSET_COUNT } = process.env;
+                      const { owner, repo } = context.repo;
+
+                      const { data: release } = await github.rest.repos.getRelease({
+                          owner,
+                          repo,
+                          release_id,
+                      });
+
+                      if (release.assets.length !== parseInt(EXPECTED_ASSET_COUNT, 10)) {
+                          core.setFailed(`Found ${release.assets.length} assets but expected ${EXPECTED_ASSET_COUNT}`);
+                      }
+
+            - name: Merge to master
+              if: inputs.final
+              run: |
+                  git checkout master
+                  git merge -X theirs staging
+                  git push origin master
+
+            - name: Publish release
+              uses: actions/github-script@v7
+              env:
+                  RELEASE_ID: ${{ steps.release.outputs.id }}
+                  FINAL: ${{ inputs.final }}
+              with:
+                  retries: 3
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  script: |
+                      const { RELEASE_ID: release_id, RELEASE_NOTES, VERSION, FINAL } = process.env;
+                      const { owner, repo } = context.repo;
+
+                      const opts = {
+                          owner,
+                          repo,
+                          release_id,
+                          tag_name: VERSION,
+                          name: VERSION,
+                          draft: false,
+                          body: RELEASE_NOTES,
+                      };
+
+                      if (FINAL == "true") {
+                          opts.prerelease = false;
+                          opts.make_latest = true;
+                      }
+
+                      github.rest.repos.updateRelease(opts);
+
+    npm:
+        name: Publish to npm
+        needs: release
+        if: inputs.npm
+        uses: matrix-org/matrix-js-sdk/.github/workflows/release-npm.yml@develop
+        secrets:
+            NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+    update-labels:
+        name: Advance release blocker labels
+        needs: release
+        runs-on: ubuntu-latest
+        steps:
+            - id: repository
+              run: echo "REPO=${GITHUB_REPOSITORY#*/}" >> $GITHUB_OUTPUT
+
+            - uses: garganshu/github-label-updater@3770d15ebfed2fe2cb06a241047bc340f774a7d1 # v1.0.0
+              with:
+                  owner: ${{ github.repository_owner }}
+                  repo: ${{ steps.repository.outputs.REPO }}
+                  token: ${{ secrets.GITHUB_TOKEN }}
+                  filter-labels: X-Upcoming-Release-Blocker
+                  remove-labels: X-Upcoming-Release-Blocker
+                  add-labels: X-Release-Blocker

.github/workflows/release-npm.yml

@@ -1,4 +1,3 @@
-# Must only be called from `release#published` triggers
 name: Publish to npm
 on:
     workflow_call:
@@ -12,9 +11,11 @@ jobs:
         steps:
             - name: 🧮 Checkout code
               uses: actions/checkout@v4
+              with:
+                  ref: staging
 
             - name: 🔧 Yarn cache
-              uses: actions/setup-node@v3
+              uses: actions/setup-node@v4
               with:
                   cache: "yarn"
                   registry-url: "https://registry.npmjs.org"
@@ -24,7 +25,7 @@ jobs:
 
             - name: 🚀 Publish to npm
               id: npm-publish
-              uses: JS-DevTools/npm-publish@fe72237be0920f7a0cafd6a966c9b929c9466e9b # v2.2.2
+              uses: JS-DevTools/npm-publish@4b07b26a2f6e0a51846e1870223e545bae91c552 # v3.0.1
               with:
                   token: ${{ secrets.NPM_TOKEN }}
                   access: public
@@ -32,7 +33,7 @@ jobs:
                   ignore-scripts: false
 
             - name: 🎖️ Add `latest` dist-tag to final releases
-              if: github.event.release.prerelease == false && steps.npm-publish.outputs.id
+              if: steps.npm-publish.outputs.id && !contains(steps.npm-publish.outputs.id, '-rc.')
               run: npm dist-tag add "$release" latest
               env:
                   NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/release.yml

@@ -1,11 +1,38 @@
 name: Release Process
 on:
-    release:
-        types: [published]
-concurrency: ${{ github.workflow }}-${{ github.ref }}
+    workflow_dispatch:
+        inputs:
+            mode:
+                description: What type of release
+                required: true
+                default: rc
+                type: choice
+                options:
+                    - rc
+                    - final
+            docs:
+                description: Publish docs
+                required: true
+                type: boolean
+                default: true
+            npm:
+                description: Publish to npm
+                required: true
+                type: boolean
+                default: true
+concurrency: ${{ github.workflow }}
 jobs:
-    jsdoc:
+    release:
+        uses: matrix-org/matrix-js-sdk/.github/workflows/release-make.yml@develop
+        secrets: inherit
+        with:
+            final: ${{ inputs.mode == 'final' }}
+            npm: ${{ inputs.npm }}
+
+    docs:
         name: Publish Documentation
+        needs: release
+        if: inputs.docs
         runs-on: ubuntu-latest
         steps:
             - name: 🧮 Checkout code
@@ -18,7 +45,7 @@ jobs:
                   path: _docs
 
             - name: 🔧 Yarn cache
-              uses: actions/setup-node@v3
+              uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
@@ -36,17 +63,14 @@ jobs:
                   yarn gendoc
                   symlinks -rc _docs
 
-            - name: 🚀 Deploy
+            - name: 🔨 Set up git
               run: |
                   git config --global user.email "releases@riot.im"
                   git config --global user.name "RiotRobot"
+
+            - name: 🚀 Deploy
+              run: |
                   git add . --all
                   git commit -m "Update docs"
                   git push
               working-directory: _docs
-
-    npm:
-        name: Publish
-        uses: matrix-org/matrix-js-sdk/.github/workflows/release-npm.yml@develop
-        secrets:
-            NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/sonarcloud.yml

@@ -17,7 +17,7 @@ jobs:
         steps:
             # We create the status here and then update it to success/failure in the `report` stage
             # This provides an easy link to this workflow_run from the PR before Cypress is done.
-            - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               with:
                   authToken: ${{ secrets.GITHUB_TOKEN }}
                   state: pending
@@ -27,7 +27,7 @@ jobs:
 
             - name: "🩻 SonarCloud Scan"
               id: sonarcloud
-              uses: matrix-org/sonarcloud-workflow-action@v2.6
+              uses: matrix-org/sonarcloud-workflow-action@v2.7
               # workflow_run fails report against the develop commit always, we don't want that for PRs
               continue-on-error: ${{ github.event.workflow_run.head_branch != 'develop' }}
               with:
@@ -42,7 +42,7 @@ jobs:
                   coverage_extract_path: coverage
                   extra_args: ${{ inputs.extra_args }}
 
-            - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               if: always()
               with:
                   authToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/static_analysis.yml

@@ -15,7 +15,7 @@ jobs:
         steps:
             - uses: actions/checkout@v4
 
-            - uses: actions/setup-node@v3
+            - uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
@@ -41,7 +41,7 @@ jobs:
         steps:
             - uses: actions/checkout@v4
 
-            - uses: actions/setup-node@v3
+            - uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
@@ -51,13 +51,29 @@ jobs:
             - name: Run Linter
               run: "yarn run lint:js"
 
+    workflow_lint:
+        name: "Workflow Lint"
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+
+            - name: Install Deps
+              run: "yarn install --frozen-lockfile"
+
+            - name: Run Linter
+              run: "yarn lint:workflows"
+
     docs:
         name: "JSDoc Checker"
         runs-on: ubuntu-latest
         steps:
             - uses: actions/checkout@v4
 
-            - uses: actions/setup-node@v3
+            - uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 

.github/workflows/tests.yml

@@ -12,19 +12,20 @@ env:
     ENABLE_COVERAGE: ${{ github.event_name != 'merge_group' }}
 jobs:
     jest:
-        name: "Jest [${{ matrix.specs }}] (Node ${{ matrix.node }})"
+        name: "Jest [${{ matrix.specs }}] (Node ${{ matrix.node == '*' && 'latest' || matrix.node }})"
         runs-on: ubuntu-latest
         timeout-minutes: 10
         strategy:
             matrix:
                 specs: [integ, unit]
-                node: [18, latest]
+                node: [18, "lts/*", 21]
         steps:
             - name: Checkout code
               uses: actions/checkout@v4
 
             - name: Setup Node
-              uses: actions/setup-node@v3
+              id: setupNode
+              uses: actions/setup-node@v4
               with:
                   cache: "yarn"
                   node-version: ${{ matrix.node }}
@@ -51,7 +52,7 @@ jobs:
 
             - name: Move coverage files into place
               if: env.ENABLE_COVERAGE == 'true'
-              run: mv coverage/lcov.info coverage/${{ matrix.node }}-${{ matrix.specs }}.lcov.info
+              run: mv coverage/lcov.info coverage/${{ steps.setupNode.output.node-version }}-${{ matrix.specs }}.lcov.info
 
             - name: Upload Artifact
               if: env.ENABLE_COVERAGE == 'true'
@@ -81,7 +82,7 @@ jobs:
         steps:
             - name: Skip SonarCloud on merge queues
               if: env.ENABLE_COVERAGE == 'false'
-              uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+              uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               with:
                   authToken: ${{ secrets.GITHUB_TOKEN }}
                   state: success

.github/workflows/triage-labelled.yml

@@ -0,0 +1,11 @@
+name: Move labelled issues to correct projects
+
+on:
+    issues:
+        types: [labeled]
+
+jobs:
+    call-triage-labelled:
+        uses: vector-im/element-web/.github/workflows/triage-labelled.yml@develop
+        secrets:
+            ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}

.github/workflows/upgrade_dependencies.yml

@@ -1,38 +0,0 @@
-name: Upgrade Dependencies
-on:
-    workflow_dispatch: {}
-    workflow_call:
-        secrets:
-            ELEMENT_BOT_TOKEN:
-                required: true
-jobs:
-    upgrade:
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v4
-
-            - uses: actions/setup-node@v3
-              with:
-                  cache: "yarn"
-
-            - name: Upgrade
-              run: yarn upgrade && yarn install
-
-            - name: Create Pull Request
-              id: cpr
-              uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 # v5
-              with:
-                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
-                  branch: actions/upgrade-deps
-                  delete-branch: true
-                  title: Upgrade dependencies
-                  labels: |
-                      Dependencies
-                      T-Task
-
-            - name: Enable automerge
-              run: gh pr merge --merge --auto "$PR_NUMBER"
-              if: steps.cpr.outputs.pull-request-operation == 'created'
-              env:
-                  GH_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
-                  PR_NUMBER: ${{ steps.cpr.outputs.pull-request-number }}

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname "$0")/_/husky.sh"
+
+npx lint-staged

.lintstagedrc

@@ -0,0 +1,4 @@
+{
+    "*.(ts|tsx)": ["eslint --fix", "prettier --write"],
+    "*.(py|md|yaml)": ["prettier --write"]
+}

CHANGELOG.md

@@ -1,3 +1,59 @@
+Changes in [30.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.2.0) (2023-12-05)
+==================================================================================================
+## ✨ Features
+
+* Only await key query after lazy members resolved ([#3902](https://github.com/matrix-org/matrix-js-sdk/pull/3902)). Contributed by @BillCarsonFr.
+
+## 🐛 Bug Fixes
+
+* Rewrite receipt-handling code ([#3901](https://github.com/matrix-org/matrix-js-sdk/pull/3901)). Contributed by @andybalaam.
+* Explicitly free some Rust-side objects ([#3911](https://github.com/matrix-org/matrix-js-sdk/pull/3911)). Contributed by @richvdh.
+* Fix type for TimestampToEventResponse.origin\_server\_ts ([#3906](https://github.com/matrix-org/matrix-js-sdk/pull/3906)). Contributed by @Half-Shot.
+
+
+Changes in [30.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.1.0) (2023-11-21)
+==================================================================================================
+## ✨ Features
+
+* Rotate per-participant keys when a member leaves ([#3833](https://github.com/matrix-org/matrix-js-sdk/pull/3833)). Contributed by @dbkr.
+* Add E2EE for embedded mode of Element Call ([#3667](https://github.com/matrix-org/matrix-js-sdk/pull/3667)). Contributed by @SimonBrandner.
+
+## 🐛 Bug Fixes
+
+* Shorten TimelineWindow when an event is removed ([#3862](https://github.com/matrix-org/matrix-js-sdk/pull/3862)). Contributed by @andybalaam.
+* Ignore receipts pointing at missing or invalid events ([#3817](https://github.com/matrix-org/matrix-js-sdk/pull/3817)). Contributed by @andybalaam.
+* Fix members being loaded from server on initial sync (defeating lazy loading) ([#3830](https://github.com/matrix-org/matrix-js-sdk/pull/3830)). Contributed by @BillCarsonFr.
+
+
+Changes in [30.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.0.1) (2023-11-13)
+==================================================================================================
+
+## 🐛 Bug Fixes
+ * Ensure `setUserCreator` is called when a store is assigned ([\#3867](https://github.com/matrix-org/matrix-js-sdk/pull/3867)). Fixes vector-im/element-web#26520. Contributed by @MidhunSureshR.
+
+Changes in [30.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.0.0) (2023-11-07)
+==================================================================================================
+
+## 🚨 BREAKING CHANGES
+ * Refactor & make base64 functions browser-safe ([\#3818](https://github.com/matrix-org/matrix-js-sdk/pull/3818)).
+
+## 🦖 Deprecations
+ * Deprecate `MatrixEvent.toJSON` ([\#3801](https://github.com/matrix-org/matrix-js-sdk/pull/3801)).
+
+## ✨ Features
+ * Element-R: Add the git sha of the binding crate to `CryptoApi#getVersion` ([\#3838](https://github.com/matrix-org/matrix-js-sdk/pull/3838)). Contributed by @florianduros.
+ * Element-R: Wire up `globalBlacklistUnverifiedDevices` field to rust crypto encryption settings ([\#3790](https://github.com/matrix-org/matrix-js-sdk/pull/3790)). Fixes vector-im/element-web#26315. Contributed by @florianduros.
+ * Element-R: Wire up room rotation ([\#3807](https://github.com/matrix-org/matrix-js-sdk/pull/3807)). Fixes vector-im/element-web#26318. Contributed by @florianduros.
+ * Element-R: Add current version of the rust-sdk and vodozemac ([\#3825](https://github.com/matrix-org/matrix-js-sdk/pull/3825)). Contributed by @florianduros.
+ * Element-R: Wire up room history visibility ([\#3805](https://github.com/matrix-org/matrix-js-sdk/pull/3805)). Fixes vector-im/element-web#26319. Contributed by @florianduros.
+ * Element-R: log when we send to-device messages ([\#3810](https://github.com/matrix-org/matrix-js-sdk/pull/3810)).
+
+## 🐛 Bug Fixes
+ * Fix reemitter not being correctly wired on user objects created in storage classes ([\#3796](https://github.com/matrix-org/matrix-js-sdk/pull/3796)). Contributed by @MidhunSureshR.
+ * Element-R: silence log errors when viewing a pending event ([\#3824](https://github.com/matrix-org/matrix-js-sdk/pull/3824)).
+ * Don't emit a closed event if the indexeddb is closed by Element ([\#3832](https://github.com/matrix-org/matrix-js-sdk/pull/3832)). Fixes vector-im/element-web#25941. Contributed by @dhenneke.
+ * Element-R: silence log errors when viewing a decryption failure ([\#3821](https://github.com/matrix-org/matrix-js-sdk/pull/3821)).
+
 Changes in [29.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v29.1.0) (2023-10-24)
 ==================================================================================================
 

docs/SUMMARY.md

@@ -4,5 +4,6 @@
 
 # Deep dive
 
+-   [Release Process](release.md)
 -   [Storage notes](storage-notes.md)
 -   [Unverified devices](warning-on-unverified-devices.md)

docs/release.md

@@ -0,0 +1,24 @@
+# Release Process
+
+## Hotfix and off-cycle releases
+
+1. Prepare the `staging` branch by using the backport automation and manually merging
+2. Go to [Releasing](#Releasing)
+
+## Release candidates
+
+1. Prepare the `staging` branch by running the [branch cut automation](https://github.com/vector-im/element-web/actions/workflows/release_prepare.yml)
+2. Go to [Releasing](#Releasing)
+
+## Releasing
+
+1. Open the [Releases page](https://github.com/matrix-org/matrix-js-sdk/releases) and inspect the draft release there
+2. Make any modifications to the release notes and tag/version as required
+3. Run [workflow](https://github.com/matrix-org/matrix-js-sdk/actions/workflows/release.yml) with the type set appropriately
+
+## Artifacts
+
+Releasing the Matrix JS SDK has just two artifacts:
+
+-   Package published to [npm](https://github.com/matrix-org/matrix-js-sdk)
+-   Docs published to [Github Pages](https://matrix-org.github.io/matrix-js-sdk/)

package.json

@@ -1,6 +1,6 @@
 {
     "name": "matrix-js-sdk",
-    "version": "29.1.0",
+    "version": "30.3.0-rc.0",
     "description": "Matrix Client-Server SDK for Javascript",
     "engines": {
         "node": ">=18.0.0"
@@ -14,10 +14,11 @@
         "build:types": "tsc -p tsconfig-build.json --emitDeclarationOnly",
         "build:compile": "babel -d lib --verbose --extensions \".ts,.js\" src",
         "gendoc": "typedoc",
-        "lint": "yarn lint:types && yarn lint:js",
+        "lint": "yarn lint:types && yarn lint:js && yarn lint:workflows",
         "lint:js": "eslint --max-warnings 0 src spec && prettier --check .",
         "lint:js-fix": "prettier --loglevel=warn --write . && eslint --fix src spec",
         "lint:types": "tsc --noEmit",
+        "lint:workflows": "find .github/workflows -type f \\( -iname '*.yaml' -o -iname '*.yml' \\) | xargs -I {} sh -c 'echo \"Linting {}\"; action-validator \"{}\"'",
         "test": "jest",
         "test:watch": "jest --watch",
         "coverage": "yarn test --coverage"
@@ -51,7 +52,7 @@
     ],
     "dependencies": {
         "@babel/runtime": "^7.12.5",
-        "@matrix-org/matrix-sdk-crypto-wasm": "^2.0.0",
+        "@matrix-org/matrix-sdk-crypto-wasm": "^3.4.0",
         "another-json": "^0.2.0",
         "bs58": "^5.0.0",
         "content-type": "^1.0.4",
@@ -66,6 +67,8 @@
         "uuid": "9"
     },
     "devDependencies": {
+        "@action-validator/cli": "^0.5.3",
+        "@action-validator/core": "^0.5.3",
         "@babel/cli": "^7.12.10",
         "@babel/core": "^7.12.10",
         "@babel/eslint-parser": "^7.12.10",
@@ -78,8 +81,8 @@
         "@babel/preset-env": "^7.12.11",
         "@babel/preset-typescript": "^7.12.7",
         "@babel/register": "^7.12.10",
-        "@casualbot/jest-sonar-reporter": "^2.2.5",
-        "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz",
+        "@casualbot/jest-sonar-reporter": "2.2.7",
+        "@matrix-org/olm": "3.2.15",
         "@types/bs58": "^4.0.1",
         "@types/content-type": "^1.1.5",
         "@types/debug": "^4.1.7",
@@ -92,10 +95,9 @@
         "@typescript-eslint/parser": "^5.45.0",
         "allchange": "^1.0.6",
         "babel-jest": "^29.0.0",
-        "babelify": "^10.0.0",
         "debug": "^4.3.4",
         "domexception": "^4.0.0",
-        "eslint": "8.49.0",
+        "eslint": "8.54.0",
         "eslint-config-google": "^0.14.0",
         "eslint-config-prettier": "^9.0.0",
         "eslint-import-resolver-typescript": "^3.5.1",
@@ -104,20 +106,20 @@
         "eslint-plugin-jsdoc": "^46.0.0",
         "eslint-plugin-matrix-org": "^1.0.0",
         "eslint-plugin-tsdoc": "^0.2.17",
-        "eslint-plugin-unicorn": "^48.0.0",
-        "exorcist": "^2.0.0",
-        "fake-indexeddb": "^4.0.0",
+        "eslint-plugin-unicorn": "^49.0.0",
+        "fake-indexeddb": "^5.0.0",
+        "fetch-mock": "9.11.0",
         "fetch-mock-jest": "^1.5.1",
+        "husky": "^8.0.3",
         "jest": "^29.0.0",
         "jest-environment-jsdom": "^29.0.0",
         "jest-localstorage-mock": "^2.4.6",
         "jest-mock": "^29.0.0",
+        "lint-staged": "^15.0.2",
         "matrix-mock-request": "^2.5.0",
         "prettier": "2.8.8",
         "rimraf": "^5.0.0",
-        "terser": "^5.5.1",
         "ts-node": "^10.9.1",
-        "tsify": "^5.0.2",
         "typedoc": "^0.24.0",
         "typedoc-plugin-coverage": "^2.1.0",
         "typedoc-plugin-mdn-links": "^3.0.3",

post-release.sh

@@ -10,28 +10,6 @@ set -e
 jq --version > /dev/null || (echo "jq is required: please install it"; kill $$)
 
 if [ "$(git branch -lr | grep origin/develop -c)" -ge 1 ]; then
-    # When merging to develop, we need revert the `main` and `typings` fields if we adjusted them previously.
-    for i in main typings browser
-    do
-        # If a `lib` prefixed value is present, it means we adjusted the field
-        # earlier at publish time, so we should revert it now.
-        if [ "$(jq -r ".matrix_lib_$i" package.json)" != "null" ]; then
-            # If there's a `src` prefixed value, use that, otherwise delete.
-            # This is used to delete the `typings` field and reset `main` back
-            # to the TypeScript source.
-            src_value=$(jq -r ".matrix_src_$i" package.json)
-            if [ "$src_value" != "null" ]; then
-                jq ".$i = .matrix_src_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
-            else
-                jq "del(.$i)" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
-            fi
-        fi
-    done
-
-    if [ -n "$(git ls-files --modified package.json)" ]; then
-        echo "Committing develop package.json"
-        git commit package.json -m "Resetting package fields for development"
-    fi
-
+    "$(dirname "$0")/scripts/release/post-merge-master.sh"
     git push origin develop
 fi

release.sh

@@ -175,18 +175,7 @@ echo "yarn version"
 # manually commit the result.
 yarn version --no-git-tag-version --new-version "$release"
 
-# For the published and dist versions of the package, we copy the
-# `matrix_lib_main` and `matrix_lib_typings` fields to `main` and `typings` (if
-# they exist). This small bit of gymnastics allows us to use the TypeScript
-# source directly for development without needing to build before linting or
-# testing.
-for i in main typings browser
-do
-    lib_value=$(jq -r ".matrix_lib_$i" package.json)
-    if [ "$lib_value" != "null" ]; then
-        jq ".$i = .matrix_lib_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
-    fi
-done
+"$(dirname "$0")/scripts/release/pre-release.sh"
 
 # commit yarn.lock if it exists, is versioned, and is modified
 if [[ -f yarn.lock && $(git status --porcelain yarn.lock | grep '^ M') ]];

scripts/release/merge-release-notes.js

@@ -0,0 +1,104 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+
+async function getRelease(github, dependency) {
+    let owner;
+    let repo;
+    let tag;
+    if (dependency.includes("/") && dependency.includes("@")) {
+        owner = dependency.split("/")[0];
+        repo = dependency.split("/")[1].split("@")[0];
+        tag = dependency.split("@")[1];
+    } else {
+        const upstreamPackageJson = JSON.parse(fs.readFileSync(`./node_modules/${dependency}/package.json`, "utf8"));
+        [owner, repo] = upstreamPackageJson.repository.url.split("/").slice(-2);
+        tag = `v${upstreamPackageJson.version}`;
+    }
+
+    const response = await github.rest.repos.getReleaseByTag({
+        owner,
+        repo,
+        tag,
+    });
+    return response.data;
+}
+
+const HEADING_PREFIX = "## ";
+
+const main = async ({ github, releaseId, dependencies }) => {
+    const { GITHUB_REPOSITORY } = process.env;
+    const [owner, repo] = GITHUB_REPOSITORY.split("/");
+
+    const sections = new Map();
+    let heading = null;
+    for (const dependency of dependencies) {
+        const release = await getRelease(github, dependency);
+        for (const line of release.body.split("\n")) {
+            if (line.startsWith(HEADING_PREFIX)) {
+                heading = line.trim();
+                sections.set(heading, []);
+                continue;
+            }
+            if (heading && line) {
+                sections.get(heading).push(line.trim());
+            }
+        }
+    }
+
+    const { data: release } = await github.rest.repos.getRelease({
+        owner,
+        repo,
+        release_id: releaseId,
+    });
+
+    const headings = ["🚨 BREAKING CHANGES", "🦖 Deprecations", "✨ Features", "🐛 Bug Fixes", "🧰 Maintenance"].map(
+        (h) => HEADING_PREFIX + h,
+    );
+
+    heading = null;
+    const output = [];
+    for (const line of [...release.body.split("\n"), null]) {
+        if (line === null || line.startsWith(HEADING_PREFIX)) {
+            // If we have a heading, and it's not the first in the list of pending headings, output the section.
+            // If we're processing the last line (null) then output all remaining sections.
+            while (headings.length > 0 && (line === null || (heading && headings[0] !== heading))) {
+                const heading = headings.shift();
+                if (sections.has(heading)) {
+                    output.push(heading);
+                    output.push(...sections.get(heading));
+                }
+            }
+
+            if (heading && sections.has(heading)) {
+                const lastIsBlank = !output.at(-1)?.trim();
+                if (lastIsBlank) output.pop();
+                output.push(...sections.get(heading));
+                if (lastIsBlank) output.push("");
+            }
+            heading = line;
+        }
+        output.push(line);
+    }
+
+    return output.join("\n");
+};
+
+// This is just for testing locally
+// Needs environment variables GITHUB_TOKEN & GITHUB_REPOSITORY
+if (require.main === module) {
+    const { Octokit } = require("@octokit/rest");
+    const github = new Octokit({ auth: process.env.GITHUB_TOKEN });
+    if (process.argv.length < 4) {
+        // eslint-disable-next-line no-console
+        console.error("Usage: node merge-release-notes.js owner/repo:release_id npm-package-name ...");
+        process.exit(1);
+    }
+    const [releaseId, ...dependencies] = process.argv.slice(2);
+    main({ github, releaseId, dependencies }).then((output) => {
+        // eslint-disable-next-line no-console
+        console.log(output);
+    });
+}
+
+module.exports = main;

scripts/release/post-merge-master.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# When merging to develop, we need revert the `main` and `typings` fields if we adjusted them previously.
+for i in main typings browser
+do
+    # If a `lib` prefixed value is present, it means we adjusted the field earlier at publish time, so we should revert it now.
+    if [ "$(jq -r ".matrix_lib_$i" package.json)" != "null" ]; then
+        # If there's a `src` prefixed value, use that, otherwise delete.
+        # This is used to delete the `typings` field and reset `main` back to the TypeScript source.
+        src_value=$(jq -r ".matrix_src_$i" package.json)
+        if [ "$src_value" != "null" ]; then
+            jq ".$i = .matrix_src_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
+        else
+            jq "del(.$i)" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
+        fi
+    fi
+done
+
+if [ -n "$(git ls-files --modified package.json)" ]; then
+    echo "Committing develop package.json"
+    git commit package.json -m "Resetting package fields for development"
+fi

scripts/release/pre-release.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# For the published and dist versions of the package,
+# we copy the `matrix_lib_main` and `matrix_lib_typings` fields to `main` and `typings` (if they exist).
+# This small bit of gymnastics allows us to use the TypeScript source directly for development without
+# needing to build before linting or testing.
+
+for i in main typings browser
+do
+    lib_value=$(jq -r ".matrix_lib_$i" package.json)
+    if [ "$lib_value" != "null" ]; then
+        jq ".$i = .matrix_lib_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
+    fi
+done

spec/integ/crypto/cross-signing.spec.ts

@@ -31,9 +31,12 @@ import {
     SELF_CROSS_SIGNING_PRIVATE_KEY_BASE64,
     SELF_CROSS_SIGNING_PUBLIC_KEY_BASE64,
     SIGNED_CROSS_SIGNING_KEYS_DATA,
+    SIGNED_TEST_DEVICE_DATA,
     USER_CROSS_SIGNING_PRIVATE_KEY_BASE64,
 } from "../../test-utils/test-data";
+import * as testData from "../../test-utils/test-data";
 import { E2EKeyResponder } from "../../test-utils/E2EKeyResponder";
+import { AccountDataAccumulator } from "../../test-utils/AccountDataAccumulator";
 
 afterEach(() => {
     // reset fake-indexeddb after each test, to make sure we don't leak connections
@@ -97,6 +100,12 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
         /** an object which intercepts `/keys/upload` requests on the test homeserver */
         new E2EKeyReceiver(homeserverUrl);
 
+        // Silence warnings from the backup manager
+        fetchMock.getOnce(new URL("/_matrix/client/v3/room_keys/version", homeserverUrl).toString(), {
+            status: 404,
+            body: { errcode: "M_NOT_FOUND" },
+        });
+
         await initCrypto(aliceClient);
     });
 
@@ -236,6 +245,53 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
                 `[${TEST_USER_ID}].[${TEST_DEVICE_ID}].signatures.[${TEST_USER_ID}].[ed25519:${SELF_CROSS_SIGNING_PUBLIC_KEY_BASE64}]`,
             );
         });
+
+        it("can bootstrapCrossSigning twice", async () => {
+            mockSetupCrossSigningRequests();
+
+            const authDict = { type: "test" };
+            await bootstrapCrossSigning(authDict);
+
+            // a second call should do nothing except GET requests
+            fetchMock.mockClear();
+            await bootstrapCrossSigning(authDict);
+            const calls = fetchMock.calls((url, opts) => opts.method != "GET");
+            expect(calls.length).toEqual(0);
+        });
+
+        newBackendOnly("will upload existing cross-signing keys to an established secret storage", async () => {
+            // This rather obscure codepath covers the case that:
+            //   - 4S is set up and working
+            //   - our device has private cross-signing keys, but has not published them to 4S
+            //
+            // To arrange that, we call `bootstrapCrossSigning` on our main device, and then (pretend to) set up 4S from
+            // a *different* device. Then, when we call `bootstrapCrossSigning` again, it should do the honours.
+
+            mockSetupCrossSigningRequests();
+            const accountDataAccumulator = new AccountDataAccumulator();
+            accountDataAccumulator.interceptGetAccountData();
+
+            const authDict = { type: "test" };
+            await bootstrapCrossSigning(authDict);
+
+            // Pretend that another device has uploaded a 4S key
+            accountDataAccumulator.accountDataEvents.set("m.secret_storage.default_key", { key: "key_id" });
+            accountDataAccumulator.accountDataEvents.set("m.secret_storage.key.key_id", {
+                key: "keykeykey",
+                algorithm: SECRET_STORAGE_ALGORITHM_V1_AES,
+            });
+
+            // Prepare for the cross-signing keys
+            const p = accountDataAccumulator.interceptSetAccountData(":type(m.cross_signing..*)");
+
+            await bootstrapCrossSigning(authDict);
+            await p;
+
+            // The cross-signing keys should have been uploaded
+            expect(accountDataAccumulator.accountDataEvents.has("m.cross_signing.master")).toBeTruthy();
+            expect(accountDataAccumulator.accountDataEvents.has("m.cross_signing.self_signing")).toBeTruthy();
+            expect(accountDataAccumulator.accountDataEvents.has("m.cross_signing.user_signing")).toBeTruthy();
+        });
     });
 
     describe("getCrossSigningStatus()", () => {
@@ -339,4 +395,48 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
             expect(userSigningKeyId).toBe(getPubKey(crossSigningKeys.user_signing_key));
         });
     });
+
+    describe("crossSignDevice", () => {
+        beforeEach(async () => {
+            jest.useFakeTimers();
+
+            // make sure that there is another device which we can sign
+            e2eKeyResponder.addDeviceKeys(SIGNED_TEST_DEVICE_DATA);
+
+            // Complete initialsync, to get the outgoing requests going
+            mockInitialApiRequests(aliceClient.getHomeserverUrl());
+            syncResponder.sendOrQueueSyncResponse({ next_batch: 1 });
+            await aliceClient.startClient();
+            await syncPromise(aliceClient);
+
+            // Wait for legacy crypto to find the device
+            await jest.advanceTimersByTimeAsync(10);
+
+            const devices = await aliceClient.getCrypto()!.getUserDeviceInfo([aliceClient.getSafeUserId()]);
+            expect(devices.get(aliceClient.getSafeUserId())!.has(testData.TEST_DEVICE_ID)).toBeTruthy();
+        });
+
+        afterEach(async () => {
+            jest.useRealTimers();
+        });
+
+        it("fails for an unknown device", async () => {
+            await expect(aliceClient.getCrypto()!.crossSignDevice("unknown")).rejects.toThrow("Unknown device");
+        });
+
+        it("cross-signs the device", async () => {
+            mockSetupCrossSigningRequests();
+            await aliceClient.getCrypto()!.bootstrapCrossSigning({});
+
+            fetchMock.mockClear();
+            await aliceClient.getCrypto()!.crossSignDevice(testData.TEST_DEVICE_ID);
+
+            // check that a sig for the device was uploaded
+            const calls = fetchMock.calls("upload-sigs");
+            expect(calls.length).toEqual(1);
+            const body = JSON.parse(calls[0][1]!.body as string);
+            const deviceSig = body[aliceClient.getSafeUserId()][testData.TEST_DEVICE_ID];
+            expect(deviceSig).toHaveProperty("signatures");
+        });
+    });
 });

spec/integ/crypto/crypto.spec.ts

@@ -23,7 +23,14 @@ import { MockResponse, MockResponseFunction } from "fetch-mock";
 import Olm from "@matrix-org/olm";
 
 import * as testUtils from "../../test-utils/test-utils";
-import { CRYPTO_BACKENDS, getSyncResponse, InitCrypto, syncPromise } from "../../test-utils/test-utils";
+import {
+    advanceTimersUntil,
+    CRYPTO_BACKENDS,
+    getSyncResponse,
+    InitCrypto,
+    mkEventCustom,
+    syncPromise,
+} from "../../test-utils/test-utils";
 import * as testData from "../../test-utils/test-data";
 import {
     BOB_SIGNED_CROSS_SIGNING_KEYS_DATA,
@@ -37,6 +44,7 @@ import {
 import { TestClient } from "../../TestClient";
 import { logger } from "../../../src/logger";
 import {
+    Category,
     ClientEvent,
     createClient,
     CryptoEvent,
@@ -53,11 +61,12 @@ import {
     Room,
     RoomMember,
     RoomStateEvent,
+    HistoryVisibility,
 } from "../../../src/matrix";
 import { DeviceInfo } from "../../../src/crypto/deviceinfo";
 import { E2EKeyReceiver } from "../../test-utils/E2EKeyReceiver";
 import { ISyncResponder, SyncResponder } from "../../test-utils/SyncResponder";
-import { escapeRegExp } from "../../../src/utils";
+import { defer, escapeRegExp } from "../../../src/utils";
 import { downloadDeviceToJsDevice } from "../../../src/rust-crypto/device-converter";
 import { flushPromises } from "../../test-utils/flushPromises";
 import {
@@ -66,13 +75,19 @@ import {
     mockSetupMegolmBackupRequests,
 } from "../../test-utils/mockEndpoints";
 import { AddSecretStorageKeyOpts } from "../../../src/secret-storage";
-import { CrossSigningKey, CryptoCallbacks, KeyBackupInfo } from "../../../src/crypto-api";
+import {
+    CrossSigningKey,
+    CryptoCallbacks,
+    EventShieldColour,
+    EventShieldReason,
+    KeyBackupInfo,
+} from "../../../src/crypto-api";
 import { E2EKeyResponder } from "../../test-utils/E2EKeyResponder";
 import { DecryptionError } from "../../../src/crypto/algorithms";
 import { IKeyBackup } from "../../../src/crypto/backup";
 import {
-    createOlmSession,
     createOlmAccount,
+    createOlmSession,
     encryptGroupSessionKey,
     encryptMegolmEvent,
     encryptMegolmEventRawPlainText,
@@ -80,6 +95,8 @@ import {
     establishOlmSession,
     getTestOlmAccountKeys,
 } from "./olm-utils";
+import { ToDevicePayload } from "../../../src/models/ToDeviceMessage";
+import { AccountDataAccumulator } from "../../test-utils/AccountDataAccumulator";
 
 afterEach(() => {
     // reset fake-indexeddb after each test, to make sure we don't leak connections
@@ -146,6 +163,27 @@ async function expectSendRoomKey(
     });
 }
 
+/**
+ * Return the event received on rooms/{roomId}/send/m.room.encrypted endpoint.
+ * See https://spec.matrix.org/latest/client-server-api/#put_matrixclientv3roomsroomidsendeventtypetxnid
+ * @returns the content of the encrypted event
+ */
+function expectEncryptedSendMessage() {
+    return new Promise<IContent>((resolve) => {
+        fetchMock.putOnce(
+            new RegExp("/send/m.room.encrypted/"),
+            (url, request) => {
+                const content = JSON.parse(request.body as string);
+                resolve(content);
+                return { event_id: "$event_id" };
+            },
+            // append to the list of intercepts on this path (since we have some tests that call
+            // this function multiple times)
+            { overwriteRoutes: false },
+        );
+    });
+}
+
 /**
  * Expect that the client sends an encrypted event
  *
@@ -159,22 +197,7 @@ async function expectSendRoomKey(
 async function expectSendMegolmMessage(
     inboundGroupSessionPromise: Promise<Olm.InboundGroupSession>,
 ): Promise<Partial<IEvent>> {
-    const encryptedMessageContent = await new Promise<IContent>((resolve) => {
-        fetchMock.putOnce(
-            new RegExp("/send/m.room.encrypted/"),
-            (url: string, opts: RequestInit): MockResponse => {
-                resolve(JSON.parse(opts.body as string));
-                return {
-                    event_id: "$event_id",
-                };
-            },
-            {
-                // append to the list of intercepts on this path (since we have some tests that call
-                // this function multiple times)
-                overwriteRoutes: false,
-            },
-        );
-    });
+    const encryptedMessageContent = await expectEncryptedSendMessage();
 
     // In some of the tests, the room key is sent *after* the actual event, so we may need to wait for it now.
     const inboundGroupSession = await inboundGroupSessionPromise;
@@ -375,6 +398,19 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
         expect(aliceClient.getCrypto()).toHaveProperty("globalBlacklistUnverifiedDevices");
     });
 
+    it("CryptoAPI.getOwnedDeviceKeys returns the correct values", async () => {
+        const homeserverUrl = aliceClient.getHomeserverUrl();
+
+        keyResponder = new E2EKeyResponder(homeserverUrl);
+        await startClientAndAwaitFirstSync();
+        keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
+
+        const deviceKeys = await aliceClient.getCrypto()!.getOwnDeviceKeys();
+
+        expect(deviceKeys.curve25519).toEqual(keyReceiver.getDeviceKey());
+        expect(deviceKeys.ed25519).toEqual(keyReceiver.getSigningKey());
+    });
+
     it("Alice receives a megolm message", async () => {
         expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
         await startClientAndAwaitFirstSync();
@@ -670,7 +706,13 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
     });
 
     it("prepareToEncrypt", async () => {
-        expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
+        const homeserverUrl = aliceClient.getHomeserverUrl();
+        keyResponder = new E2EKeyResponder(homeserverUrl);
+        keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
+
+        const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
+        keyResponder.addDeviceKeys(testDeviceKeys);
+
         await startClientAndAwaitFirstSync();
         aliceClient.setGlobalErrorOnUnknownDevices(false);
 
@@ -678,10 +720,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
         syncResponder.sendOrQueueSyncResponse(getSyncResponse(["@bob:xyz"]));
         await syncPromise(aliceClient);
 
-        // we expect alice first to query bob's keys...
-        expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-
-        // ... and then claim one of his OTKs
+        // Alice should claim one of Bob's OTKs
         expectAliceKeyClaim(getTestKeysClaimResponse("@bob:xyz"));
 
         // fire off the prepare request
@@ -698,18 +737,20 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
     it("Alice sends a megolm message with GlobalErrorOnUnknownDevices=false", async () => {
         aliceClient.setGlobalErrorOnUnknownDevices(false);
-        expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
+        const homeserverUrl = aliceClient.getHomeserverUrl();
+        keyResponder = new E2EKeyResponder(homeserverUrl);
+        keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
+
+        const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
+        keyResponder.addDeviceKeys(testDeviceKeys);
+
         await startClientAndAwaitFirstSync();
 
         // Alice shares a room with Bob
         syncResponder.sendOrQueueSyncResponse(getSyncResponse(["@bob:xyz"]));
         await syncPromise(aliceClient);
 
-        // Once we send the message, Alice will check Bob's device list (twice, because reasons) ...
-        expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-        expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-
-        // ... and claim one of his OTKs ...
+        // ... and claim one of Bob's OTKs ...
         expectAliceKeyClaim(getTestKeysClaimResponse("@bob:xyz"));
 
         // ... and send an m.room_key message
@@ -724,18 +765,20 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
     it("We should start a new megolm session after forceDiscardSession", async () => {
         aliceClient.setGlobalErrorOnUnknownDevices(false);
-        expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
+        const homeserverUrl = aliceClient.getHomeserverUrl();
+        keyResponder = new E2EKeyResponder(homeserverUrl);
+        keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
+
+        const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
+        keyResponder.addDeviceKeys(testDeviceKeys);
+
         await startClientAndAwaitFirstSync();
 
         // Alice shares a room with Bob
         syncResponder.sendOrQueueSyncResponse(getSyncResponse(["@bob:xyz"]));
         await syncPromise(aliceClient);
 
-        // Once we send the message, Alice will check Bob's device list (twice, because reasons) ...
-        expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-        expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-
-        // ... and claim one of his OTKs ...
+        // ... and claim one of Bob's OTKs ...
         expectAliceKeyClaim(getTestKeysClaimResponse("@bob:xyz"));
 
         // ... and send an m.room_key message
@@ -922,6 +965,210 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 aliceClient.sendTextMessage(ROOM_ID, "test"),
             ]);
         });
+
+        it("should send a m.unverified code in toDevice messages to an unverified device when globalBlacklistUnverifiedDevices=true", async () => {
+            aliceClient.getCrypto()!.globalBlacklistUnverifiedDevices = true;
+
+            expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
+            await startClientAndAwaitFirstSync();
+            await establishOlmSession(aliceClient, keyReceiver, syncResponder, testOlmAccount);
+
+            // Tell alice we share a room with bob
+            syncResponder.sendOrQueueSyncResponse(getSyncResponse(["@bob:xyz"]));
+            await syncPromise(aliceClient);
+
+            // Force alice to download bob keys
+            expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
+
+            // Wait to receive the toDevice message and return bob device content
+            const toDevicePromise = new Promise<ToDevicePayload>((resolve) => {
+                fetchMock.putOnce(new RegExp("/sendToDevice/m.room_key.withheld/"), (url, request) => {
+                    const content = JSON.parse(request.body as string);
+                    resolve(content.messages["@bob:xyz"]["DEVICE_ID"]);
+                    return {};
+                });
+            });
+
+            // Mock endpoint of message sending
+            fetchMock.put(new RegExp("/send/"), { event_id: "$event_id" });
+
+            await aliceClient.sendTextMessage(ROOM_ID, "test");
+
+            // Finally, check that the toDevice message has the m.unverified code
+            const toDeviceContent = await toDevicePromise;
+            expect(toDeviceContent.code).toBe("m.unverified");
+        });
+    });
+
+    describe("Session should rotate according to encryption settings", () => {
+        /**
+         * Send a message to bob and get the encrypted message
+         * @returns {Promise<IContent>} The encrypted message
+         */
+        async function sendEncryptedMessage(): Promise<IContent> {
+            const [encryptedMessage] = await Promise.all([
+                expectEncryptedSendMessage(),
+                aliceClient.sendTextMessage(ROOM_ID, "test"),
+            ]);
+            return encryptedMessage;
+        }
+
+        afterEach(() => {
+            jest.useRealTimers();
+        });
+
+        newBackendOnly("should rotate the session after 2 messages", async () => {
+            expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
+            await startClientAndAwaitFirstSync();
+            const p2pSession = await establishOlmSession(aliceClient, keyReceiver, syncResponder, testOlmAccount);
+
+            const syncResponse = getSyncResponse(["@bob:xyz"]);
+            // Every 2 messages in the room, the session should be rotated
+            syncResponse.rooms[Category.Join][ROOM_ID].state.events[0].content = {
+                algorithm: "m.megolm.v1.aes-sha2",
+                rotation_period_msgs: 2,
+            };
+
+            // Tell alice we share a room with bob
+            syncResponder.sendOrQueueSyncResponse(syncResponse);
+            await syncPromise(aliceClient);
+
+            // Force alice to download bob keys
+            expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
+
+            // Send a message to bob and get the encrypted message
+            const [encryptedMessage] = await Promise.all([
+                sendEncryptedMessage(),
+                expectSendRoomKey("@bob:xyz", testOlmAccount, p2pSession),
+            ]);
+
+            // Check that the session id exists
+            const sessionId = encryptedMessage.session_id;
+            expect(sessionId).toBeDefined();
+
+            // Send a second message to bob and get the current message
+            const secondEncryptedMessage = await sendEncryptedMessage();
+
+            // Check that the same session id is shared between the two messages
+            const secondSessionId = secondEncryptedMessage.session_id;
+            expect(secondSessionId).toBe(sessionId);
+
+            // The session should be rotated, we are expecting the room key to be sent
+            const [thirdEncryptedMessage] = await Promise.all([
+                sendEncryptedMessage(),
+                expectSendRoomKey("@bob:xyz", testOlmAccount, p2pSession),
+            ]);
+
+            // The session is rotated every 2 messages, we should have a new session id
+            const thirdSessionId = thirdEncryptedMessage.session_id;
+            expect(thirdSessionId).not.toBe(sessionId);
+        });
+
+        newBackendOnly("should rotate the session after 1h", async () => {
+            expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
+            await startClientAndAwaitFirstSync();
+            const p2pSession = await establishOlmSession(aliceClient, keyReceiver, syncResponder, testOlmAccount);
+
+            // We need to fake the timers to advance the time
+            jest.useFakeTimers();
+
+            const syncResponse = getSyncResponse(["@bob:xyz"]);
+
+            // The minimum rotation period is 1h
+            // https://github.com/matrix-org/matrix-rust-sdk/blob/f75b2cd1d0981db42751dadb08c826740af1018e/crates/matrix-sdk-crypto/src/olm/group_sessions/outbound.rs#L410-L415
+            const oneHourInMs = 60 * 60 * 1000;
+
+            // Every 1h the session should be rotated
+            syncResponse.rooms[Category.Join][ROOM_ID].state.events[0].content = {
+                algorithm: "m.megolm.v1.aes-sha2",
+                rotation_period_ms: oneHourInMs,
+            };
+
+            // Tell alice we share a room with bob
+            syncResponder.sendOrQueueSyncResponse(syncResponse);
+            await syncPromise(aliceClient);
+
+            // Force alice to download bob keys
+            expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
+
+            // Send a message to bob and get the encrypted message
+            const [encryptedMessage] = await Promise.all([
+                sendEncryptedMessage(),
+                expectSendRoomKey("@bob:xyz", testOlmAccount, p2pSession),
+            ]);
+
+            // Check that the session id exists
+            const sessionId = encryptedMessage.session_id;
+            expect(sessionId).toBeDefined();
+
+            // Advance the time by 1h
+            jest.advanceTimersByTime(oneHourInMs);
+
+            // Send a second message to bob and get the encrypted message
+            const [secondEncryptedMessage] = await Promise.all([
+                sendEncryptedMessage(),
+                expectSendRoomKey("@bob:xyz", testOlmAccount, p2pSession),
+            ]);
+
+            // The session should be rotated
+            const secondSessionId = secondEncryptedMessage.session_id;
+            expect(secondSessionId).not.toBe(sessionId);
+        });
+    });
+
+    newBackendOnly("should rotate the session when the history visibility changes", async () => {
+        expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
+        await startClientAndAwaitFirstSync();
+        const p2pSession = await establishOlmSession(aliceClient, keyReceiver, syncResponder, testOlmAccount);
+
+        // Tell alice we share a room with bob
+        syncResponder.sendOrQueueSyncResponse(getSyncResponse(["@bob:xyz"]));
+        await syncPromise(aliceClient);
+
+        // Force alice to download bob keys
+        expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
+
+        // Send a message to bob and get the current session id
+        let [, , encryptedMessage] = await Promise.all([
+            aliceClient.sendTextMessage(ROOM_ID, "test"),
+            expectSendRoomKey("@bob:xyz", testOlmAccount, p2pSession),
+            expectEncryptedSendMessage(),
+        ]);
+
+        // Check that the session id exists
+        const sessionId = encryptedMessage.session_id;
+        expect(sessionId).toBeDefined();
+
+        // Change history visibility in sync response
+        const syncResponse = getSyncResponse([]);
+        syncResponse.rooms[Category.Join][ROOM_ID].timeline.events.push(
+            mkEventCustom({
+                sender: TEST_USER_ID,
+                type: "m.room.history_visibility",
+                state_key: "",
+                content: {
+                    history_visibility: HistoryVisibility.Invited,
+                },
+            }),
+        );
+
+        // Update the new visibility
+        syncResponder.sendOrQueueSyncResponse(syncResponse);
+        await syncPromise(aliceClient);
+
+        // Resend a message to bob and get the new session id
+        [, , encryptedMessage] = await Promise.all([
+            aliceClient.sendTextMessage(ROOM_ID, "test"),
+            expectSendRoomKey("@bob:xyz", testOlmAccount, p2pSession),
+            expectEncryptedSendMessage(),
+        ]);
+
+        // Check that the new session id exists
+        const newSessionId = encryptedMessage.session_id;
+        expect(newSessionId).toBeDefined();
+
+        // Check that the session id has changed
+        expect(sessionId).not.toEqual(newSessionId);
     });
 
     oldBackendOnly("We should start a new megolm session when a device is blocked", async () => {
@@ -1700,6 +1947,105 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
         await sendEventPromise;
     });
 
+    describe("getEncryptionInfoForEvent", () => {
+        it("handles outgoing events", async () => {
+            aliceClient.setGlobalErrorOnUnknownDevices(false);
+            expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
+            await startClientAndAwaitFirstSync();
+
+            // Alice shares a room with Bob
+            syncResponder.sendOrQueueSyncResponse(getSyncResponse(["@bob:xyz"]));
+            await syncPromise(aliceClient);
+
+            // Once we send the message, Alice will check Bob's device list (twice, because reasons) ...
+            expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
+            expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
+
+            // ... and claim one of his OTKs ...
+            expectAliceKeyClaim(getTestKeysClaimResponse("@bob:xyz"));
+
+            // ... and send an m.room_key message ...
+            const inboundGroupSessionPromise = expectSendRoomKey("@bob:xyz", testOlmAccount);
+
+            // ... and finally, send the room key. We block the response until `sendRoomMessageDefer` completes.
+            const sendRoomMessageDefer = defer<MockResponse>();
+            const reqProm = new Promise<IContent>((resolve) => {
+                fetchMock.putOnce(
+                    new RegExp("/send/m.room.encrypted/"),
+                    async (url: string, opts: RequestInit): Promise<MockResponse> => {
+                        resolve(JSON.parse(opts.body as string));
+                        return await sendRoomMessageDefer.promise;
+                    },
+                    {
+                        // append to the list of intercepts on this path (since we have some tests that call
+                        // this function multiple times)
+                        overwriteRoutes: false,
+                    },
+                );
+            });
+
+            // Now we start to send the message
+            const sendProm = aliceClient.sendTextMessage(testData.TEST_ROOM_ID, "test");
+
+            // and wait for the outgoing requests
+            const inboundGroupSession = await inboundGroupSessionPromise;
+            const encryptedMessageContent = await reqProm;
+            const msg: any = inboundGroupSession.decrypt(encryptedMessageContent!.ciphertext);
+            logger.log("Decrypted received megolm message", msg);
+
+            // at this point, the request to send the room message has been made, but not completed.
+            // get hold of the pending event, and see what getEncryptionInfoForEvent makes of it
+            const pending = aliceClient.getRoom(testData.TEST_ROOM_ID)!.getPendingEvents();
+            expect(pending.length).toEqual(1);
+            const encInfo = await aliceClient.getCrypto()!.getEncryptionInfoForEvent(pending[0]);
+            expect(encInfo!.shieldColour).toEqual(EventShieldColour.NONE);
+            expect(encInfo!.shieldReason).toBeNull();
+
+            // release the send request
+            const resp = { event_id: "$event_id" };
+            sendRoomMessageDefer.resolve(resp);
+            expect(await sendProm).toEqual(resp);
+
+            // still pending at this point
+            expect(aliceClient.getRoom(testData.TEST_ROOM_ID)!.getPendingEvents().length).toEqual(1);
+
+            // echo the event back
+            const fullEvent = {
+                event_id: "$event_id",
+                type: "m.room.encrypted",
+                sender: aliceClient.getUserId(),
+                origin_server_ts: Date.now(),
+                content: encryptedMessageContent,
+            };
+            syncResponder.sendOrQueueSyncResponse({
+                next_batch: 1,
+                rooms: { join: { [testData.TEST_ROOM_ID]: { timeline: { events: [fullEvent] } } } },
+            });
+            await syncPromise(aliceClient);
+
+            const timelineEvents = aliceClient.getRoom(testData.TEST_ROOM_ID)!.getLiveTimeline()!.getEvents();
+            const lastEvent = timelineEvents[timelineEvents.length - 1];
+            expect(lastEvent.getId()).toEqual("$event_id");
+
+            // now check getEncryptionInfoForEvent again
+            const encInfo2 = await aliceClient.getCrypto()!.getEncryptionInfoForEvent(lastEvent);
+            let expectedEncryptionInfo;
+            if (backend === "rust-sdk") {
+                // rust crypto does not trust its own device until it is cross-signed.
+                expectedEncryptionInfo = {
+                    shieldColour: EventShieldColour.RED,
+                    shieldReason: EventShieldReason.UNSIGNED_DEVICE,
+                };
+            } else {
+                expectedEncryptionInfo = {
+                    shieldColour: EventShieldColour.NONE,
+                    shieldReason: null,
+                };
+            }
+            expect(encInfo2).toEqual(expectedEncryptionInfo);
+        });
+    });
+
     describe("Lazy-loading member lists", () => {
         let p2pSession: Olm.Session;
 
@@ -1727,13 +2073,17 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             });
         }
 
-        oldBackendOnly("Sending an event initiates a member list sync", async () => {
+        it("Sending an event initiates a member list sync", async () => {
+            const homeserverUrl = aliceClient.getHomeserverUrl();
+            keyResponder = new E2EKeyResponder(homeserverUrl);
+            keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
+
+            const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
+            keyResponder.addDeviceKeys(testDeviceKeys);
+
             // we expect a call to the /members list...
             const memberListPromise = expectMembershipRequest(ROOM_ID, ["@bob:xyz"]);
 
-            // then a request for bob's devices...
-            expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-
             // then a to-device with the room_key
             const inboundGroupSessionPromise = expectSendRoomKey("@bob:xyz", testOlmAccount, p2pSession);
 
@@ -1746,13 +2096,17 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             await Promise.all([sendPromise, megolmMessagePromise, memberListPromise]);
         });
 
-        oldBackendOnly("loading the membership list inhibits a later load", async () => {
+        it("loading the membership list inhibits a later load", async () => {
+            const homeserverUrl = aliceClient.getHomeserverUrl();
+            keyResponder = new E2EKeyResponder(homeserverUrl);
+            keyResponder.addKeyReceiver("@alice:localhost", keyReceiver);
+
+            const testDeviceKeys = getTestOlmAccountKeys(testOlmAccount, "@bob:xyz", "DEVICE_ID");
+            keyResponder.addDeviceKeys(testDeviceKeys);
+
             const room = aliceClient.getRoom(ROOM_ID)!;
             await Promise.all([room.loadMembersIfNeeded(), expectMembershipRequest(ROOM_ID, ["@bob:xyz"])]);
 
-            // expect a request for bob's devices...
-            expectAliceKeyQuery(getTestKeysQueryResponse("@bob:xyz"));
-
             // then a to-device with the room_key
             const inboundGroupSessionPromise = expectSendRoomKey("@bob:xyz", testOlmAccount, p2pSession);
 
@@ -2085,12 +2439,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
     });
 
     describe("Secret Storage and Key Backup", () => {
-        /**
-         * The account data events to be returned by the sync.
-         * Will be updated when fecthMock intercepts calls to PUT `/_matrix/client/v3/user/:userId/account_data/`.
-         * Will be used by `sendSyncResponseWithUpdatedAccountData`
-         */
-        let accountDataEvents: Map<String, any>;
+        let accountDataAccumulator: AccountDataAccumulator;
 
         /**
          * Create a fake secret storage key
@@ -2103,76 +2452,19 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
         beforeEach(async () => {
             createSecretStorageKey.mockClear();
-            accountDataEvents = new Map();
+            accountDataAccumulator = new AccountDataAccumulator();
             expectAliceKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
             await startClientAndAwaitFirstSync();
         });
 
-        function mockGetAccountData() {
-            fetchMock.get(
-                `path:/_matrix/client/v3/user/:userId/account_data/:type`,
-                (url) => {
-                    const type = url.split("/").pop();
-                    const existing = accountDataEvents.get(type!);
-                    if (existing) {
-                        // return it
-                        return {
-                            status: 200,
-                            body: existing.content,
-                        };
-                    } else {
-                        // 404
-                        return {
-                            status: 404,
-                            body: { errcode: "M_NOT_FOUND", error: "Account data not found." },
-                        };
-                    }
-                },
-                { overwriteRoutes: true },
-            );
-        }
-
         /**
          * Create a mock to respond to the PUT request `/_matrix/client/v3/user/:userId/account_data/m.cross_signing.${key}`
          * Resolved when the cross signing key is uploaded
          * https://spec.matrix.org/v1.6/client-server-api/#put_matrixclientv3useruseridaccount_datatype
          */
-        function awaitCrossSigningKeyUpload(key: string): Promise<Record<string, {}>> {
-            return new Promise((resolve) => {
-                // Called when the cross signing key is uploaded
-                fetchMock.put(
-                    `express:/_matrix/client/v3/user/:userId/account_data/m.cross_signing.${key}`,
-                    (url: string, options: RequestInit) => {
-                        const content = JSON.parse(options.body as string);
-                        const type = url.split("/").pop();
-                        // update account data for sync response
-                        accountDataEvents.set(type!, content);
-                        resolve(content.encrypted);
-                        return {};
-                    },
-                );
-            });
-        }
-
-        /**
-         * Send in the sync response the current account data events, as stored by `accountDataEvents`.
-         */
-        function sendSyncResponseWithUpdatedAccountData() {
-            try {
-                syncResponder.sendOrQueueSyncResponse({
-                    next_batch: 1,
-                    account_data: {
-                        events: Array.from(accountDataEvents, ([type, content]) => ({
-                            type: type,
-                            content: content,
-                        })),
-                    },
-                });
-            } catch (err) {
-                // Might fail with "Cannot queue more than one /sync response" if called too often.
-                // It's ok if it fails here, the sync response is cumulative and will contain
-                // the latest account data.
-            }
+        async function awaitCrossSigningKeyUpload(key: string): Promise<Record<string, {}>> {
+            const content = await accountDataAccumulator.interceptSetAccountData(`m.cross_signing.${key}`);
+            return content.encrypted;
         }
 
         /**
@@ -2180,28 +2472,18 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
          * Resolved when a key is uploaded (ie in `body.content.key`)
          * https://spec.matrix.org/v1.6/client-server-api/#put_matrixclientv3useruseridaccount_datatype
          */
-        function awaitSecretStorageKeyStoredInAccountData(): Promise<string> {
-            return new Promise((resolve) => {
-                // This url is called multiple times during the secret storage bootstrap process
-                // When we received the newly generated key, we return it
-                fetchMock.put(
-                    "express:/_matrix/client/v3/user/:userId/account_data/:type(m.secret_storage.*)",
-                    (url: string, options: RequestInit) => {
-                        const type = url.split("/").pop();
-                        const content = JSON.parse(options.body as string);
-
-                        // update account data for sync response
-                        accountDataEvents.set(type!, content);
-
-                        if (content.key) {
-                            resolve(content.key);
-                        }
-                        sendSyncResponseWithUpdatedAccountData();
-                        return {};
-                    },
-                    { overwriteRoutes: true },
-                );
-            });
+        async function awaitSecretStorageKeyStoredInAccountData(): Promise<string> {
+            // eslint-disable-next-line no-constant-condition
+            while (true) {
+                const content = await accountDataAccumulator.interceptSetAccountData(":type(m.secret_storage.*)", {
+                    repeat: 1,
+                    overwriteRoutes: true,
+                });
+                accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
+                if (content.key) {
+                    return content.key;
+                }
+            }
         }
 
         function awaitMegolmBackupKeyUpload(): Promise<Record<string, {}>> {
@@ -2212,7 +2494,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                     (url: string, options: RequestInit) => {
                         const content = JSON.parse(options.body as string);
                         // update account data for sync response
-                        accountDataEvents.set("m.megolm_backup.v1", content);
+                        accountDataAccumulator.accountDataEvents.set("m.megolm_backup.v1", content);
                         resolve(content.encrypted);
                         return {};
                     },
@@ -2277,7 +2559,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             await bootstrapPromise;
 
             // Return the newly created key in the sync response
-            sendSyncResponseWithUpdatedAccountData();
+            accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
 
             // Finally ensure backup is working
             await aliceClient.getCrypto()!.checkKeyBackupAndEnable();
@@ -2299,7 +2581,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
             );
 
             it("Should create a 4S key", async () => {
-                mockGetAccountData();
+                accountDataAccumulator.interceptGetAccountData();
 
                 const awaitAccountData = awaitAccountDataUpdate("m.secret_storage.default_key");
 
@@ -2311,7 +2593,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 const secretStorageKey = await awaitSecretStorageKeyStoredInAccountData();
 
                 // Return the newly created key in the sync response
-                sendSyncResponseWithUpdatedAccountData();
+                accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
 
                 // Finally, wait for bootstrapSecretStorage to finished
                 await bootstrapPromise;
@@ -2335,7 +2617,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 await awaitSecretStorageKeyStoredInAccountData();
 
                 // Return the newly created key in the sync response
-                sendSyncResponseWithUpdatedAccountData();
+                accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
 
                 // Wait for bootstrapSecretStorage to finished
                 await bootstrapPromise;
@@ -2359,7 +2641,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 await awaitSecretStorageKeyStoredInAccountData();
 
                 // Return the newly created key in the sync response
-                sendSyncResponseWithUpdatedAccountData();
+                accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
 
                 // Wait for bootstrapSecretStorage to finished
                 await bootstrapPromise;
@@ -2373,7 +2655,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 await awaitSecretStorageKeyStoredInAccountData();
 
                 // Return the newly created key in the sync response
-                sendSyncResponseWithUpdatedAccountData();
+                accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
 
                 // Wait for bootstrapSecretStorage to finished
                 await bootstrapPromise;
@@ -2397,7 +2679,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 const secretStorageKey = await awaitSecretStorageKeyStoredInAccountData();
 
                 // Return the newly created key in the sync response
-                sendSyncResponseWithUpdatedAccountData();
+                accountDataAccumulator.sendSyncResponseWithUpdatedAccountData(syncResponder);
 
                 // Wait for the cross signing keys to be uploaded
                 const [masterKey, userSigningKey, selfSigningKey] = await Promise.all([
@@ -2483,7 +2765,9 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 fetchMock.get("express:/_matrix/client/v3/room_keys/keys", keyBackupData);
 
                 // should be able to restore from 4S
-                const importResult = await aliceClient.restoreKeyBackupWithSecretStorage(check!.backupInfo!);
+                const importResult = await advanceTimersUntil(
+                    aliceClient.restoreKeyBackupWithSecretStorage(check!.backupInfo!),
+                );
                 expect(importResult.imported).toStrictEqual(1);
             });
 
@@ -2548,6 +2832,19 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
 
                 const newBackupUploadPromise = awaitMegolmBackupKeyUpload();
 
+                // Track calls to scheduleAllGroupSessionsForBackup. This is
+                // only relevant on legacy encryption.
+                const scheduleAllGroupSessionsForBackup = jest.fn();
+                if (backend === "libolm") {
+                    aliceClient.crypto!.backupManager.scheduleAllGroupSessionsForBackup =
+                        scheduleAllGroupSessionsForBackup;
+                } else {
+                    // With Rust crypto, we don't need to call this function, so
+                    // we call the dummy value here so we pass our later
+                    // expectation.
+                    scheduleAllGroupSessionsForBackup();
+                }
+
                 await aliceClient.getCrypto()!.resetKeyBackup();
                 await awaitDeleteCalled;
                 await newBackupStatusUpdate;
@@ -2559,6 +2856,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string,
                 expect(nextVersion).toBeDefined();
                 expect(nextVersion).not.toEqual(currentVersion);
                 expect(nextKey).not.toEqual(currentBackupKey);
+                expect(scheduleAllGroupSessionsForBackup).toHaveBeenCalled();
 
                 // The `deleteKeyBackupVersion` API is deprecated but has been modified to work with both crypto backend
                 // ensure that it works anyhow

spec/integ/crypto/megolm-backup.spec.ts

@@ -18,15 +18,23 @@ import fetchMock from "fetch-mock-jest";
 import "fake-indexeddb/auto";
 import { IDBFactory } from "fake-indexeddb";
 
-import { createClient, CryptoEvent, ICreateClientOpts, MatrixClient, TypedEventEmitter } from "../../../src";
+import { createClient, CryptoEvent, ICreateClientOpts, IEvent, MatrixClient, TypedEventEmitter } from "../../../src";
 import { SyncResponder } from "../../test-utils/SyncResponder";
 import { E2EKeyReceiver } from "../../test-utils/E2EKeyReceiver";
 import { E2EKeyResponder } from "../../test-utils/E2EKeyResponder";
 import { mockInitialApiRequests } from "../../test-utils/mockEndpoints";
-import { awaitDecryption, CRYPTO_BACKENDS, InitCrypto, syncPromise } from "../../test-utils/test-utils";
+import {
+    advanceTimersUntil,
+    awaitDecryption,
+    CRYPTO_BACKENDS,
+    InitCrypto,
+    syncPromise,
+} from "../../test-utils/test-utils";
 import * as testData from "../../test-utils/test-data";
 import { KeyBackupInfo } from "../../../src/crypto-api/keybackup";
 import { IKeyBackup } from "../../../src/crypto/backup";
+import { flushPromises } from "../../test-utils/flushPromises";
+import { defer, IDeferred } from "../../../src/utils";
 
 const ROOM_ID = testData.TEST_ROOM_ID;
 
@@ -110,9 +118,9 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
     /** an object which intercepts `/keys/query` requests on the test homeserver */
     let e2eKeyResponder: E2EKeyResponder;
 
-    jest.useFakeTimers();
-
     beforeEach(async () => {
+        jest.useFakeTimers();
+
         // anything that we don't have a specific matcher for silently returns a 404
         fetchMock.catch(404);
         fetchMock.config.warnOnFallback = false;
@@ -134,6 +142,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
         await jest.runAllTimersAsync();
 
         fetchMock.mockReset();
+        jest.restoreAllMocks();
     });
 
     async function initTestClient(opts: Partial<ICreateClientOpts> = {}): Promise<MatrixClient> {
@@ -149,64 +158,131 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
         return client;
     }
 
-    it("Alice checks key backups when receiving a message she can't decrypt", async function () {
-        const syncResponse = {
+    describe("Key backup check on UTD message", () => {
+        // sync response which contains an encrypted event
+        const SYNC_RESPONSE = {
             next_batch: 1,
-            rooms: {
-                join: {
-                    [ROOM_ID]: {
-                        timeline: {
-                            events: [testData.ENCRYPTED_EVENT],
-                        },
-                    },
-                },
-            },
+            rooms: { join: { [ROOM_ID]: { timeline: { events: [testData.ENCRYPTED_EVENT] } } } },
         };
 
-        fetchMock.get("express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id", (url, request) => {
-            // check that the version is correct
-            const version = new URLSearchParams(new URL(url).search).get("version");
-            if (version == "1") {
-                return testData.CURVE25519_KEY_BACKUP_DATA;
-            } else {
-                return {
-                    status: 403,
-                    body: {
-                        current_version: "1",
-                        errcode: "M_WRONG_ROOM_KEYS_VERSION",
-                        error: "Wrong backup version.",
-                    },
-                };
+        const EXPECTED_URL =
+            [
+                "https://alice-server.com/_matrix/client/v3/room_keys/keys",
+                encodeURIComponent(testData.TEST_ROOM_ID),
+                encodeURIComponent(testData.MEGOLM_SESSION_DATA.session_id),
+            ].join("/") + "?version=1";
+
+        /** Flush promises enough times to get the crypto stacks to make the backup request */
+        async function flushBackupRequest() {
+            // we have to run flushPromises lots of times. It seems like each time the rust code touches indexeddb,
+            // it needs another round of flushPromises to progress, or something.
+            for (let i = 0; i < 10; i++) {
+                await flushPromises();
             }
+        }
+
+        beforeEach(async () => {
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+
+            // ignore requests to send room key requests
+            fetchMock.put("express:/_matrix/client/v3/sendToDevice/m.room_key_request/:request_id", {});
+
+            aliceClient = await initTestClient();
+            const aliceCrypto = aliceClient.getCrypto()!;
+            await aliceCrypto.storeSessionBackupPrivateKey(
+                Buffer.from(testData.BACKUP_DECRYPTION_KEY_BASE64, "base64"),
+                testData.SIGNED_BACKUP_DATA.version!,
+            );
+
+            // start after saving the private key
+            await aliceClient.startClient();
+
+            // tell Alice to trust the dummy device that signed the backup, and re-check the backup.
+            // XXX: should we automatically re-check after a device becomes verified?
+            await waitForDeviceList();
+            await aliceClient.getCrypto()!.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
+            await aliceClient.getCrypto()!.checkKeyBackupAndEnable();
         });
 
-        fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+        it("Alice checks key backups when receiving a message she can't decrypt", async () => {
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id", (url, request) => {
+                // check that the version is correct
+                const version = new URLSearchParams(new URL(url).search).get("version");
+                if (version == "1") {
+                    return testData.CURVE25519_KEY_BACKUP_DATA;
+                } else {
+                    return {
+                        status: 403,
+                        body: {
+                            current_version: "1",
+                            errcode: "M_WRONG_ROOM_KEYS_VERSION",
+                            error: "Wrong backup version.",
+                        },
+                    };
+                }
+            });
 
-        aliceClient = await initTestClient();
-        const aliceCrypto = aliceClient.getCrypto()!;
-        await aliceCrypto.storeSessionBackupPrivateKey(
-            Buffer.from(testData.BACKUP_DECRYPTION_KEY_BASE64, "base64"),
-            testData.SIGNED_BACKUP_DATA.version!,
-        );
+            // Send Alice a message that she won't be able to decrypt, and check that she fetches the key from the backup.
+            syncResponder.sendOrQueueSyncResponse(SYNC_RESPONSE);
+            await syncPromise(aliceClient);
 
-        // start after saving the private key
-        await aliceClient.startClient();
+            const room = aliceClient.getRoom(ROOM_ID)!;
+            const event = room.getLiveTimeline().getEvents()[0];
+            await advanceTimersUntil(awaitDecryption(event, { waitOnDecryptionFailure: true }));
 
-        // tell Alice to trust the dummy device that signed the backup, and re-check the backup.
-        // XXX: should we automatically re-check after a device becomes verified?
-        await waitForDeviceList();
-        await aliceCrypto.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
-        await aliceClient.getCrypto()!.checkKeyBackupAndEnable();
+            expect(event.getContent()).toEqual(testData.CLEAR_EVENT.content);
+        });
 
-        // Now, send Alice a message that she won't be able to decrypt, and check that she fetches the key from the backup.
-        syncResponder.sendOrQueueSyncResponse(syncResponse);
-        await syncPromise(aliceClient);
+        it("handles error on backup query gracefully", async () => {
+            jest.spyOn(console, "error").mockImplementation(() => {});
 
-        const room = aliceClient.getRoom(ROOM_ID)!;
-        const event = room.getLiveTimeline().getEvents()[0];
-        await awaitDecryption(event, { waitOnDecryptionFailure: true });
+            fetchMock.get(
+                "express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id",
+                { status: 404, body: { errcode: "M_NOT_FOUND" } },
+                { name: "getKey" },
+            );
 
-        expect(event.getContent()).toEqual(testData.CLEAR_EVENT.content);
+            // Send Alice a message that she won't be able to decrypt
+            syncResponder.sendOrQueueSyncResponse(SYNC_RESPONSE);
+            await flushBackupRequest();
+
+            const calls = fetchMock.calls("getKey");
+            expect(calls.length).toEqual(1);
+            expect(calls[0][0]).toEqual(EXPECTED_URL);
+
+            await flushBackupRequest();
+
+            // we should not have logged an error.
+            // eslint-disable-next-line no-console
+            expect(console.error).not.toHaveBeenCalled();
+        });
+
+        it("Only queries once", async () => {
+            fetchMock.get(
+                "express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id",
+                { status: 404, body: { errcode: "M_NOT_FOUND" } },
+                { name: "getKey" },
+            );
+
+            // Send Alice a message that she won't be able to decrypt
+            syncResponder.sendOrQueueSyncResponse(SYNC_RESPONSE);
+            await flushBackupRequest();
+            const calls = fetchMock.calls("getKey");
+            expect(calls.length).toEqual(1);
+            expect(calls[0][0]).toEqual(EXPECTED_URL);
+
+            fetchMock.resetHistory();
+
+            // another message
+            const event2 = { ...testData.ENCRYPTED_EVENT, event_id: "$event2" };
+            const syncResponse2 = {
+                next_batch: 1,
+                rooms: { join: { [ROOM_ID]: { timeline: { events: [event2] } } } },
+            };
+            syncResponder.sendOrQueueSyncResponse(syncResponse2);
+            await flushBackupRequest();
+            expect(fetchMock.calls("getKey").length).toEqual(0);
+        });
     });
 
     describe("recover from backup", () => {
@@ -240,14 +316,16 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
                 onKeyCached = resolve;
             });
 
-            const result = await aliceClient.restoreKeyBackupWithRecoveryKey(
-                testData.BACKUP_DECRYPTION_KEY_BASE58,
-                undefined,
-                undefined,
-                check!.backupInfo!,
-                {
-                    cacheCompleteCallback: () => onKeyCached(),
-                },
+            const result = await advanceTimersUntil(
+                aliceClient.restoreKeyBackupWithRecoveryKey(
+                    testData.BACKUP_DECRYPTION_KEY_BASE58,
+                    undefined,
+                    undefined,
+                    check!.backupInfo!,
+                    {
+                        cacheCompleteCallback: () => onKeyCached(),
+                    },
+                ),
             );
 
             expect(result.imported).toStrictEqual(1);
@@ -255,7 +333,9 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
             await awaitKeyCached;
 
             // The key should be now cached
-            const afterCache = await aliceClient.restoreKeyBackupWithCache(undefined, undefined, check!.backupInfo!);
+            const afterCache = await advanceTimersUntil(
+                aliceClient.restoreKeyBackupWithCache(undefined, undefined, check!.backupInfo!),
+            );
 
             expect(afterCache.imported).toStrictEqual(1);
         });
@@ -278,11 +358,13 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
 
             const check = await aliceCrypto.checkKeyBackupAndEnable();
 
-            const result = await aliceClient.restoreKeyBackupWithRecoveryKey(
-                testData.BACKUP_DECRYPTION_KEY_BASE58,
-                ROOM_ID,
-                testData.MEGOLM_SESSION_DATA.session_id,
-                check!.backupInfo!,
+            const result = await advanceTimersUntil(
+                aliceClient.restoreKeyBackupWithRecoveryKey(
+                    testData.BACKUP_DECRYPTION_KEY_BASE58,
+                    ROOM_ID,
+                    testData.MEGOLM_SESSION_DATA.session_id,
+                    check!.backupInfo!,
+                ),
             );
 
             expect(result.imported).toStrictEqual(1);
@@ -807,6 +889,146 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
         });
     });
 
+    describe("Backup Changed from other sessions", () => {
+        beforeEach(async () => {
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+
+            // ignore requests to send room key requests
+            fetchMock.put("express:/_matrix/client/v3/sendToDevice/m.room_key_request/:request_id", {});
+
+            aliceClient = await initTestClient();
+            const aliceCrypto = aliceClient.getCrypto()!;
+            await aliceCrypto.storeSessionBackupPrivateKey(
+                Buffer.from(testData.BACKUP_DECRYPTION_KEY_BASE64, "base64"),
+                testData.SIGNED_BACKUP_DATA.version!,
+            );
+
+            // start after saving the private key
+            await aliceClient.startClient();
+
+            // tell Alice to trust the dummy device that signed the backup, and re-check the backup.
+            // XXX: should we automatically re-check after a device becomes verified?
+            await waitForDeviceList();
+            await aliceClient.getCrypto()!.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
+            await aliceClient.getCrypto()!.checkKeyBackupAndEnable();
+        });
+
+        // let aliceClient: MatrixClient;
+
+        const SYNC_RESPONSE = {
+            next_batch: 1,
+            rooms: { join: { [ROOM_ID]: { timeline: { events: [testData.ENCRYPTED_EVENT] } } } },
+        };
+
+        it("If current backup has changed, the manager should switch to the new one on UTD", async () => {
+            // =====
+            // First ensure that the client checks for keys using the backup version 1
+            /// =====
+
+            fetchMock.get(
+                "express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id",
+                (url, request) => {
+                    // check that the version is correct
+                    const version = new URLSearchParams(new URL(url).search).get("version");
+                    if (version == "1") {
+                        return testData.CURVE25519_KEY_BACKUP_DATA;
+                    } else {
+                        return {
+                            status: 403,
+                            body: {
+                                current_version: "1",
+                                errcode: "M_WRONG_ROOM_KEYS_VERSION",
+                                error: "Wrong backup version.",
+                            },
+                        };
+                    }
+                },
+                { overwriteRoutes: true },
+            );
+
+            // Send Alice a message that she won't be able to decrypt, and check that she fetches the key from the backup.
+            syncResponder.sendOrQueueSyncResponse(SYNC_RESPONSE);
+            await syncPromise(aliceClient);
+
+            const room = aliceClient.getRoom(ROOM_ID)!;
+            const event = room.getLiveTimeline().getEvents()[0];
+            await advanceTimersUntil(awaitDecryption(event, { waitOnDecryptionFailure: true }));
+
+            expect(event.getContent()).toEqual(testData.CLEAR_EVENT.content);
+
+            // =====
+            // Second suppose now that the backup has changed to version 2
+            /// =====
+
+            const newBackup = {
+                ...testData.SIGNED_BACKUP_DATA,
+                version: "2",
+            };
+
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", newBackup, { overwriteRoutes: true });
+            // suppose the new key is now known
+            const aliceCrypto = aliceClient.getCrypto()!;
+            await aliceCrypto.storeSessionBackupPrivateKey(
+                Buffer.from(testData.BACKUP_DECRYPTION_KEY_BASE64, "base64"),
+                newBackup.version,
+            );
+
+            // A check backup should happen at some point
+            await aliceCrypto.checkKeyBackupAndEnable();
+
+            const awaitHasQueriedNewBackup: IDeferred<void> = defer<void>();
+
+            fetchMock.get(
+                "express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id",
+                (url, request) => {
+                    // check that the version is correct
+                    const version = new URLSearchParams(new URL(url).search).get("version");
+                    if (version == newBackup.version) {
+                        awaitHasQueriedNewBackup.resolve();
+                        return testData.CURVE25519_KEY_BACKUP_DATA;
+                    } else {
+                        // awaitHasQueriedOldBackup.resolve();
+                        return {
+                            status: 403,
+                            body: {
+                                current_version: "2",
+                                errcode: "M_WRONG_ROOM_KEYS_VERSION",
+                                error: "Wrong backup version.",
+                            },
+                        };
+                    }
+                },
+                { overwriteRoutes: true },
+            );
+
+            // Send Alice a message that she won't be able to decrypt, and check that she fetches the key from the new backup.
+            const newMessage: Partial<IEvent> = {
+                type: "m.room.encrypted",
+                room_id: "!room:id",
+                sender: "@alice:localhost",
+                content: {
+                    algorithm: "m.megolm.v1.aes-sha2",
+                    ciphertext:
+                        "AwgAEpABKvf9FqPW52zeHfeVTn90a3jlBLlx7g6VDEkc2089RQUJoWpSJRiK13E83rN41wgGFJccyfoCr7ZDGJeuGYMGETTrgnLQhLs6JmyPf37JYkzxW8uS8rGUKEqTFQriKhibHVLvVacOlSIObUiKU/V3r176XuixqZF/4eyK9A22JNpInbgI10ZUT6LnApH9LR3FpZbE2zImf1uNPuvp7r0xQbW7CcJjqpH+qTPBD5zFdFnMkc2SnbXCsIOaX11Dm0krWfQz7iA26ZnI1nyZnyh7XPrCnJCRsuQH",
+                    device_id: "WVMJGTSSVB",
+                    sender_key: "E5RiY/YCIrHWaF4u416CqvblC6udK2jt9SJ/h1QeLS0",
+                    session_id: "ybnW+LGdUhoS4fHm1DAEphukO3sZ1GCqZD7UQz7L+GA",
+                },
+                event_id: "$event2",
+                origin_server_ts: 1507753887000,
+            };
+
+            const nextSyncResponse = {
+                next_batch: 2,
+                rooms: { join: { [ROOM_ID]: { timeline: { events: [newMessage] } } } },
+            };
+            syncResponder.sendOrQueueSyncResponse(nextSyncResponse);
+            await syncPromise(aliceClient);
+
+            await awaitHasQueriedNewBackup.promise;
+        });
+    });
+
     /** make sure that the client knows about the dummy device */
     async function waitForDeviceList(): Promise<void> {
         // Completing the initial sync will make the device list download outdated device lists (of which our own

spec/integ/crypto/olm-utils.ts

@@ -17,10 +17,11 @@ limitations under the License.
 import Olm from "@matrix-org/olm";
 import anotherjson from "another-json";
 
-import { IContent, IDeviceKeys, IEvent, MatrixClient } from "../../../src";
+import { IContent, IDeviceKeys, IDownloadKeyResult, IEvent, Keys, MatrixClient, SigningKeys } from "../../../src";
 import { IE2EKeyReceiver } from "../../test-utils/E2EKeyReceiver";
 import { ISyncResponder } from "../../test-utils/SyncResponder";
 import { syncPromise } from "../../test-utils/test-utils";
+import { KeyBackupInfo } from "../../../src/crypto-api";
 
 /**
  * @module
@@ -60,6 +61,117 @@ export function getTestOlmAccountKeys(olmAccount: Olm.Account, userId: string, d
     return testDeviceKeys;
 }
 
+/**
+ * Bootstrap cross signing for the given Olm account.
+ *
+ * Will generate the cross signing keys and sign them with the master key, and returns the `IDownloadKeyResult`
+ * that can be directly fed into a test e2eKeyResponder.
+ *
+ * The cross-signing keys are randomly generated, similar to how the olm account keys are generated. There may not
+ * be any value in using static vectors, as the device keys change at every test run.
+ *
+ * If some `KeyBackupInfo` are provided, the `auth_data` of each backup info will be signed with the
+ * master key, meaning the backups will be then trusted after verification.
+ *
+ * @param olmAccount - The Olm account object to use for signing the device keys.
+ * @param userId - The user ID to associate with the device keys.
+ * @param deviceId - The device ID to associate with the device keys.
+ * @param keyBackupInfo - Optional key backup infos to sign with the master key.
+ * @returns A valid keys/query response that can be fed into a test e2eKeyResponder.
+ */
+export function bootstrapCrossSigningTestOlmAccount(
+    olmAccount: Olm.Account,
+    userId: string,
+    deviceId: string,
+    keyBackupInfo: KeyBackupInfo[] = [],
+): Partial<IDownloadKeyResult> {
+    const olmAliceMSK = new global.Olm.PkSigning();
+    const masterPrivkey = olmAliceMSK.generate_seed();
+    const masterPubkey = olmAliceMSK.init_with_seed(masterPrivkey);
+
+    const olmAliceUSK = new global.Olm.PkSigning();
+    const userPrivkey = olmAliceUSK.generate_seed();
+    const userPubkey = olmAliceUSK.init_with_seed(userPrivkey);
+
+    const olmAliceSSK = new global.Olm.PkSigning();
+    const sskPrivkey = olmAliceSSK.generate_seed();
+    const sskPubkey = olmAliceSSK.init_with_seed(sskPrivkey);
+
+    const mskInfo: Keys = {
+        user_id: userId,
+        usage: ["master"],
+        keys: {
+            ["ed25519:" + masterPubkey]: masterPubkey,
+        },
+    };
+
+    const sskInfo: Partial<SigningKeys> = {
+        user_id: userId,
+        usage: ["self_signing"],
+        keys: {
+            ["ed25519:" + sskPubkey]: sskPubkey,
+        },
+    };
+    // sign the ssk with the msk
+    const sskSig = olmAliceMSK.sign(anotherjson.stringify(sskInfo));
+    sskInfo.signatures = {
+        [userId]: {
+            ["ed25519:" + masterPubkey]: sskSig,
+        },
+    };
+
+    const uskInfo: Partial<SigningKeys> = {
+        user_id: userId,
+        usage: ["user_signing"],
+        keys: {
+            ["ed25519:" + userPubkey]: userPubkey,
+        },
+    };
+
+    // sign the usk with the msk
+    const uskSig = olmAliceMSK.sign(anotherjson.stringify(uskInfo));
+    uskInfo.signatures = {
+        [userId]: {
+            ["ed25519:" + masterPubkey]: uskSig,
+        },
+    };
+
+    // get the device keys and sign them with the ssk (the device is then cross signed)
+    const deviceKeys = getTestOlmAccountKeys(olmAccount, userId, deviceId);
+
+    const copy = Object.assign({}, deviceKeys);
+    delete copy.signatures;
+    const crossSignature = olmAliceSSK.sign(anotherjson.stringify(copy));
+
+    // add the signature
+    deviceKeys.signatures![userId]["ed25519:" + sskPubkey] = crossSignature;
+
+    // if we have some key backup info, sign them with the msk
+    keyBackupInfo.forEach((info) => {
+        const unsignedAuthData = Object.assign({}, info.auth_data);
+        delete unsignedAuthData.signatures;
+        const backupSignature = olmAliceMSK.sign(anotherjson.stringify(unsignedAuthData));
+
+        info.auth_data.signatures = {
+            [userId]: {
+                ["ed25519:" + masterPubkey]: backupSignature,
+            },
+        };
+    });
+
+    // clean the olm resources as we don't need them anymore
+    olmAliceMSK.free();
+    olmAliceSSK.free();
+    olmAliceUSK.free();
+
+    return {
+        master_keys: { [userId]: mskInfo },
+        user_signing_keys: { [userId]: uskInfo as SigningKeys },
+        self_signing_keys: { [userId]: sskInfo as SigningKeys },
+        device_keys: { [userId]: { [deviceId]: deviceKeys } },
+    };
+}
+
 /** start an Olm session with a given recipient */
 export async function createOlmSession(
     olmAccount: Olm.Account,
@@ -218,6 +330,47 @@ export function encryptGroupSessionKey(opts: {
     });
 }
 
+/**
+ * Test utility to correctly encrypt a secret send event to a test device using the provided p2p session.
+ *
+ * @param opts - the options for the secret send event
+ * @returns the to-device event, ready to be returned in a sync response for the test device.
+ */
+export function encryptSecretSend(opts: {
+    /** the sender's user id */
+    sender: string;
+    /** recipient's user id */
+    recipient: string;
+    /** the recipient's curve25519 key */
+    recipientCurve25519Key: string;
+    /** the recipient's ed25519 key */
+    recipientEd25519Key: string;
+    /** sender's olm account */
+    olmAccount: Olm.Account;
+    /** sender's olm session with the recipient */
+    p2pSession: Olm.Session;
+    /** The requestId of the secret request that this secret send is replying. */
+    requestId: string;
+    /** The secret value */
+    secret: string;
+}): ToDeviceEvent {
+    const senderKeys = JSON.parse(opts.olmAccount.identity_keys());
+    return encryptOlmEvent({
+        sender: opts.sender,
+        senderKey: senderKeys.curve25519,
+        senderSigningKey: senderKeys.ed25519,
+        recipient: opts.recipient,
+        recipientCurve25519Key: opts.recipientCurve25519Key,
+        recipientEd25519Key: opts.recipientEd25519Key,
+        p2pSession: opts.p2pSession,
+        plaincontent: {
+            request_id: opts.requestId,
+            secret: opts.secret,
+        },
+        plaintype: "m.secret.send",
+    });
+}
+
 /**
  * Establish an Olm Session with the test user
  *

spec/integ/crypto/verification.spec.ts

@@ -26,6 +26,7 @@ import Olm from "@matrix-org/olm";
 import {
     createClient,
     CryptoEvent,
+    DeviceVerification,
     IContent,
     ICreateClientOpts,
     IEvent,
@@ -43,7 +44,7 @@ import {
     Verifier,
     VerifierEvent,
 } from "../../../src/crypto-api/verification";
-import { escapeRegExp } from "../../../src/utils";
+import { defer, escapeRegExp } from "../../../src/utils";
 import {
     awaitDecryption,
     CRYPTO_BACKENDS,
@@ -54,10 +55,12 @@ import {
 } from "../../test-utils/test-utils";
 import { SyncResponder } from "../../test-utils/SyncResponder";
 import {
+    BACKUP_DECRYPTION_KEY_BASE64,
     BOB_ONE_TIME_KEYS,
     BOB_SIGNED_CROSS_SIGNING_KEYS_DATA,
     BOB_SIGNED_TEST_DEVICE_DATA,
     BOB_TEST_USER_ID,
+    CURVE25519_KEY_BACKUP_DATA,
     MASTER_CROSS_SIGNING_PUBLIC_KEY_BASE64,
     SIGNED_CROSS_SIGNING_KEYS_DATA,
     SIGNED_TEST_DEVICE_DATA,
@@ -69,7 +72,16 @@ import {
 import { mockInitialApiRequests } from "../../test-utils/mockEndpoints";
 import { E2EKeyResponder } from "../../test-utils/E2EKeyResponder";
 import { E2EKeyReceiver } from "../../test-utils/E2EKeyReceiver";
-import { createOlmSession, encryptGroupSessionKey, encryptMegolmEvent, ToDeviceEvent } from "./olm-utils";
+import {
+    bootstrapCrossSigningTestOlmAccount,
+    createOlmSession,
+    encryptGroupSessionKey,
+    encryptMegolmEvent,
+    encryptSecretSend,
+    ToDeviceEvent,
+} from "./olm-utils";
+import { KeyBackupInfo } from "../../../src/crypto-api";
+import { encodeBase64 } from "../../../src/base64";
 
 // The verification flows use javascript timers to set timeouts. We tell jest to use mock timer implementations
 // to ensure that we don't end up with dangling timeouts.
@@ -1160,6 +1172,289 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
         );
     });
 
+    describe("Secrets are gossiped after verification", () => {
+        // We use a legacy olm session as the existing session.
+        // This will give us access to low level olm functions in order to
+        // simulate a backup key request with proper olm encryption.
+        let testOlmAccount: Olm.Account;
+        const olmDeviceId = "OLM_DEVICE";
+        let usermasterPubKey: string;
+
+        const matchingBackupInfo: KeyBackupInfo = {
+            algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+            version: "1",
+            auth_data: {
+                public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+            },
+        };
+
+        const nonMatchingBackupInfo: KeyBackupInfo = {
+            algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+            version: "1",
+            auth_data: {
+                public_key: "EjDwCYkwp1R0i33ctD73Wg2/Og0mOBr066Spjqqaqqo",
+            },
+        };
+
+        const unknownAlgorithmBackupInfo: KeyBackupInfo = {
+            algorithm: "m.megolm_backup.foo_bar",
+            version: "1",
+            auth_data: {
+                public_key: "EjDwCYkwp1R0i33ctD73Wg2/Og0mOBr066Spjqqaqqo",
+            },
+        };
+
+        beforeEach(async () => {
+            // create a test olm device which we will use to communicate with alice. We use libolm to implement this.
+            await Olm.init();
+            testOlmAccount = new Olm.Account();
+            testOlmAccount.create();
+
+            const bootstrapped = bootstrapCrossSigningTestOlmAccount(testOlmAccount, TEST_USER_ID, olmDeviceId, [
+                matchingBackupInfo,
+                nonMatchingBackupInfo,
+            ]);
+
+            e2eKeyResponder.addDeviceKeys(bootstrapped.device_keys![TEST_USER_ID]![olmDeviceId]);
+            e2eKeyResponder.addCrossSigningData(bootstrapped);
+
+            usermasterPubKey = Object.values(bootstrapped.master_keys![TEST_USER_ID].keys)[0];
+
+            aliceClient = await startTestClient();
+            syncResponder.sendOrQueueSyncResponse(getSyncResponse([TEST_USER_ID]));
+            await syncPromise(aliceClient);
+            // DeviceList has a sleep(5) which we need to make happen
+            await jest.advanceTimersByTimeAsync(10);
+
+            // The client should now know about the olm device
+            const devices = await aliceClient.getCrypto()!.getUserDeviceInfo([TEST_USER_ID]);
+            expect(devices.get(TEST_USER_ID)!.keys()).toContain(olmDeviceId);
+        });
+
+        afterEach(async () => {
+            aliceClient?.stopClient();
+            testOlmAccount?.free();
+
+            // Allow in-flight things to complete before we tear down the test
+            await jest.runAllTimersAsync();
+
+            fetchMock.mockReset();
+        });
+
+        newBackendOnly("Should request cross signing keys after verification", async () => {
+            const requestPromises = mockSecretRequestAndGetPromises();
+
+            await doInteractiveVerification();
+
+            // The secret must have been requested
+            await requestPromises.get("m.cross_signing.master");
+            await requestPromises.get("m.cross_signing.user_signing");
+            await requestPromises.get("m.cross_signing.self_signing");
+        });
+
+        newBackendOnly("Should accept the backup decryption key gossip if valid", async () => {
+            const requestPromises = mockSecretRequestAndGetPromises();
+
+            await doInteractiveVerification();
+
+            const requestId = await requestPromises.get("m.megolm_backup.v1");
+
+            const keyBackupIsCached = emitPromise(aliceClient, CryptoEvent.KeyBackupDecryptionKeyCached);
+
+            await sendBackupGossipAndExpectVersion(requestId!, BACKUP_DECRYPTION_KEY_BASE64, matchingBackupInfo);
+
+            await keyBackupIsCached;
+
+            // the backup secret should be cached
+            const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
+            expect(cachedKey).toBeTruthy();
+            expect(encodeBase64(cachedKey!)).toEqual(BACKUP_DECRYPTION_KEY_BASE64);
+        });
+
+        newBackendOnly("Should not accept the backup decryption key gossip if private key do not match", async () => {
+            const requestPromises = mockSecretRequestAndGetPromises();
+
+            await doInteractiveVerification();
+
+            const requestId = await requestPromises.get("m.megolm_backup.v1");
+
+            await sendBackupGossipAndExpectVersion(requestId!, BACKUP_DECRYPTION_KEY_BASE64, nonMatchingBackupInfo);
+
+            // We are lacking a way to signal that the secret has been received, so we wait a bit..
+            jest.useRealTimers();
+            await new Promise((resolve) => {
+                setTimeout(resolve, 500);
+            });
+            jest.useFakeTimers();
+
+            // the backup secret should not be cached
+            const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
+            expect(cachedKey).toBeNull();
+        });
+
+        newBackendOnly("Should not accept the backup decryption key gossip if backup not trusted", async () => {
+            const requestPromises = mockSecretRequestAndGetPromises();
+
+            await doInteractiveVerification();
+
+            const requestId = await requestPromises.get("m.megolm_backup.v1");
+
+            const infoCopy = Object.assign({}, matchingBackupInfo);
+            delete infoCopy.auth_data.signatures;
+
+            await sendBackupGossipAndExpectVersion(requestId!, BACKUP_DECRYPTION_KEY_BASE64, infoCopy);
+
+            // We are lacking a way to signal that the secret has been received, so we wait a bit..
+            jest.useRealTimers();
+            await new Promise((resolve) => {
+                setTimeout(resolve, 500);
+            });
+            jest.useFakeTimers();
+
+            // the backup secret should not be cached
+            const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
+            expect(cachedKey).toBeNull();
+        });
+
+        newBackendOnly("Should not accept the backup decryption key gossip if backup algorithm unknown", async () => {
+            const requestPromises = mockSecretRequestAndGetPromises();
+
+            await doInteractiveVerification();
+
+            const requestId = await requestPromises.get("m.megolm_backup.v1");
+
+            await sendBackupGossipAndExpectVersion(
+                requestId!,
+                BACKUP_DECRYPTION_KEY_BASE64,
+                unknownAlgorithmBackupInfo,
+            );
+
+            // We are lacking a way to signal that the secret has been received, so we wait a bit..
+            jest.useRealTimers();
+            await new Promise((resolve) => {
+                setTimeout(resolve, 500);
+            });
+            jest.useFakeTimers();
+
+            // the backup secret should not be cached
+            const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
+            expect(cachedKey).toBeNull();
+        });
+
+        newBackendOnly("Should not accept an invalid backup decryption key", async () => {
+            const requestPromises = mockSecretRequestAndGetPromises();
+
+            await doInteractiveVerification();
+
+            const requestId = await requestPromises.get("m.megolm_backup.v1");
+
+            await sendBackupGossipAndExpectVersion(requestId!, "InvalidSecret", matchingBackupInfo);
+
+            // We are lacking a way to signal that the secret has been received, so we wait a bit..
+            jest.useRealTimers();
+            await new Promise((resolve) => {
+                setTimeout(resolve, 500);
+            });
+            jest.useFakeTimers();
+
+            // the backup secret should not be cached
+            const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
+            expect(cachedKey).toBeNull();
+        });
+
+        /**
+         * Common test setup for gossiping secrets.
+         * Creates a peer to peer session, sends the secret, mockup the version API, send the secret back from sync, then await for the backup check.
+         */
+        async function sendBackupGossipAndExpectVersion(
+            requestId: string,
+            secret: string,
+            expectBackup: KeyBackupInfo,
+        ) {
+            const p2pSession = await createOlmSession(testOlmAccount, e2eKeyReceiver);
+
+            const toDeviceEvent = encryptSecretSend({
+                sender: aliceClient.getUserId()!,
+                recipient: aliceClient.getUserId()!,
+                recipientCurve25519Key: e2eKeyReceiver.getDeviceKey(),
+                recipientEd25519Key: e2eKeyReceiver.getSigningKey(),
+                p2pSession: p2pSession,
+                olmAccount: testOlmAccount,
+                requestId: requestId!,
+                secret: secret,
+            });
+
+            const expectBackupCheck = new Promise((resolve) => {
+                fetchMock.get(
+                    "express:/_matrix/client/v3/room_keys/version",
+                    (url, request) => {
+                        resolve(undefined);
+                        return expectBackup;
+                    },
+                    {
+                        overwriteRoutes: true,
+                    },
+                );
+            });
+
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys", CURVE25519_KEY_BACKUP_DATA);
+
+            // The dummy device sends the secret
+            returnToDeviceMessageFromSync(toDeviceEvent);
+
+            await expectBackupCheck;
+        }
+
+        /**
+         * Do an interactive verification between alice and the dummy device.
+         */
+        async function doInteractiveVerification(): Promise<void> {
+            // Do a QR code verification for simplicity
+
+            // Alice sends a m.key.verification.request
+            const [, request] = await Promise.all([
+                expectSendToDeviceMessage("m.key.verification.request"),
+                aliceClient.getCrypto()!.requestDeviceVerification(TEST_USER_ID, olmDeviceId),
+            ]);
+            const transactionId = request.transactionId!;
+
+            // The dummy device replies with an m.key.verification.ready, indicating it can show a QR code
+            returnToDeviceMessageFromSync(
+                buildReadyMessage(transactionId, ["m.qr_code.show.v1", "m.reciprocate.v1"], olmDeviceId),
+            );
+            await waitForVerificationRequestChanged(request);
+
+            const currentDeviceKey = e2eKeyReceiver.getSigningKey();
+            // the dummy device shows a QR code
+            const sharedSecret = "SUPERSEKRET";
+            // use mode 0x01, self-verifying in which the current device does trust the master key
+            const mode = 0x01;
+            const qrCodeBuffer = buildQRCode(transactionId, usermasterPubKey, currentDeviceKey, sharedSecret, mode);
+
+            // Alice scans the QR code
+            const sendToDevicePromise = expectSendToDeviceMessage("m.key.verification.start");
+            const verifier = await request.scanQRCode(qrCodeBuffer);
+
+            await sendToDevicePromise;
+
+            const verificationPromise = verifier.verify();
+            // the dummy device confirms that Alice scanned the QR code, by replying with a done
+            returnToDeviceMessageFromSync(buildDoneMessage(transactionId));
+
+            // Alice also replies with a 'done'
+            await expectSendToDeviceMessage("m.key.verification.done");
+
+            // ... and the whole thing should be done!
+            await verificationPromise;
+
+            // The other device should now be verified.
+            const otherDevice = (await aliceClient.getCrypto()!.getUserDeviceInfo([TEST_USER_ID]))
+                .get(TEST_USER_ID)!
+                .get(olmDeviceId);
+            expect(otherDevice?.verified).toEqual(DeviceVerification.Verified);
+        }
+    });
+
     async function startTestClient(opts: Partial<ICreateClientOpts> = {}): Promise<MatrixClient> {
         const client = createClient({
             baseUrl: TEST_HOMESERVER_URL,
@@ -1221,6 +1516,52 @@ function expectSendToDeviceMessage(msgtype: string): Promise<{ messages: any }>
     });
 }
 
+/**
+ * Utility to add all needed mocks for secret requesting (to-device of type `m.secret.request`).
+ *
+ * The following secrets are mocked: `m.cross_signing.master`, `m.cross_signing.self_signing`,
+ * `m.cross_signing.user_signing`, `m.megolm_backup.v1`.
+ *
+ *  @returns a map of secret name to promise that will resolve (with the id of the secret request) when the secret is requested.
+ */
+function mockSecretRequestAndGetPromises(): Map<string, Promise<string>> {
+    const mskRequestDefer = defer<string>();
+    const sskRequestDefer = defer<string>();
+    const uskRequestDefer = defer<string>();
+    const backupKeyRequestDefer = defer<string>();
+
+    fetchMock.put(
+        new RegExp(`/_matrix/client/(r0|v3)/sendToDevice/m.secret.request`),
+        (url: string, opts: RequestInit): MockResponse => {
+            const messages = JSON.parse(opts.body as string).messages[TEST_USER_ID];
+            // rust crypto broadcasts to all devices, old crypto to a specific device, take the first one
+            const content = Object.values(messages)[0] as any;
+            if (content.action == "request") {
+                const name = content.name;
+                const requestId = content.request_id;
+                if (name == "m.cross_signing.user_signing") {
+                    uskRequestDefer.resolve(requestId);
+                } else if (name == "m.cross_signing.master") {
+                    mskRequestDefer.resolve(requestId);
+                } else if (name == "m.cross_signing.self_signing") {
+                    sskRequestDefer.resolve(requestId);
+                } else if (name == "m.megolm_backup.v1") {
+                    backupKeyRequestDefer.resolve(requestId);
+                }
+            }
+            return {};
+        },
+        { overwriteRoutes: true },
+    );
+
+    const promiseMap = new Map<string, Promise<string>>();
+    promiseMap.set("m.cross_signing.master", mskRequestDefer.promise);
+    promiseMap.set("m.cross_signing.self_signing", sskRequestDefer.promise);
+    promiseMap.set("m.cross_signing.user_signing", uskRequestDefer.promise);
+    promiseMap.set("m.megolm_backup.v1", backupKeyRequestDefer.promise);
+    return promiseMap;
+}
+
 /** wait for the verification request to emit a 'Change' event */
 function waitForVerificationRequestChanged(request: VerificationRequest): Promise<void> {
     return new Promise<void>((resolve) => {
@@ -1265,12 +1606,16 @@ function buildRequestMessage(transactionId: string): { type: string; content: ob
     };
 }
 
-/** build an m.key.verification.ready to-device message originating from the dummy device */
-function buildReadyMessage(transactionId: string, methods: string[]): { type: string; content: object } {
+/** build an m.key.verification.ready to-device message originating from the given `fromDevice` (default to `TEST_DEVICE_ID` if not provided) */
+function buildReadyMessage(
+    transactionId: string,
+    methods: string[],
+    fromDevice?: string,
+): { type: string; content: object } {
     return {
         type: "m.key.verification.ready",
         content: {
-            from_device: TEST_DEVICE_ID,
+            from_device: fromDevice || TEST_DEVICE_ID,
             methods: methods,
             transaction_id: transactionId,
         },
@@ -1368,14 +1713,20 @@ function buildDoneMessage(transactionId: string) {
     };
 }
 
-function buildQRCode(transactionId: string, key1Base64: string, key2Base64: string, sharedSecret: string): Uint8Array {
+function buildQRCode(
+    transactionId: string,
+    key1Base64: string,
+    key2Base64: string,
+    sharedSecret: string,
+    mode = 0x02,
+): Uint8Array {
     // https://spec.matrix.org/v1.7/client-server-api/#qr-code-format
 
     const qrCodeBuffer = Buffer.alloc(150); // oversize
     let idx = 0;
     idx += qrCodeBuffer.write("MATRIX", idx, "ascii");
     idx = qrCodeBuffer.writeUInt8(0x02, idx); // version
-    idx = qrCodeBuffer.writeUInt8(0x02, idx); // mode
+    idx = qrCodeBuffer.writeUInt8(mode, idx); // mode
     idx = qrCodeBuffer.writeInt16BE(transactionId.length, idx);
     idx += qrCodeBuffer.write(transactionId, idx, "ascii");
 

spec/integ/matrix-client-event-emitter.spec.ts

@@ -194,6 +194,37 @@ describe("MatrixClient events", function () {
             expect(fired).toBe(true);
         });
 
+        it("should emit User events when presence data is absent in first sync", async () => {
+            const MODIFIED_SYNC_DATA: any = structuredClone(SYNC_DATA);
+            delete MODIFIED_SYNC_DATA["presence"];
+            const MODIFIED_NEXT_SYNC_DATA: any = structuredClone(NEXT_SYNC_DATA);
+            MODIFIED_NEXT_SYNC_DATA.presence = {
+                events: [
+                    utils.mkPresence({
+                        user: "@foo:bar",
+                        name: "Foo Bar",
+                        presence: "online",
+                    }),
+                ],
+            };
+            httpBackend!.when("GET", "/sync").respond(200, MODIFIED_SYNC_DATA);
+            httpBackend!.when("GET", "/sync").respond(200, MODIFIED_NEXT_SYNC_DATA);
+            let fired = false;
+            client!.on(UserEvent.Presence, function (event, user) {
+                fired = true;
+                expect(user).toBeTruthy();
+                expect(event).toBeTruthy();
+                if (!user || !event) {
+                    return;
+                }
+                expect(event.event).toEqual(MODIFIED_NEXT_SYNC_DATA.presence.events[0]);
+                expect(user.presence).toEqual(MODIFIED_NEXT_SYNC_DATA.presence.events[0]?.content?.presence);
+            });
+            client!.startClient();
+            await httpBackend!.flushAllExpected();
+            expect(fired).toBe(true);
+        });
+
         it("should emit Room events", function () {
             httpBackend!.when("GET", "/sync").respond(200, SYNC_DATA);
             httpBackend!.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);

spec/integ/matrix-client-methods.spec.ts

@@ -1204,73 +1204,20 @@ describe("MatrixClient", function () {
 
     describe("requestLoginToken", () => {
         it("should hit the expected API endpoint with UIA", async () => {
-            jest.spyOn(client.http, "getUrl");
-            httpBackend
-                .when("GET", "/capabilities")
-                .respond(200, { capabilities: { "m.get_login_token": { enabled: true } } });
             const response = {};
             const uiaData = {};
             const prom = client.requestLoginToken(uiaData);
             httpBackend.when("POST", "/v1/login/get_token", { auth: uiaData }).respond(200, response);
             await httpBackend.flush("");
             expect(await prom).toStrictEqual(response);
-            expect(client.http.getUrl).toHaveLastReturnedWith(
-                expect.objectContaining({
-                    href: "http://alice.localhost.test.server/_matrix/client/v1/login/get_token",
-                }),
-            );
         });
 
         it("should hit the expected API endpoint without UIA", async () => {
-            jest.spyOn(client.http, "getUrl");
-            httpBackend
-                .when("GET", "/capabilities")
-                .respond(200, { capabilities: { "m.get_login_token": { enabled: true } } });
             const response = { login_token: "xyz", expires_in_ms: 5000 };
             const prom = client.requestLoginToken();
             httpBackend.when("POST", "/v1/login/get_token", {}).respond(200, response);
             await httpBackend.flush("");
-            // check that expires_in has been populated for compatibility with r0
-            expect(await prom).toStrictEqual({ ...response, expires_in: 5 });
-            expect(client.http.getUrl).toHaveLastReturnedWith(
-                expect.objectContaining({
-                    href: "http://alice.localhost.test.server/_matrix/client/v1/login/get_token",
-                }),
-            );
-        });
-
-        it("should still hit the stable endpoint when capability is disabled (but present)", async () => {
-            jest.spyOn(client.http, "getUrl");
-            httpBackend
-                .when("GET", "/capabilities")
-                .respond(200, { capabilities: { "m.get_login_token": { enabled: false } } });
-            const response = { login_token: "xyz", expires_in_ms: 5000 };
-            const prom = client.requestLoginToken();
-            httpBackend.when("POST", "/v1/login/get_token", {}).respond(200, response);
-            await httpBackend.flush("");
-            // check that expires_in has been populated for compatibility with r0
-            expect(await prom).toStrictEqual({ ...response, expires_in: 5 });
-            expect(client.http.getUrl).toHaveLastReturnedWith(
-                expect.objectContaining({
-                    href: "http://alice.localhost.test.server/_matrix/client/v1/login/get_token",
-                }),
-            );
-        });
-
-        it("should hit the r0 endpoint for fallback", async () => {
-            jest.spyOn(client.http, "getUrl");
-            httpBackend.when("GET", "/capabilities").respond(200, {});
-            const response = { login_token: "xyz", expires_in: 5 };
-            const prom = client.requestLoginToken();
-            httpBackend.when("POST", "/unstable/org.matrix.msc3882/login/token", {}).respond(200, response);
-            await httpBackend.flush("");
-            // check that expires_in has been populated for compatibility with r1
-            expect(await prom).toStrictEqual({ ...response, expires_in_ms: 5000 });
-            expect(client.http.getUrl).toHaveLastReturnedWith(
-                expect.objectContaining({
-                    href: "http://alice.localhost.test.server/_matrix/client/unstable/org.matrix.msc3882/login/token",
-                }),
-            );
+            expect(await prom).toStrictEqual(response);
         });
     });
 

spec/integ/matrix-client-unread-notifications.spec.ts

@@ -28,32 +28,70 @@ import {
     NotificationCountType,
     RelationType,
     Room,
+    fixNotificationCountOnDecryption,
 } from "../../src";
 import { TestClient } from "../TestClient";
 import { ReceiptType } from "../../src/@types/read_receipts";
 import { mkThread } from "../test-utils/thread";
 import { SyncState } from "../../src/sync";
 
+const userA = "@alice:localhost";
+const userB = "@bob:localhost";
+const selfUserId = userA;
+const selfAccessToken = "aseukfgwef";
+
+function setupTestClient(): [MatrixClient, HttpBackend] {
+    const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
+    const httpBackend = testClient.httpBackend;
+    const client = testClient.client;
+    httpBackend!.when("GET", "/versions").respond(200, {});
+    httpBackend!.when("GET", "/pushrules").respond(200, {});
+    httpBackend!.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
+    return [client, httpBackend];
+}
+
+describe("Notification count fixing", () => {
+    let client: MatrixClient | undefined;
+
+    beforeEach(() => {
+        [client] = setupTestClient();
+    });
+
+    it("doesn't increment notification count for events that can't be found in a room", async () => {
+        const roomId = "!room:localhost";
+
+        client!.startClient({ threadSupport: true });
+        const room = new Room(roomId, client!, selfUserId);
+        jest.spyOn(client!, "getRoom").mockImplementation((id) => (id === roomId ? room : null));
+
+        const event = new MatrixEvent({
+            room_id: roomId,
+            type: "m.reaction",
+            event_id: "$foo",
+            content: {
+                "m.relates_to": {
+                    rel_type: RelationType.Annotation,
+                    event_id: "$foo",
+                    key: "x",
+                },
+            },
+        });
+
+        jest.spyOn(event, "getPushActions").mockReturnValue({
+            notify: true,
+            tweaks: {},
+        });
+
+        fixNotificationCountOnDecryption(client!, event);
+
+        expect(room.getUnreadNotificationCount(NotificationCountType.Total)).toBe(0);
+    });
+});
+
 describe("MatrixClient syncing", () => {
-    const userA = "@alice:localhost";
-    const userB = "@bob:localhost";
-
-    const selfUserId = userA;
-    const selfAccessToken = "aseukfgwef";
-
     let client: MatrixClient | undefined;
     let httpBackend: HttpBackend | undefined;
 
-    const setupTestClient = (): [MatrixClient, HttpBackend] => {
-        const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
-        const httpBackend = testClient.httpBackend;
-        const client = testClient.client;
-        httpBackend!.when("GET", "/versions").respond(200, {});
-        httpBackend!.when("GET", "/pushrules").respond(200, {});
-        httpBackend!.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
-        return [client, httpBackend];
-    };
-
     beforeEach(() => {
         [client, httpBackend] = setupTestClient();
     });

spec/test-utils/AccountDataAccumulator.ts

@@ -0,0 +1,108 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import fetchMock from "fetch-mock-jest";
+import { MockOptionsMethodPut } from "fetch-mock";
+
+import { ISyncResponder } from "./SyncResponder";
+
+/**
+ *  An object which intercepts `account_data` get and set requests via fetch-mock.
+ */
+export class AccountDataAccumulator {
+    /**
+     * The account data events to be returned by the sync.
+     * Will be updated when fetchMock intercepts calls to PUT `/_matrix/client/v3/user/:userId/account_data/`.
+     * Will be used by `sendSyncResponseWithUpdatedAccountData`
+     */
+    public accountDataEvents: Map<String, any> = new Map();
+
+    /**
+     * Intercept requests to set a particular type of account data.
+     *
+     * Once it is set, its data is stored (for future return by `interceptGetAccountData` etc) and the resolved promise is
+     * resolved.
+     *
+     * @param accountDataType - type of account data to be intercepted
+     * @param opts - options to pass to fetchMock
+     * @returns a Promise which will resolve (with the content of the account data) once it is set.
+     */
+    public interceptSetAccountData(accountDataType: string, opts?: MockOptionsMethodPut): Promise<any> {
+        return new Promise((resolve) => {
+            // Called when the cross signing key is uploaded
+            fetchMock.put(
+                `express:/_matrix/client/v3/user/:userId/account_data/${accountDataType}`,
+                (url: string, options: RequestInit) => {
+                    const content = JSON.parse(options.body as string);
+                    const type = url.split("/").pop();
+                    // update account data for sync response
+                    this.accountDataEvents.set(type!, content);
+                    resolve(content);
+                    return {};
+                },
+                opts,
+            );
+        });
+    }
+
+    /**
+     * Intercept all requests to get account data
+     */
+    public interceptGetAccountData(): void {
+        fetchMock.get(
+            `express:/_matrix/client/v3/user/:userId/account_data/:type`,
+            (url) => {
+                const type = url.split("/").pop();
+                const existing = this.accountDataEvents.get(type!);
+                if (existing) {
+                    // return it
+                    return {
+                        status: 200,
+                        body: existing,
+                    };
+                } else {
+                    // 404
+                    return {
+                        status: 404,
+                        body: { errcode: "M_NOT_FOUND", error: "Account data not found." },
+                    };
+                }
+            },
+            { overwriteRoutes: true },
+        );
+    }
+
+    /**
+     * Send a sync response the current account data events.
+     */
+    public sendSyncResponseWithUpdatedAccountData(syncResponder: ISyncResponder): void {
+        try {
+            syncResponder.sendOrQueueSyncResponse({
+                next_batch: 1,
+                account_data: {
+                    events: Array.from(this.accountDataEvents, ([type, content]) => ({
+                        type: type,
+                        content: content,
+                    })),
+                },
+            });
+        } catch (err) {
+            // Might fail with "Cannot queue more than one /sync response" if called too often.
+            // It's ok if it fails here, the sync response is cumulative and will contain
+            // the latest account data.
+        }
+    }
+}

spec/test-utils/test-utils.ts

@@ -315,6 +315,7 @@ export interface IMessageOpts {
     event?: boolean;
     relatesTo?: IEventRelation;
     ts?: number;
+    unsigned?: IUnsigned;
 }
 
 /**
@@ -537,8 +538,6 @@ export async function awaitDecryption(
     });
 }
 
-export const emitPromise = (e: EventEmitter, k: string): Promise<any> => new Promise((r) => e.once(k, r));
-
 export const mkPusher = (extra: Partial<IPusher> = {}): IPusher => ({
     app_display_name: "app",
     app_id: "123",
@@ -561,3 +560,25 @@ CRYPTO_BACKENDS["rust-sdk"] = (client: MatrixClient) => client.initRustCrypto();
 if (global.Olm) {
     CRYPTO_BACKENDS["libolm"] = (client: MatrixClient) => client.initCrypto();
 }
+
+export const emitPromise = (e: EventEmitter, k: string): Promise<any> => new Promise((r) => e.once(k, r));
+
+/**
+ * Advance the fake timers in a loop until the given promise resolves or rejects.
+ *
+ * Returns the result of the promise.
+ *
+ * This can be useful when there are multiple steps in the code which require an iteration of the event loop.
+ */
+export async function advanceTimersUntil<T>(promise: Promise<T>): Promise<T> {
+    let resolved = false;
+    promise.finally(() => {
+        resolved = true;
+    });
+
+    while (!resolved) {
+        await jest.advanceTimersByTimeAsync(1);
+    }
+
+    return await promise;
+}

spec/test-utils/thread.ts

@@ -161,3 +161,23 @@ export const mkThread = ({
 
     return { thread, rootEvent, events };
 };
+
+/**
+ * Create a thread, and make sure the events are added to the thread and the
+ * room's timeline as if they came in via sync.
+ *
+ * Note that mkThread doesn't actually add the events properly to the room.
+ */
+export const populateThread = ({
+    room,
+    client,
+    authorId,
+    participantUserIds,
+    length = 2,
+    ts = 1,
+}: MakeThreadProps): MakeThreadResult => {
+    const ret = mkThread({ room, client, authorId, participantUserIds, length, ts });
+    ret.thread.initialEventsFetched = true;
+    room.addLiveEvents(ret.events);
+    return ret;
+};

spec/unit/base64.spec.ts

@@ -0,0 +1,88 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { TextEncoder, TextDecoder } from "util";
+import NodeBuffer from "node:buffer";
+
+import { decodeBase64, encodeBase64, encodeUnpaddedBase64, encodeUnpaddedBase64Url } from "../../src/base64";
+
+describe.each(["browser", "node"])("Base64 encoding (%s)", (env) => {
+    let origBuffer = Buffer;
+
+    beforeAll(() => {
+        if (env === "browser") {
+            origBuffer = Buffer;
+            // @ts-ignore
+            // eslint-disable-next-line no-global-assign
+            Buffer = undefined;
+
+            global.atob = NodeBuffer.atob;
+            global.btoa = NodeBuffer.btoa;
+        }
+    });
+
+    afterAll(() => {
+        // eslint-disable-next-line no-global-assign
+        Buffer = origBuffer;
+        // @ts-ignore
+        global.atob = undefined;
+        // @ts-ignore
+        global.btoa = undefined;
+    });
+
+    it("Should decode properly encoded data", () => {
+        const decoded = new TextDecoder().decode(decodeBase64("ZW5jb2RpbmcgaGVsbG8gd29ybGQ="));
+
+        expect(decoded).toStrictEqual("encoding hello world");
+    });
+
+    it("Should encode unpadded URL-safe base64", () => {
+        const toEncode = "?????";
+        const data = new TextEncoder().encode(toEncode);
+
+        const encoded = encodeUnpaddedBase64Url(data);
+        expect(encoded).toEqual("Pz8_Pz8");
+    });
+
+    it("Should decode URL-safe base64", () => {
+        const decoded = new TextDecoder().decode(decodeBase64("Pz8_Pz8="));
+
+        expect(decoded).toStrictEqual("?????");
+    });
+
+    it("Encode unpadded should not have padding", () => {
+        const toEncode = "encoding hello world";
+        const data = new TextEncoder().encode(toEncode);
+
+        const paddedEncoded = encodeBase64(data);
+        const unpaddedEncoded = encodeUnpaddedBase64(data);
+
+        expect(paddedEncoded).not.toEqual(unpaddedEncoded);
+
+        const padding = paddedEncoded.charAt(paddedEncoded.length - 1);
+        expect(padding).toStrictEqual("=");
+    });
+
+    it("Decode should be indifferent to padding", () => {
+        const withPadding = "ZW5jb2RpbmcgaGVsbG8gd29ybGQ=";
+        const withoutPadding = "ZW5jb2RpbmcgaGVsbG8gd29ybGQ";
+
+        const decodedPad = decodeBase64(withPadding);
+        const decodedNoPad = decodeBase64(withoutPadding);
+
+        expect(decodedPad).toStrictEqual(decodedNoPad);
+    });
+});

spec/unit/crypto.spec.ts

@@ -115,6 +115,16 @@ describe("Crypto", function () {
         expect(Crypto.getOlmVersion()[0]).toEqual(3);
     });
 
+    it("getVersion() should return the current version of the olm library", async () => {
+        const client = new TestClient("@alice:example.com", "deviceid").client;
+        await client.initCrypto();
+
+        const olmVersionTuple = Crypto.getOlmVersion();
+        expect(client.getCrypto()?.getVersion()).toBe(
+            `Olm ${olmVersionTuple[0]}.${olmVersionTuple[1]}.${olmVersionTuple[2]}`,
+        );
+    });
+
     describe("encrypted events", function () {
         it("provides encryption information for events from unverified senders", async function () {
             const client = new TestClient("@alice:example.com", "deviceid").client;
@@ -346,7 +356,6 @@ describe("Crypto", function () {
 
         let crypto: Crypto;
         let mockBaseApis: MatrixClient;
-        let mockRoomList: RoomList;
 
         let fakeEmitter: EventEmitter;
 
@@ -380,19 +389,10 @@ describe("Crypto", function () {
                 isGuest: jest.fn(),
                 emit: jest.fn(),
             } as unknown as MatrixClient;
-            mockRoomList = {} as unknown as RoomList;
 
             fakeEmitter = new EventEmitter();
 
-            crypto = new Crypto(
-                mockBaseApis,
-                "@alice:home.server",
-                "FLIBBLE",
-                clientStore,
-                cryptoStore,
-                mockRoomList,
-                [],
-            );
+            crypto = new Crypto(mockBaseApis, "@alice:home.server", "FLIBBLE", clientStore, cryptoStore, []);
             crypto.registerEventHandlers(fakeEmitter as any);
             await crypto.init();
         });
@@ -1331,15 +1331,9 @@ describe("Crypto", function () {
                 setRoomEncryption: jest.fn().mockResolvedValue(undefined),
             } as unknown as RoomList;
 
-            crypto = new Crypto(
-                mockClient,
-                "@alice:home.server",
-                "FLIBBLE",
-                clientStore,
-                cryptoStore,
-                mockRoomList,
-                [],
-            );
+            crypto = new Crypto(mockClient, "@alice:home.server", "FLIBBLE", clientStore, cryptoStore, []);
+            // @ts-ignore we are injecting a mock into a private property
+            crypto.roomList = mockRoomList;
         });
 
         it("should set the algorithm if called for a known room", async () => {

spec/unit/crypto/base64.spec.ts

@@ -1,52 +0,0 @@
-/*
-Copyright 2023 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import { TextEncoder, TextDecoder } from "util";
-
-import { decodeBase64, encodeBase64, encodeUnpaddedBase64 } from "../../../src/common-crypto/base64";
-
-describe("Crypto Base64 encoding", () => {
-    it("Should decode properly encoded data", async () => {
-        const toEncode = "encoding hello world";
-        const encoded = encodeBase64(new TextEncoder().encode(toEncode));
-        const decoded = new TextDecoder().decode(decodeBase64(encoded));
-
-        expect(decoded).toStrictEqual(toEncode);
-    });
-
-    it("Encode unpadded should not have padding", async () => {
-        const toEncode = "encoding hello world";
-        const data = new TextEncoder().encode(toEncode);
-
-        const paddedEncoded = encodeBase64(data);
-        const unpaddedEncoded = encodeUnpaddedBase64(data);
-
-        expect(paddedEncoded).not.toEqual(unpaddedEncoded);
-
-        const padding = paddedEncoded.charAt(paddedEncoded.length - 1);
-        expect(padding).toStrictEqual("=");
-    });
-
-    it("Decode should be indifferent to padding", async () => {
-        const withPadding = "ZW5jb2RpbmcgaGVsbG8gd29ybGQ=";
-        const withoutPadding = "ZW5jb2RpbmcgaGVsbG8gd29ybGQ";
-
-        const decodedPad = decodeBase64(withPadding);
-        const decodedNoPad = decodeBase64(withoutPadding);
-
-        expect(decodedPad).toStrictEqual(decodedNoPad);
-    });
-});

spec/unit/crypto/secrets.spec.ts

@@ -28,6 +28,7 @@ import { DeviceInfo } from "../../../src/crypto/deviceinfo";
 import { ISignatures } from "../../../src/@types/signed";
 import { ICurve25519AuthData } from "../../../src/crypto/keybackup";
 import { SecretStorageKeyDescription, SECRET_STORAGE_ALGORITHM_V1_AES } from "../../../src/secret-storage";
+import { decodeBase64 } from "../../../src/base64";
 
 async function makeTestClient(
     userInfo: { userId: string; deviceId: string },
@@ -275,13 +276,13 @@ describe("Secrets", function () {
 
     describe("bootstrap", function () {
         // keys used in some of the tests
-        const XSK = new Uint8Array(olmlib.decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="));
+        const XSK = new Uint8Array(decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="));
         const XSPubKey = "DRb8pFVJyEJ9OWvXeUoM0jq/C2Wt+NxzBZVuk2nRb+0";
-        const USK = new Uint8Array(olmlib.decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="));
+        const USK = new Uint8Array(decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="));
         const USPubKey = "CUpoiTtHiyXpUmd+3ohb7JVxAlUaOG1NYs9Jlx8soQU";
-        const SSK = new Uint8Array(olmlib.decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="));
+        const SSK = new Uint8Array(decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="));
         const SSPubKey = "0DfNsRDzEvkCLA0gD3m7VAGJ5VClhjEsewI35xq873Q";
-        const SSSSKey = new Uint8Array(olmlib.decodeBase64("XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0="));
+        const SSSSKey = new Uint8Array(decodeBase64("XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0="));
 
         it("bootstraps when no storage or cross-signing keys locally", async function () {
             const key = new Uint8Array(16);

spec/unit/crypto/verification/secret_request.spec.ts

@@ -16,7 +16,7 @@ limitations under the License.
 
 import "../../../olm-loader";
 import { MatrixClient, MatrixEvent } from "../../../../src/matrix";
-import { encodeBase64 } from "../../../../src/crypto/olmlib";
+import { encodeBase64 } from "../../../../src/base64";
 import "../../../../src/crypto"; // import this to cycle-break
 import { CrossSigningInfo } from "../../../../src/crypto/CrossSigning";
 import { VerificationRequest } from "../../../../src/crypto/verification/request/VerificationRequest";

spec/unit/event-mapper.spec.ts

@@ -34,6 +34,7 @@ describe("eventMapperFor", function () {
                 getRoom(roomId: string): Room | null {
                     return rooms.find((r) => r.roomId === roomId) ?? null;
                 },
+                setUserCreator(_) {},
             } as IStore,
             scheduler: {
                 setProcessFunction: jest.fn(),

spec/unit/matrix-client.spec.ts

@@ -328,6 +328,7 @@ describe("MatrixClient", function () {
                 "storeFilter",
                 "startup",
                 "deleteAllData",
+                "setUserCreator",
             ] as const
         ).reduce((r, k) => {
             r[k] = jest.fn();

spec/unit/matrixrtc/CallMembership.spec.ts

@@ -85,7 +85,7 @@ describe("CallMembership", () => {
 
     it("considers memberships expired when local age large", () => {
         const fakeEvent = makeMockEvent(1000);
-        fakeEvent.getLocalAge = jest.fn().mockReturnValue(6000);
+        fakeEvent.localTimestamp = Date.now() - 6000;
         const membership = new CallMembership(fakeEvent, membershipTemplate);
         expect(membership.isExpired()).toEqual(true);
     });

spec/unit/matrixrtc/MatrixRTCSession.spec.ts

@@ -14,11 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { EventTimeline, EventType, MatrixClient, Room } from "../../../src";
+import { EventTimeline, EventType, MatrixClient, MatrixError, MatrixEvent, Room } from "../../../src";
 import { CallMembershipData } from "../../../src/matrixrtc/CallMembership";
 import { MatrixRTCSession, MatrixRTCSessionEvent } from "../../../src/matrixrtc/MatrixRTCSession";
+import { EncryptionKeysEventContent } from "../../../src/matrixrtc/types";
 import { randomString } from "../../../src/randomstring";
-import { makeMockRoom, mockRTCEvent } from "./mocks";
+import { makeMockRoom, makeMockRoomState, mockRTCEvent } from "./mocks";
 
 const membershipTemplate: CallMembershipData = {
     call_id: "",
@@ -59,13 +60,14 @@ describe("MatrixRTCSession", () => {
         expect(sess?.memberships[0].deviceId).toEqual("AAAAAAA");
         expect(sess?.memberships[0].membershipID).toEqual("bloop");
         expect(sess?.memberships[0].isExpired()).toEqual(false);
+        expect(sess?.callId).toEqual("");
     });
 
     it("ignores expired memberships events", () => {
         const expiredMembership = Object.assign({}, membershipTemplate);
         expiredMembership.expires = 1000;
         expiredMembership.device_id = "EXPIRED";
-        const mockRoom = makeMockRoom([membershipTemplate, expiredMembership], () => 10000);
+        const mockRoom = makeMockRoom([membershipTemplate, expiredMembership], 10000);
 
         sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
         expect(sess?.memberships.length).toEqual(1);
@@ -184,8 +186,15 @@ describe("MatrixRTCSession", () => {
 
     describe("joining", () => {
         let mockRoom: Room;
+        let sendStateEventMock: jest.Mock;
+        let sendEventMock: jest.Mock;
 
         beforeEach(() => {
+            sendStateEventMock = jest.fn();
+            sendEventMock = jest.fn();
+            client.sendStateEvent = sendStateEventMock;
+            client.sendEvent = sendEventMock;
+
             mockRoom = makeMockRoom([]);
             sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
         });
@@ -205,8 +214,6 @@ describe("MatrixRTCSession", () => {
         });
 
         it("sends a membership event when joining a call", () => {
-            client.sendStateEvent = jest.fn();
-
             sess!.joinRoomSession([mockFocus]);
 
             expect(client.sendStateEvent).toHaveBeenCalledWith(
@@ -230,9 +237,6 @@ describe("MatrixRTCSession", () => {
         });
 
         it("does nothing if join called when already joined", () => {
-            const sendStateEventMock = jest.fn();
-            client.sendStateEvent = sendStateEventMock;
-
             sess!.joinRoomSession([mockFocus]);
 
             expect(client.sendStateEvent).toHaveBeenCalledTimes(1);
@@ -262,7 +266,7 @@ describe("MatrixRTCSession", () => {
                 const timeElapsed = 60 * 60 * 1000 - 1000;
                 mockRoom.getLiveTimeline().getState(EventTimeline.FORWARDS)!.getStateEvents = jest
                     .fn()
-                    .mockReturnValue(mockRTCEvent(eventContent.memberships, mockRoom.roomId, () => timeElapsed));
+                    .mockReturnValue(mockRTCEvent(eventContent.memberships, mockRoom.roomId, timeElapsed));
 
                 const eventReSentPromise = new Promise<Record<string, any>>((r) => {
                     resolveFn = (_roomId: string, _type: string, val: Record<string, any>) => {
@@ -299,15 +303,244 @@ describe("MatrixRTCSession", () => {
                 jest.useRealTimers();
             }
         });
+
+        it("creates a key when joining", () => {
+            sess!.joinRoomSession([mockFocus], true);
+            const keys = sess?.getKeysForParticipant("@alice:example.org", "AAAAAAA");
+            expect(keys).toHaveLength(1);
+
+            const allKeys = sess!.getEncryptionKeys();
+            expect(allKeys).toBeTruthy();
+            expect(Array.from(allKeys)).toHaveLength(1);
+        });
+
+        it("sends keys when joining", async () => {
+            const eventSentPromise = new Promise((resolve) => {
+                sendEventMock.mockImplementation(resolve);
+            });
+
+            sess!.joinRoomSession([mockFocus], true);
+
+            await eventSentPromise;
+
+            expect(sendEventMock).toHaveBeenCalledWith(expect.stringMatching(".*"), "io.element.call.encryption_keys", {
+                call_id: "",
+                device_id: "AAAAAAA",
+                keys: [
+                    {
+                        index: 0,
+                        key: expect.stringMatching(".*"),
+                    },
+                ],
+            });
+        });
+
+        it("retries key sends", async () => {
+            jest.useFakeTimers();
+            let firstEventSent = false;
+
+            try {
+                const eventSentPromise = new Promise<void>((resolve) => {
+                    sendEventMock.mockImplementation(() => {
+                        if (!firstEventSent) {
+                            jest.advanceTimersByTime(10000);
+
+                            firstEventSent = true;
+                            const e = new Error() as MatrixError;
+                            e.data = {};
+                            throw e;
+                        } else {
+                            resolve();
+                        }
+                    });
+                });
+
+                sess!.joinRoomSession([mockFocus], true);
+                jest.advanceTimersByTime(10000);
+
+                await eventSentPromise;
+
+                expect(sendEventMock).toHaveBeenCalledTimes(2);
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+
+        it("cancels key send event that fail", async () => {
+            const eventSentinel = {} as unknown as MatrixEvent;
+
+            client.cancelPendingEvent = jest.fn();
+            sendEventMock.mockImplementation(() => {
+                const e = new Error() as MatrixError;
+                e.data = {};
+                e.event = eventSentinel;
+                throw e;
+            });
+
+            sess!.joinRoomSession([mockFocus], true);
+
+            expect(client.cancelPendingEvent).toHaveBeenCalledWith(eventSentinel);
+        });
+
+        it("Re-sends key if a new member joins", async () => {
+            jest.useFakeTimers();
+            try {
+                const mockRoom = makeMockRoom([membershipTemplate]);
+                sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+                const keysSentPromise1 = new Promise((resolve) => {
+                    sendEventMock.mockImplementation(resolve);
+                });
+
+                sess.joinRoomSession([mockFocus], true);
+                await keysSentPromise1;
+
+                sendEventMock.mockClear();
+                jest.advanceTimersByTime(10000);
+
+                const keysSentPromise2 = new Promise((resolve) => {
+                    sendEventMock.mockImplementation(resolve);
+                });
+
+                const onMembershipsChanged = jest.fn();
+                sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
+
+                const member2 = Object.assign({}, membershipTemplate, {
+                    device_id: "BBBBBBB",
+                });
+
+                mockRoom.getLiveTimeline().getState = jest
+                    .fn()
+                    .mockReturnValue(makeMockRoomState([membershipTemplate, member2], mockRoom.roomId, undefined));
+                sess.onMembershipUpdate();
+
+                await keysSentPromise2;
+
+                expect(sendEventMock).toHaveBeenCalled();
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+
+        it("Rotates key if a member leaves", async () => {
+            jest.useFakeTimers();
+            try {
+                const member2 = Object.assign({}, membershipTemplate, {
+                    device_id: "BBBBBBB",
+                });
+                const mockRoom = makeMockRoom([membershipTemplate, member2]);
+                sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+                const onMyEncryptionKeyChanged = jest.fn();
+                sess.on(
+                    MatrixRTCSessionEvent.EncryptionKeyChanged,
+                    (_key: Uint8Array, _idx: number, participantId: string) => {
+                        if (participantId === `${client.getUserId()}:${client.getDeviceId()}`) {
+                            onMyEncryptionKeyChanged();
+                        }
+                    },
+                );
+
+                const keysSentPromise1 = new Promise<EncryptionKeysEventContent>((resolve) => {
+                    sendEventMock.mockImplementation((_roomId, _evType, payload) => resolve(payload));
+                });
+
+                sess.joinRoomSession([mockFocus], true);
+                const firstKeysPayload = await keysSentPromise1;
+                expect(firstKeysPayload.keys).toHaveLength(1);
+
+                sendEventMock.mockClear();
+
+                const keysSentPromise2 = new Promise<EncryptionKeysEventContent>((resolve) => {
+                    sendEventMock.mockImplementation((_roomId, _evType, payload) => resolve(payload));
+                });
+
+                mockRoom.getLiveTimeline().getState = jest
+                    .fn()
+                    .mockReturnValue(makeMockRoomState([membershipTemplate], mockRoom.roomId, undefined));
+                sess.onMembershipUpdate();
+
+                jest.advanceTimersByTime(10000);
+
+                const secondKeysPayload = await keysSentPromise2;
+
+                expect(secondKeysPayload.keys).toHaveLength(2);
+                expect(onMyEncryptionKeyChanged).toHaveBeenCalledTimes(2);
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+
+        it("Doesn't re-send key immediately", async () => {
+            const realSetImmediate = setImmediate;
+            jest.useFakeTimers();
+            try {
+                const mockRoom = makeMockRoom([membershipTemplate]);
+                sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+                const keysSentPromise1 = new Promise((resolve) => {
+                    sendEventMock.mockImplementation(resolve);
+                });
+
+                sess.joinRoomSession([mockFocus], true);
+                await keysSentPromise1;
+
+                sendEventMock.mockClear();
+
+                const onMembershipsChanged = jest.fn();
+                sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
+
+                const member2 = Object.assign({}, membershipTemplate, {
+                    device_id: "BBBBBBB",
+                });
+
+                mockRoom.getLiveTimeline().getState = jest
+                    .fn()
+                    .mockReturnValue(makeMockRoomState([membershipTemplate, member2], mockRoom.roomId, undefined));
+                sess.onMembershipUpdate();
+
+                await new Promise((resolve) => {
+                    realSetImmediate(resolve);
+                });
+
+                expect(sendEventMock).not.toHaveBeenCalled();
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+    });
+
+    it("Does not emits if no membership changes", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+        const onMembershipsChanged = jest.fn();
+        sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
+        sess.onMembershipUpdate();
+
+        expect(onMembershipsChanged).not.toHaveBeenCalled();
+    });
+
+    it("Emits on membership changes", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+        const onMembershipsChanged = jest.fn();
+        sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
+
+        mockRoom.getLiveTimeline().getState = jest
+            .fn()
+            .mockReturnValue(makeMockRoomState([], mockRoom.roomId, undefined));
+        sess.onMembershipUpdate();
+
+        expect(onMembershipsChanged).toHaveBeenCalled();
     });
 
     it("emits an event at the time a membership event expires", () => {
         jest.useFakeTimers();
         try {
-            let eventAge = 0;
-
             const membership = Object.assign({}, membershipTemplate);
-            const mockRoom = makeMockRoom([membership], () => eventAge);
+            const mockRoom = makeMockRoom([membership], 0);
 
             sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
             const membershipObject = sess.memberships[0];
@@ -315,7 +548,6 @@ describe("MatrixRTCSession", () => {
             const onMembershipsChanged = jest.fn();
             sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
 
-            eventAge = 61 * 1000 * 1000;
             jest.advanceTimersByTime(61 * 1000 * 1000);
 
             expect(onMembershipsChanged).toHaveBeenCalledWith([membershipObject], []);
@@ -326,47 +558,49 @@ describe("MatrixRTCSession", () => {
     });
 
     it("prunes expired memberships on update", () => {
-        client.sendStateEvent = jest.fn();
+        jest.useFakeTimers();
+        try {
+            client.sendStateEvent = jest.fn();
 
-        let eventAge = 0;
-
-        const mockRoom = makeMockRoom(
-            [
+            const mockMemberships = [
                 Object.assign({}, membershipTemplate, {
                     device_id: "OTHERDEVICE",
                     expires: 1000,
                 }),
-            ],
-            () => eventAge,
-        );
-        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+            ];
+            const mockRoomNoExpired = makeMockRoom(mockMemberships, 0);
 
-        // sanity check
-        expect(sess.memberships).toHaveLength(1);
-        expect(sess.memberships[0].deviceId).toEqual("OTHERDEVICE");
+            sess = MatrixRTCSession.roomSessionForRoom(client, mockRoomNoExpired);
 
-        eventAge = 10000;
+            // sanity check
+            expect(sess.memberships).toHaveLength(1);
+            expect(sess.memberships[0].deviceId).toEqual("OTHERDEVICE");
 
-        sess.joinRoomSession([mockFocus]);
+            jest.advanceTimersByTime(10000);
 
-        expect(client.sendStateEvent).toHaveBeenCalledWith(
-            mockRoom!.roomId,
-            EventType.GroupCallMemberPrefix,
-            {
-                memberships: [
-                    {
-                        application: "m.call",
-                        scope: "m.room",
-                        call_id: "",
-                        device_id: "AAAAAAA",
-                        expires: 3600000,
-                        foci_active: [mockFocus],
-                        membershipID: expect.stringMatching(".*"),
-                    },
-                ],
-            },
-            "@alice:example.org",
-        );
+            sess.joinRoomSession([mockFocus]);
+
+            expect(client.sendStateEvent).toHaveBeenCalledWith(
+                mockRoomNoExpired!.roomId,
+                EventType.GroupCallMemberPrefix,
+                {
+                    memberships: [
+                        {
+                            application: "m.call",
+                            scope: "m.room",
+                            call_id: "",
+                            device_id: "AAAAAAA",
+                            expires: 3600000,
+                            foci_active: [mockFocus],
+                            membershipID: expect.stringMatching(".*"),
+                        },
+                    ],
+                },
+                "@alice:example.org",
+            );
+        } finally {
+            jest.useRealTimers();
+        }
     });
 
     it("fills in created_ts for other memberships on update", () => {
@@ -409,4 +643,76 @@ describe("MatrixRTCSession", () => {
             "@alice:example.org",
         );
     });
+
+    it("collects keys from encryption events", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        sess.onCallEncryption({
+            getType: jest.fn().mockReturnValue("io.element.call.encryption_keys"),
+            getContent: jest.fn().mockReturnValue({
+                device_id: "bobsphone",
+                call_id: "",
+                keys: [
+                    {
+                        index: 0,
+                        key: "dGhpcyBpcyB0aGUga2V5",
+                    },
+                ],
+            }),
+            getSender: jest.fn().mockReturnValue("@bob:example.org"),
+        } as unknown as MatrixEvent);
+
+        const bobKeys = sess.getKeysForParticipant("@bob:example.org", "bobsphone")!;
+        expect(bobKeys).toHaveLength(1);
+        expect(bobKeys[0]).toEqual(Buffer.from("this is the key", "utf-8"));
+    });
+
+    it("collects keys at non-zero indices", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        sess.onCallEncryption({
+            getType: jest.fn().mockReturnValue("io.element.call.encryption_keys"),
+            getContent: jest.fn().mockReturnValue({
+                device_id: "bobsphone",
+                call_id: "",
+                keys: [
+                    {
+                        index: 4,
+                        key: "dGhpcyBpcyB0aGUga2V5",
+                    },
+                ],
+            }),
+            getSender: jest.fn().mockReturnValue("@bob:example.org"),
+        } as unknown as MatrixEvent);
+
+        const bobKeys = sess.getKeysForParticipant("@bob:example.org", "bobsphone")!;
+        expect(bobKeys).toHaveLength(5);
+        expect(bobKeys[0]).toBeFalsy();
+        expect(bobKeys[1]).toBeFalsy();
+        expect(bobKeys[2]).toBeFalsy();
+        expect(bobKeys[3]).toBeFalsy();
+        expect(bobKeys[4]).toEqual(Buffer.from("this is the key", "utf-8"));
+    });
+
+    it("ignores keys event for the local participant", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        sess.onCallEncryption({
+            getType: jest.fn().mockReturnValue("io.element.call.encryption_keys"),
+            getContent: jest.fn().mockReturnValue({
+                device_id: client.getDeviceId(),
+                call_id: "",
+                keys: [
+                    {
+                        index: 4,
+                        key: "dGhpcyBpcyB0aGUga2V5",
+                    },
+                ],
+            }),
+            getSender: jest.fn().mockReturnValue(client.getUserId()),
+        } as unknown as MatrixEvent);
+
+        const myKeys = sess.getKeysForParticipant(client.getUserId()!, client.getDeviceId()!)!;
+        expect(myKeys).toBeFalsy();
+    });
 });

spec/unit/matrixrtc/MatrixRTCSessionManager.spec.ts

@@ -14,7 +14,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { ClientEvent, EventTimeline, MatrixClient } from "../../../src";
+import {
+    ClientEvent,
+    EventTimeline,
+    EventType,
+    IRoomTimelineData,
+    MatrixClient,
+    MatrixEvent,
+    RoomEvent,
+} from "../../../src";
 import { RoomStateEvent } from "../../../src/models/room-state";
 import { CallMembershipData } from "../../../src/matrixrtc/CallMembership";
 import { MatrixRTCSessionManagerEvents } from "../../../src/matrixrtc/MatrixRTCSessionManager";
@@ -78,4 +86,26 @@ describe("MatrixRTCSessionManager", () => {
 
         expect(onEnded).toHaveBeenCalledWith(room1.roomId, client.matrixRTC.getActiveRoomSession(room1));
     });
+
+    it("Calls onCallEncryption on encryption keys event", () => {
+        const room1 = makeMockRoom([membershipTemplate]);
+        jest.spyOn(client, "getRooms").mockReturnValue([room1]);
+        jest.spyOn(client, "getRoom").mockReturnValue(room1);
+
+        client.emit(ClientEvent.Room, room1);
+        const onCallEncryptionMock = jest.fn();
+        client.matrixRTC.getRoomSession(room1).onCallEncryption = onCallEncryptionMock;
+
+        const timelineEvent = {
+            getType: jest.fn().mockReturnValue(EventType.CallEncryptionKeysPrefix),
+            getContent: jest.fn().mockReturnValue({}),
+            getSender: jest.fn().mockReturnValue("@mock:user.example"),
+            getRoomId: jest.fn().mockReturnValue("!room:id"),
+            sender: {
+                userId: "@mock:user.example",
+            },
+        } as unknown as MatrixEvent;
+        client.emit(RoomEvent.Timeline, timelineEvent, undefined, undefined, false, {} as IRoomTimelineData);
+        expect(onCallEncryptionMock).toHaveBeenCalled();
+    });
 });

spec/unit/matrixrtc/mocks.ts

@@ -18,37 +18,29 @@ import { EventType, MatrixEvent, Room } from "../../../src";
 import { CallMembershipData } from "../../../src/matrixrtc/CallMembership";
 import { randomString } from "../../../src/randomstring";
 
-export function makeMockRoom(
-    memberships: CallMembershipData[],
-    getLocalAge: (() => number) | undefined = undefined,
-): Room {
+export function makeMockRoom(memberships: CallMembershipData[], localAge: number | null = null): Room {
     const roomId = randomString(8);
+    // Caching roomState here so it does not get recreated when calling `getLiveTimeline.getState()`
+    const roomState = makeMockRoomState(memberships, roomId, localAge);
     return {
         roomId: roomId,
         getLiveTimeline: jest.fn().mockReturnValue({
-            getState: jest.fn().mockReturnValue(makeMockRoomState(memberships, roomId, getLocalAge)),
+            getState: jest.fn().mockReturnValue(roomState),
         }),
     } as unknown as Room;
 }
 
-function makeMockRoomState(memberships: CallMembershipData[], roomId: string, getLocalAge: (() => number) | undefined) {
+export function makeMockRoomState(memberships: CallMembershipData[], roomId: string, localAge: number | null = null) {
+    const event = mockRTCEvent(memberships, roomId, localAge);
     return {
         getStateEvents: (_: string, stateKey: string) => {
-            const event = mockRTCEvent(memberships, roomId, getLocalAge);
-
             if (stateKey !== undefined) return event;
             return [event];
         },
     };
 }
 
-export function mockRTCEvent(
-    memberships: CallMembershipData[],
-    roomId: string,
-    getLocalAge: (() => number) | undefined,
-): MatrixEvent {
-    const getLocalAgeFn = getLocalAge ?? (() => 10);
-
+export function mockRTCEvent(memberships: CallMembershipData[], roomId: string, localAge: number | null): MatrixEvent {
     return {
         getType: jest.fn().mockReturnValue(EventType.GroupCallMemberPrefix),
         getContent: jest.fn().mockReturnValue({
@@ -56,8 +48,7 @@ export function mockRTCEvent(
         }),
         getSender: jest.fn().mockReturnValue("@mock:user.example"),
         getTs: jest.fn().mockReturnValue(1000),
-        getLocalAge: getLocalAgeFn,
-        localTimestamp: Date.now(),
+        localTimestamp: Date.now() - (localAge ?? 10),
         getRoomId: jest.fn().mockReturnValue(roomId),
         sender: {
             userId: "@mock:user.example",

spec/unit/models/event.spec.ts

@@ -14,10 +14,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { MockedObject } from "jest-mock";
+
 import { MatrixEvent, MatrixEventEvent } from "../../../src/models/event";
 import { emitPromise } from "../../test-utils/test-utils";
 import { Crypto, IEventDecryptionResult } from "../../../src/crypto";
-import { IAnnotatedPushRule, PushRuleActionName, TweakName } from "../../../src";
+import {
+    IAnnotatedPushRule,
+    MatrixClient,
+    PushRuleActionName,
+    Room,
+    THREAD_RELATION_TYPE,
+    TweakName,
+} from "../../../src";
 
 describe("MatrixEvent", () => {
     it("should create copies of itself", () => {
@@ -61,31 +70,264 @@ describe("MatrixEvent", () => {
         expect(a.toSnapshot().isEquivalentTo(b)).toBe(false);
     });
 
-    it("should prune clearEvent when being redacted", () => {
-        const ev = new MatrixEvent({
-            type: "m.room.message",
-            content: {
-                body: "Test",
-            },
-            event_id: "$event1:server",
+    describe("redaction", () => {
+        it("should prune clearEvent when being redacted", () => {
+            const ev = createEvent("$event1:server", "Test");
+
+            expect(ev.getContent().body).toBe("Test");
+            expect(ev.getWireContent().body).toBe("Test");
+            ev.makeEncrypted("m.room.encrypted", { ciphertext: "xyz" }, "", "");
+            expect(ev.getContent().body).toBe("Test");
+            expect(ev.getWireContent().body).toBeUndefined();
+            expect(ev.getWireContent().ciphertext).toBe("xyz");
+
+            const mockClient = {} as unknown as MockedObject<MatrixClient>;
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const redaction = createRedaction(ev.getId()!);
+
+            ev.makeRedacted(redaction, room);
+            expect(ev.getContent().body).toBeUndefined();
+            expect(ev.getWireContent().body).toBeUndefined();
+            expect(ev.getWireContent().ciphertext).toBeUndefined();
         });
 
-        expect(ev.getContent().body).toBe("Test");
-        expect(ev.getWireContent().body).toBe("Test");
-        ev.makeEncrypted("m.room.encrypted", { ciphertext: "xyz" }, "", "");
-        expect(ev.getContent().body).toBe("Test");
-        expect(ev.getWireContent().body).toBeUndefined();
-        expect(ev.getWireContent().ciphertext).toBe("xyz");
+        it("should remain in the main timeline when redacted", async () => {
+            // Given an event in the main timeline
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const ev = createEvent("$event1:server");
 
-        const redaction = new MatrixEvent({
-            type: "m.room.redaction",
-            redacts: ev.getId(),
+            await room.addLiveEvents([ev]);
+            await room.createThreadsTimelineSets();
+            expect(ev.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([ev.getId()]);
+
+            // When I redact it
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then it remains in the main timeline
+            expect(ev.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([ev.getId()]);
         });
 
-        ev.makeRedacted(redaction);
-        expect(ev.getContent().body).toBeUndefined();
-        expect(ev.getWireContent().body).toBeUndefined();
-        expect(ev.getWireContent().ciphertext).toBeUndefined();
+        it("should keep thread roots in both timelines when redacted", async () => {
+            // Given a thread exists
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev]);
+            await room.createThreadsTimelineSets();
+            expect(threadRoot.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId()]);
+
+            // When I redact the thread root
+            const redaction = createRedaction(ev.getId()!);
+            threadRoot.makeRedacted(redaction, room);
+
+            // Then it remains in the main timeline and the thread
+            expect(threadRoot.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId()]);
+        });
+
+        it("should move into the main timeline when redacted", async () => {
+            // Given an event in a thread
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev]);
+            await room.createThreadsTimelineSets();
+            expect(ev.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId()]);
+
+            // When I redact it
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then it disappears from the thread and appears in the main timeline
+            expect(ev.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId(), ev.getId()]);
+            expect(threadLiveEventIds(room, 0)).not.toContain(ev.getId());
+        });
+
+        it("should move reactions to a redacted event into the main timeline", async () => {
+            // Given an event in a thread with a reaction
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+            const reaction = createReactionEvent("$reaction:server", ev.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev, reaction]);
+            await room.createThreadsTimelineSets();
+            expect(reaction.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId(), reaction.getId()]);
+
+            // When I redact the event
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then the reaction moves into the main timeline
+            expect(reaction.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId(), ev.getId(), reaction.getId()]);
+            expect(threadLiveEventIds(room, 0)).not.toContain(reaction.getId());
+        });
+
+        it("should move edits of a redacted event into the main timeline", async () => {
+            // Given an event in a thread with a reaction
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+            const edit = createEditEvent("$edit:server", ev.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev, edit]);
+            await room.createThreadsTimelineSets();
+            expect(edit.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId(), edit.getId()]);
+
+            // When I redact the event
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then the edit moves into the main timeline
+            expect(edit.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId(), ev.getId(), edit.getId()]);
+            expect(threadLiveEventIds(room, 0)).not.toContain(edit.getId());
+        });
+
+        it("should move reactions to replies to replies a redacted event into the main timeline", async () => {
+            // Given an event in a thread with a reaction
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+            const reply1 = createReplyEvent("$reply1:server", ev.getId()!);
+            const reply2 = createReplyEvent("$reply2:server", reply1.getId()!);
+            const reaction = createReactionEvent("$reaction:server", reply2.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev, reply1, reply2, reaction]);
+            await room.createThreadsTimelineSets();
+            expect(reaction.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([
+                threadRoot.getId(),
+                ev.getId(),
+                reply1.getId(),
+                reply2.getId(),
+                reaction.getId(),
+            ]);
+
+            // When I redact the event
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then the replies move to the main thread and the reaction disappears
+            expect(reaction.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([
+                threadRoot.getId(),
+                ev.getId(),
+                reply1.getId(),
+                reply2.getId(),
+                reaction.getId(),
+            ]);
+            expect(threadLiveEventIds(room, 0)).not.toContain(reply1.getId());
+            expect(threadLiveEventIds(room, 0)).not.toContain(reply2.getId());
+            expect(threadLiveEventIds(room, 0)).not.toContain(reaction.getId());
+        });
+
+        function createMockClient(): MatrixClient {
+            return {
+                supportsThreads: jest.fn().mockReturnValue(true),
+                decryptEventIfNeeded: jest.fn().mockReturnThis(),
+                getUserId: jest.fn().mockReturnValue("@user:server"),
+            } as unknown as MockedObject<MatrixClient>;
+        }
+
+        function createEvent(eventId: string, body?: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.message",
+                content: {
+                    body: body ?? eventId,
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createThreadedEvent(eventId: string, threadRootId: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.message",
+                content: {
+                    "body": eventId,
+                    "m.relates_to": {
+                        rel_type: THREAD_RELATION_TYPE.name,
+                        event_id: threadRootId,
+                    },
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createEditEvent(eventId: string, repliedToId: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.message",
+                content: {
+                    "body": "Edited",
+                    "m.new_content": {
+                        body: "Edited",
+                    },
+                    "m.relates_to": {
+                        event_id: repliedToId,
+                        rel_type: "m.replace",
+                    },
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createReplyEvent(eventId: string, repliedToId: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.message",
+                content: {
+                    "m.relates_to": {
+                        event_id: repliedToId,
+                        key: "x",
+                        rel_type: "m.in_reply_to",
+                    },
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createReactionEvent(eventId: string, reactedToId: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.reaction",
+                content: {
+                    "m.relates_to": {
+                        event_id: reactedToId,
+                        key: "x",
+                        rel_type: "m.annotation",
+                    },
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createRedaction(redactedEventid: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.redaction",
+                redacts: redactedEventid,
+            });
+        }
     });
 
     describe("applyVisibilityEvent", () => {
@@ -330,3 +572,19 @@ describe("MatrixEvent", () => {
         expect(stateEvent.threadRootId).toBeUndefined();
     });
 });
+
+function mainTimelineLiveEventIds(room: Room): Array<string> {
+    return room
+        .getLiveTimeline()
+        .getEvents()
+        .map((e) => e.getId()!);
+}
+
+function threadLiveEventIds(room: Room, threadIndex: number): Array<string> {
+    return room
+        .getThreads()
+        [threadIndex].getUnfilteredTimelineSet()
+        .getLiveTimeline()
+        .getEvents()
+        .map((e) => e.getId()!);
+}

spec/unit/models/room-receipts.spec.ts

@@ -0,0 +1,541 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { FeatureSupport, MatrixClient, MatrixEvent, ReceiptContent, THREAD_RELATION_TYPE, Thread } from "../../../src";
+import { Room } from "../../../src/models/room";
+
+/**
+ * Note, these tests check the functionality of the RoomReceipts class, but most
+ * of them access that functionality via the surrounding Room class, because a
+ * room is required for RoomReceipts to function, and this matches the pattern
+ * of how this code is used in the wild.
+ */
+describe("RoomReceipts", () => {
+    beforeAll(() => {
+        jest.replaceProperty(Thread, "hasServerSideSupport", FeatureSupport.Stable);
+    });
+
+    afterAll(() => {
+        jest.restoreAllMocks();
+    });
+
+    it("reports events unread if there are no receipts", () => {
+        // Given there are no receipts in the room
+        const room = createRoom();
+        const [event] = createEvent();
+        room.addLiveEvents([event]);
+
+        // When I ask about any event, then it is unread
+        expect(room.hasUserReadEvent(readerId, event.getId()!)).toBe(false);
+    });
+
+    it("reports events we sent as read even if there are no (real) receipts", () => {
+        // Given there are no receipts in the room
+        const room = createRoom();
+        const [event] = createEventSentBy(readerId);
+        room.addLiveEvents([event]);
+
+        // When I ask about an event I sent, it is read (because a synthetic
+        // receipt was created and stored in RoomReceipts)
+        expect(room.hasUserReadEvent(readerId, event.getId()!)).toBe(true);
+    });
+
+    it("reports read if we receive an unthreaded receipt for this event", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [event, eventId] = createEvent();
+        room.addLiveEvents([event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for this event+user
+        room.addReceipt(createReceipt(readerId, event));
+
+        // Then that event is read
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+    });
+
+    it("reports read if we receive an unthreaded receipt for a later event", () => {
+        // Given we have 2 events
+        const room = createRoom();
+        const [event1, event1Id] = createEvent();
+        const [event2] = createEvent();
+        room.addLiveEvents([event1, event2]);
+
+        // When we receive a receipt for the later event
+        room.addReceipt(createReceipt(readerId, event2));
+
+        // Then the earlier one is read
+        expect(room.hasUserReadEvent(readerId, event1Id)).toBe(true);
+    });
+
+    it("reports read for a non-live event if we receive an unthreaded receipt for a live one", () => {
+        // Given we have 2 events: one live and one old
+        const room = createRoom();
+        const [oldEvent, oldEventId] = createEvent();
+        const [liveEvent] = createEvent();
+        room.addLiveEvents([liveEvent]);
+        createOldTimeline(room, [oldEvent]);
+
+        // When we receive a receipt for the live event
+        room.addReceipt(createReceipt(readerId, liveEvent));
+
+        // Then the earlier one is read
+        expect(room.hasUserReadEvent(readerId, oldEventId)).toBe(true);
+    });
+
+    it("compares by timestamp if two events are in separate old timelines", () => {
+        // Given we have 2 events, both in old timelines, with event2 after
+        // event1 in terms of timestamps
+        const room = createRoom();
+        const [event1, event1Id] = createEvent();
+        const [event2, event2Id] = createEvent();
+        event1.event.origin_server_ts = 1;
+        event2.event.origin_server_ts = 2;
+        createOldTimeline(room, [event1]);
+        createOldTimeline(room, [event2]);
+
+        // When we receive a receipt for the older event
+        room.addReceipt(createReceipt(readerId, event1));
+
+        // Then the earlier one is read and the later one is not
+        expect(room.hasUserReadEvent(readerId, event1Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(false);
+    });
+
+    it("reports unread if we receive an unthreaded receipt for an earlier event", () => {
+        // Given we have 2 events
+        const room = createRoom();
+        const [event1] = createEvent();
+        const [event2, event2Id] = createEvent();
+        room.addLiveEvents([event1, event2]);
+
+        // When we receive a receipt for the earlier event
+        room.addReceipt(createReceipt(readerId, event1));
+
+        // Then the later one is unread
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(false);
+    });
+
+    it("reports unread if we receive an unthreaded receipt for a different user", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [event, eventId] = createEvent();
+        room.addLiveEvents([event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for another user
+        room.addReceipt(createReceipt(otherUserId, event));
+
+        // Then the event is still unread since the receipt was not for us
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // But it's read for the other person
+        expect(room.hasUserReadEvent(otherUserId, eventId)).toBe(true);
+    });
+
+    it("reports events we sent as read even if an earlier receipt arrives", () => {
+        // Given we sent an event after some other event
+        const room = createRoom();
+        const [previousEvent] = createEvent();
+        const [myEvent] = createEventSentBy(readerId);
+        room.addLiveEvents([previousEvent, myEvent]);
+
+        // And I just received a receipt for the previous event
+        room.addReceipt(createReceipt(readerId, previousEvent));
+
+        // When I ask about the event I sent, it is read (because of synthetic receipts)
+        expect(room.hasUserReadEvent(readerId, myEvent.getId()!)).toBe(true);
+    });
+
+    it("considers events after ones we sent to be unread", () => {
+        // Given we sent an event, then another event came in
+        const room = createRoom();
+        const [myEvent] = createEventSentBy(readerId);
+        const [laterEvent] = createEvent();
+        room.addLiveEvents([myEvent, laterEvent]);
+
+        // When I ask about the later event, it is unread (because it's after the synthetic receipt)
+        expect(room.hasUserReadEvent(readerId, laterEvent.getId()!)).toBe(false);
+    });
+
+    it("correctly reports readness even when receipts arrive out of order", () => {
+        // Given we have 3 events
+        const room = createRoom();
+        const [event1] = createEvent();
+        const [event2, event2Id] = createEvent();
+        const [event3, event3Id] = createEvent();
+        room.addLiveEvents([event1, event2, event3]);
+
+        // When we receive receipts for the older events out of order
+        room.addReceipt(createReceipt(readerId, event2));
+        room.addReceipt(createReceipt(readerId, event1));
+
+        // Then we correctly ignore the older receipt
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, event3Id)).toBe(false);
+    });
+
+    it("reports read if we receive a threaded receipt for this event (main)", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [event, eventId] = createEvent();
+        room.addLiveEvents([event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for this event+user
+        room.addReceipt(createThreadedReceipt(readerId, event, "main"));
+
+        // Then that event is read
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+    });
+
+    it("reports read if we receive a threaded receipt for this event (non-main)", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event, eventId] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for this event on this thread
+        room.addReceipt(createThreadedReceipt(readerId, event, rootId));
+
+        // Then that event is read
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+    });
+
+    it("reports read if we receive an threaded receipt for a later event", () => {
+        // Given we have 2 events in a thread
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event1, event1Id] = createThreadedEvent(root);
+        const [event2] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event1, event2]);
+
+        // When we receive a receipt for the later event
+        room.addReceipt(createThreadedReceipt(readerId, event2, rootId));
+
+        // Then the earlier one is read
+        expect(room.hasUserReadEvent(readerId, event1Id)).toBe(true);
+    });
+
+    it("reports unread if we receive an threaded receipt for an earlier event", () => {
+        // Given we have 2 events in a thread
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event1] = createThreadedEvent(root);
+        const [event2, event2Id] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event1, event2]);
+
+        // When we receive a receipt for the earlier event
+        room.addReceipt(createThreadedReceipt(readerId, event1, rootId));
+
+        // Then the later one is unread
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(false);
+    });
+
+    it("reports unread if we receive an threaded receipt for a different user", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event, eventId] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for another user
+        room.addReceipt(createThreadedReceipt(otherUserId, event, rootId));
+
+        // Then the event is still unread since the receipt was not for us
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // But it's read for the other person
+        expect(room.hasUserReadEvent(otherUserId, eventId)).toBe(true);
+    });
+
+    it("reports unread if we receive a receipt for a later event in a different thread", () => {
+        // Given 2 events exist in different threads
+        const room = createRoom();
+        const [root1] = createEvent();
+        const [root2] = createEvent();
+        const [thread1, thread1Id] = createThreadedEvent(root1);
+        const [thread2] = createThreadedEvent(root2);
+        setupThread(room, root1);
+        setupThread(room, root2);
+        room.addLiveEvents([root1, root2, thread1, thread2]);
+
+        // When we receive a receipt for the later event
+        room.addReceipt(createThreadedReceipt(readerId, thread2, root2.getId()!));
+
+        // Then the old one is still unread since the receipt was not for this thread
+        expect(room.hasUserReadEvent(readerId, thread1Id)).toBe(false);
+    });
+
+    it("correctly reports readness even when threaded receipts arrive out of order", () => {
+        // Given we have 3 events
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event1] = createThreadedEvent(root);
+        const [event2, event2Id] = createThreadedEvent(root);
+        const [event3, event3Id] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event1, event2, event3]);
+
+        // When we receive receipts for the older events out of order
+        room.addReceipt(createThreadedReceipt(readerId, event2, rootId));
+        room.addReceipt(createThreadedReceipt(readerId, event1, rootId));
+
+        // Then we correctly ignore the older receipt
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, event3Id)).toBe(false);
+    });
+
+    it("correctly reports readness when mixing threaded and unthreaded receipts", () => {
+        // Given we have a setup from this presentation:
+        // https://docs.google.com/presentation/d/1H1gxRmRFAm8d71hCILWmpOYezsvdlb7cB6ANl-20Gns/edit?usp=sharing
+        //
+        //                     Main1----\
+        //                       |       ---Thread1a <- threaded receipt
+        //                       |             |
+        //                       |          Thread1b
+        // threaded receipt -> Main2--\
+        //                       |     ----------------Thread2a <- unthreaded receipt
+        //                     Main3                      |
+        //                                             Thread2b <- threaded receipt
+        //
+        const room = createRoom();
+        const [main1, main1Id] = createEvent();
+        const [main2, main2Id] = createEvent();
+        const [main3, main3Id] = createEvent();
+        const [thread1a, thread1aId] = createThreadedEvent(main1);
+        const [thread1b, thread1bId] = createThreadedEvent(main1);
+        const [thread2a, thread2aId] = createThreadedEvent(main2);
+        const [thread2b, thread2bId] = createThreadedEvent(main2);
+        setupThread(room, main1);
+        setupThread(room, main2);
+        room.addLiveEvents([main1, thread1a, thread1b, main2, thread2a, main3, thread2b]);
+
+        // And the timestamps on the events are consistent with the order above
+        main1.event.origin_server_ts = 1;
+        thread1a.event.origin_server_ts = 2;
+        thread1b.event.origin_server_ts = 3;
+        main2.event.origin_server_ts = 4;
+        thread2a.event.origin_server_ts = 5;
+        main3.event.origin_server_ts = 6;
+        thread2b.event.origin_server_ts = 7;
+        // (Note: in principle, we have the information needed to order these
+        // events without using their timestamps, since they all came in via
+        // addLiveEvents. In reality, some of them would have come in via the
+        // /relations API, making it impossible to get the correct ordering
+        // without MSC4033, which is why we fall back to timestamps. I.e. we
+        // definitely could fix the code to make the above
+        // timestamp-manipulation unnecessary, but it would only make this test
+        // neater, not actually help in the real world.)
+
+        // When the receipts arrive
+        room.addReceipt(createThreadedReceipt(readerId, main2, "main"));
+        room.addReceipt(createThreadedReceipt(readerId, thread1a, main1Id));
+        room.addReceipt(createReceipt(readerId, thread2a));
+        room.addReceipt(createThreadedReceipt(readerId, thread2b, main2Id));
+
+        // Then we correctly identify that only main3 is unread
+        expect(room.hasUserReadEvent(readerId, main1Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, main2Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, main3Id)).toBe(false);
+        expect(room.hasUserReadEvent(readerId, thread1aId)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, thread1bId)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, thread2aId)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, thread2bId)).toBe(true);
+    });
+
+    describe("dangling receipts", () => {
+        it("reports unread if the unthreaded receipt is in a dangling state", () => {
+            const room = createRoom();
+            const [event, eventId] = createEvent();
+            // When we receive a receipt for this event+user
+            room.addReceipt(createReceipt(readerId, event));
+
+            // The event is not added in the room
+            // So the receipt is in a dangling state
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+            // Add the event to the room
+            // The receipt is removed from the dangling state
+            room.addLiveEvents([event]);
+
+            // Then the event is read
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+        });
+
+        it("reports unread if the threaded receipt is in a dangling state", () => {
+            const room = createRoom();
+            const [root, rootId] = createEvent();
+            const [event, eventId] = createThreadedEvent(root);
+            setupThread(room, root);
+
+            // When we receive a receipt for this event+user
+            room.addReceipt(createThreadedReceipt(readerId, event, rootId));
+
+            // The event is not added in the room
+            // So the receipt is in a dangling state
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+            // Add the events to the room
+            // The receipt is removed from the dangling state
+            room.addLiveEvents([root, event]);
+
+            // Then the event is read
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+        });
+
+        it("should handle multiple dangling receipts for the same event", () => {
+            const room = createRoom();
+            const [event, eventId] = createEvent();
+            // When we receive a receipt for this event+user
+            room.addReceipt(createReceipt(readerId, event));
+            // We receive another receipt in the same event for another user
+            room.addReceipt(createReceipt(otherUserId, event));
+
+            // The event is not added in the room
+            // So the receipt is in a dangling state
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+            // Add the event to the room
+            // The two receipts should be processed
+            room.addLiveEvents([event]);
+
+            // Then the event is read
+            // We expect that the receipt of `otherUserId` didn't replace/erase the receipt of `readerId`
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+        });
+    });
+});
+
+function createFakeClient(): MatrixClient {
+    return {
+        getUserId: jest.fn(),
+        getEventMapper: jest.fn().mockReturnValue(jest.fn()),
+        isInitialSyncComplete: jest.fn().mockReturnValue(true),
+        supportsThreads: jest.fn().mockReturnValue(true),
+        fetchRoomEvent: jest.fn().mockResolvedValue({}),
+        paginateEventTimeline: jest.fn(),
+        canSupport: { get: jest.fn() },
+    } as unknown as MatrixClient;
+}
+
+const senderId = "sender:s.ss";
+const readerId = "reader:r.rr";
+const otherUserId = "other:o.oo";
+
+function createRoom(): Room {
+    return new Room("!rid", createFakeClient(), "@u:s.nz", { timelineSupport: true });
+}
+
+let idCounter = 0;
+function nextId(): string {
+    return "$" + (idCounter++).toString(10);
+}
+
+/**
+ * Create an event and return it and its ID.
+ */
+function createEvent(): [MatrixEvent, string] {
+    return createEventSentBy(senderId);
+}
+
+/**
+ * Create an event with the supplied sender and return it and its ID.
+ */
+function createEventSentBy(customSenderId: string): [MatrixEvent, string] {
+    const event = new MatrixEvent({ sender: customSenderId, event_id: nextId() });
+    return [event, event.getId()!];
+}
+
+/**
+ * Create an event in the thread of the supplied root and return it and its ID.
+ */
+function createThreadedEvent(root: MatrixEvent): [MatrixEvent, string] {
+    const rootEventId = root.getId()!;
+    const event = new MatrixEvent({
+        sender: senderId,
+        event_id: nextId(),
+        content: {
+            "m.relates_to": {
+                event_id: rootEventId,
+                rel_type: THREAD_RELATION_TYPE.name,
+                ["m.in_reply_to"]: {
+                    event_id: rootEventId,
+                },
+            },
+        },
+    });
+    return [event, event.getId()!];
+}
+
+function createReceipt(userId: string, referencedEvent: MatrixEvent): MatrixEvent {
+    const content: ReceiptContent = {
+        [referencedEvent.getId()!]: {
+            "m.read": {
+                [userId]: {
+                    ts: 123,
+                },
+            },
+        },
+    };
+
+    return new MatrixEvent({
+        type: "m.receipt",
+        content,
+    });
+}
+
+function createThreadedReceipt(userId: string, referencedEvent: MatrixEvent, threadId: string): MatrixEvent {
+    const content: ReceiptContent = {
+        [referencedEvent.getId()!]: {
+            "m.read": {
+                [userId]: {
+                    ts: 123,
+                    thread_id: threadId,
+                },
+            },
+        },
+    };
+
+    return new MatrixEvent({
+        type: "m.receipt",
+        content,
+    });
+}
+
+/**
+ * Create a timeline in the timeline set that is not the live timeline.
+ */
+function createOldTimeline(room: Room, events: MatrixEvent[]) {
+    const oldTimeline = room.getUnfilteredTimelineSet().addTimeline();
+    room.getUnfilteredTimelineSet().addEventsToTimeline(events, true, oldTimeline);
+}
+
+/**
+ * Perform the hacks required for this room to create a thread based on the root
+ * event supplied.
+ */
+function setupThread(room: Room, root: MatrixEvent) {
+    const thread = room.createThread(root.getId()!, root, [root], false);
+    thread.initialEventsFetched = true;
+}

spec/unit/models/thread.spec.ts

@@ -19,7 +19,7 @@ import { mocked } from "jest-mock";
 import { MatrixClient, PendingEventOrdering } from "../../../src/client";
 import { Room, RoomEvent } from "../../../src/models/room";
 import { FeatureSupport, Thread, THREAD_RELATION_TYPE, ThreadEvent } from "../../../src/models/thread";
-import { makeThreadEvent, mkThread } from "../../test-utils/thread";
+import { makeThreadEvent, mkThread, populateThread } from "../../test-utils/thread";
 import { TestClient } from "../../TestClient";
 import { emitPromise, mkEdit, mkMessage, mkReaction, mock } from "../../test-utils/test-utils";
 import { Direction, EventStatus, EventType, MatrixEvent } from "../../../src";
@@ -149,20 +149,38 @@ describe("Thread", () => {
         });
 
         it("considers other events with no RR as unread", () => {
-            const { thread, events } = mkThread({
+            // Given a long thread exists
+            const { thread, events } = populateThread({
                 room,
                 client,
-                authorId: myUserId,
-                participantUserIds: [myUserId],
+                authorId: "@other:foo.com",
+                participantUserIds: ["@other:foo.com"],
                 length: 25,
                 ts: 190,
             });
 
-            // Before alice's last unthreaded receipt
-            expect(thread.hasUserReadEvent("@alice:example.org", events.at(1)!.getId() ?? "")).toBeTruthy();
+            const event1 = events.at(1)!;
+            const event2 = events.at(2)!;
+            const event24 = events.at(24)!;
 
-            // After alice's last unthreaded receipt
-            expect(thread.hasUserReadEvent("@alice:example.org", events.at(-1)!.getId() ?? "")).toBeFalsy();
+            // And we have read the second message in it with an unthreaded receipt
+            const receipt = new MatrixEvent({
+                type: "m.receipt",
+                room_id: room.roomId,
+                content: {
+                    // unthreaded receipt for the second message in the thread
+                    [event2.getId()!]: {
+                        [ReceiptType.Read]: {
+                            [myUserId]: { ts: 200 },
+                        },
+                    },
+                },
+            });
+            room.addReceipt(receipt);
+
+            // Then we have read the first message in the thread, and not the last
+            expect(thread.hasUserReadEvent(myUserId, event1.getId()!)).toBe(true);
+            expect(thread.hasUserReadEvent(myUserId, event24.getId()!)).toBe(false);
         });
 
         it("considers event as read if there's a more recent unthreaded receipt", () => {
@@ -481,13 +499,13 @@ describe("Thread", () => {
 
                 // And a thread with an added event (with later timestamp)
                 const userId = "user1";
-                const { thread, message } = await createThreadAndEvent(client, 1, 100, userId);
+                const { thread, message2 } = await createThreadAnd2Events(client, 1, 100, 200, userId);
 
                 // Then a receipt was added to the thread
                 const receipt = thread.getReadReceiptForUserId(userId);
                 expect(receipt).toBeTruthy();
-                expect(receipt?.eventId).toEqual(message.getId());
-                expect(receipt?.data.ts).toEqual(100);
+                expect(receipt?.eventId).toEqual(message2.getId());
+                expect(receipt?.data.ts).toEqual(200);
                 expect(receipt?.data.thread_id).toEqual(thread.id);
 
                 // (And the receipt was synthetic)
@@ -505,14 +523,14 @@ describe("Thread", () => {
 
                 // And a thread with an added event with a lower timestamp than its other events
                 const userId = "user1";
-                const { thread } = await createThreadAndEvent(client, 200, 100, userId);
+                const { thread, message1 } = await createThreadAnd2Events(client, 300, 200, 100, userId);
 
-                // Then no receipt was added to the thread (the receipt is still
-                // for the thread root). This happens because since we have no
+                // Then the receipt is for the first message, because its
+                // timestamp is later. This happens because since we have no
                 // recursive relations support, we know that sometimes events
                 // appear out of order, so we have to check their timestamps as
                 // a guess of the correct order.
-                expect(thread.getReadReceiptForUserId(userId)?.eventId).toEqual(thread.rootEvent?.getId());
+                expect(thread.getReadReceiptForUserId(userId)?.eventId).toEqual(message1.getId());
             });
         });
 
@@ -530,11 +548,11 @@ describe("Thread", () => {
 
                 // And a thread with an added event (with later timestamp)
                 const userId = "user1";
-                const { thread, message } = await createThreadAndEvent(client, 1, 100, userId);
+                const { thread, message2 } = await createThreadAnd2Events(client, 1, 100, 200, userId);
 
                 // Then a receipt was added to the thread
                 const receipt = thread.getReadReceiptForUserId(userId);
-                expect(receipt?.eventId).toEqual(message.getId());
+                expect(receipt?.eventId).toEqual(message2.getId());
             });
 
             it("Creates a local echo receipt even for events BEFORE an existing receipt", async () => {
@@ -550,22 +568,24 @@ describe("Thread", () => {
 
                 // And a thread with an added event with a lower timestamp than its other events
                 const userId = "user1";
-                const { thread, message } = await createThreadAndEvent(client, 200, 100, userId);
+                const { thread, message2 } = await createThreadAnd2Events(client, 300, 200, 100, userId);
 
-                // Then a receipt was added to the thread, because relations
-                // recursion is available, so we trust the server to have
-                // provided us with events in the right order.
+                // Then a receipt was added for the last message, even though it
+                // has lower ts, because relations recursion is available, so we
+                // trust the server to have provided us with events in the right
+                // order.
                 const receipt = thread.getReadReceiptForUserId(userId);
-                expect(receipt?.eventId).toEqual(message.getId());
+                expect(receipt?.eventId).toEqual(message2.getId());
             });
         });
 
-        async function createThreadAndEvent(
+        async function createThreadAnd2Events(
             client: MatrixClient,
             rootTs: number,
-            eventTs: number,
+            message1Ts: number,
+            message2Ts: number,
             userId: string,
-        ): Promise<{ thread: Thread; message: MatrixEvent }> {
+        ): Promise<{ thread: Thread; message1: MatrixEvent; message2: MatrixEvent }> {
             const room = new Room("room1", client, userId);
 
             // Given a thread
@@ -576,24 +596,41 @@ describe("Thread", () => {
                 participantUserIds: [],
                 ts: rootTs,
             });
-            // Sanity: the current receipt is for the thread root
-            expect(thread.getReadReceiptForUserId(userId)?.eventId).toEqual(thread.rootEvent?.getId());
+            // Sanity: there is no read receipt on the thread yet because the
+            // thread events don't get properly added to the room by mkThread.
+            expect(thread.getReadReceiptForUserId(userId)).toBeNull();
 
             const awaitTimelineEvent = new Promise<void>((res) => thread.on(RoomEvent.Timeline, () => res()));
 
-            // When we add a message that is before the latest receipt
-            const message = makeThreadEvent({
+            // Add a message with ts message1Ts
+            const message1 = makeThreadEvent({
                 event: true,
                 rootEventId: thread.id,
                 replyToEventId: thread.id,
                 user: userId,
                 room: room.roomId,
-                ts: eventTs,
+                ts: message1Ts,
             });
-            await thread.addEvent(message, false, true);
+            await thread.addEvent(message1, false, true);
             await awaitTimelineEvent;
 
-            return { thread, message };
+            // Sanity: the thread now has a properly-added event, so this event
+            // has a synthetic receipt.
+            expect(thread.getReadReceiptForUserId(userId)?.eventId).toEqual(message1.getId());
+
+            // Add a message with ts message2Ts
+            const message2 = makeThreadEvent({
+                event: true,
+                rootEventId: thread.id,
+                replyToEventId: thread.id,
+                user: userId,
+                room: room.roomId,
+                ts: message2Ts,
+            });
+            await thread.addEvent(message2, false, true);
+            await awaitTimelineEvent;
+
+            return { thread, message1, message2 };
         }
 
         function createClientWithEventMapper(canSupport: Map<Feature, ServerSupport> = new Map()): MatrixClient {
@@ -675,6 +712,69 @@ describe("Thread", () => {
             });
         });
     });
+
+    describe("addEvent", () => {
+        describe("Given server support for threads", () => {
+            let previousThreadHasServerSideSupport: FeatureSupport;
+
+            beforeAll(() => {
+                previousThreadHasServerSideSupport = Thread.hasServerSideSupport;
+                Thread.hasServerSideSupport = FeatureSupport.Stable;
+            });
+
+            afterAll(() => {
+                Thread.hasServerSideSupport = previousThreadHasServerSideSupport;
+            });
+
+            it("Adds events even if they appear out of order", async () => {
+                // Given a thread exists
+                const client = createClient();
+                const user = "@alice:matrix.org";
+                const room = "!room:z";
+                const thread = await createThread(client, user, room);
+                const prevNumEvents = thread.timeline.length;
+
+                // When two messages come in but the later one has an older timestamp
+                const message1 = createThreadMessage(thread.id, user, room, "message1");
+                const message2 = createThreadMessage(thread.id, user, room, "message2");
+                message2.localTimestamp -= 10000;
+
+                await thread.addEvent(message1, false);
+                await thread.addEvent(message2, false);
+
+                // Then both events end up in the timeline
+                expect(thread.timeline.length - prevNumEvents).toEqual(2);
+                const lastEvent = thread.timeline.at(-1)!;
+                const secondLastEvent = thread.timeline.at(-2)!;
+                expect(lastEvent).toBe(message2);
+                expect(secondLastEvent).toBe(message1);
+            });
+
+            it("Adds events to start even if they appear out of order", async () => {
+                // Given a thread exists
+                const client = createClient();
+                const user = "@alice:matrix.org";
+                const room = "!room:z";
+                const thread = await createThread(client, user, room);
+                const prevNumEvents = thread.timeline.length;
+
+                // When two messages come in but the later one has an older timestamp
+                const message1 = createThreadMessage(thread.id, user, room, "message1");
+                const message2 = createThreadMessage(thread.id, user, room, "message2");
+                message2.localTimestamp -= 10000;
+
+                await thread.addEvent(message1, false);
+                await thread.addEvent(message2, true);
+
+                // Then both events end up in the timeline
+                expect(thread.timeline.length - prevNumEvents).toEqual(2);
+                const lastEvent = thread.timeline.at(-1)!;
+                const firstEvent = thread.timeline.at(0)!;
+                expect(lastEvent).toBe(message1);
+                expect(firstEvent).toBe(message2);
+            });
+        });
+    });
 });
 
 /**

spec/unit/notifications.spec.ts

@@ -106,6 +106,8 @@ describe("fixNotificationCountOnDecryption", () => {
             mockClient,
         );
 
+        room.addLiveEvents([event]);
+
         THREAD_ID = event.getId()!;
         threadEvent = mkEvent({
             type: EventType.RoomMessage,

spec/unit/randomstring.spec.ts

@@ -0,0 +1,66 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { decodeBase64 } from "../../src/base64";
+import {
+    randomLowercaseString,
+    randomString,
+    randomUppercaseString,
+    secureRandomBase64Url,
+} from "../../src/randomstring";
+
+describe("Random strings", () => {
+    it.each([8, 16, 32])("secureRandomBase64 generates %i valid base64 bytes", (n: number) => {
+        const randb641 = secureRandomBase64Url(n);
+        const randb642 = secureRandomBase64Url(n);
+
+        expect(randb641).not.toEqual(randb642);
+
+        const decoded = decodeBase64(randb641);
+        expect(decoded).toHaveLength(n);
+    });
+
+    it.each([8, 16, 32])("randomString generates string of %i characters", (n: number) => {
+        const rand1 = randomString(n);
+        const rand2 = randomString(n);
+
+        expect(rand1).not.toEqual(rand2);
+
+        expect(rand1).toHaveLength(n);
+    });
+
+    it.each([8, 16, 32])("randomLowercaseString generates lowercase string of %i characters", (n: number) => {
+        const rand1 = randomLowercaseString(n);
+        const rand2 = randomLowercaseString(n);
+
+        expect(rand1).not.toEqual(rand2);
+
+        expect(rand1).toHaveLength(n);
+
+        expect(rand1.toLowerCase()).toEqual(rand1);
+    });
+
+    it.each([8, 16, 32])("randomUppercaseString generates lowercase string of %i characters", (n: number) => {
+        const rand1 = randomUppercaseString(n);
+        const rand2 = randomUppercaseString(n);
+
+        expect(rand1).not.toEqual(rand2);
+
+        expect(rand1).toHaveLength(n);
+
+        expect(rand1.toUpperCase()).toEqual(rand1);
+    });
+});

spec/unit/read-receipt.spec.ts

@@ -43,8 +43,10 @@ const THREAD_ID = "$thread_event_id";
 const ROOM_ID = "!123:matrix.org";
 
 describe("Read receipt", () => {
+    let threadRoot: MatrixEvent;
     let threadEvent: MatrixEvent;
     let roomEvent: MatrixEvent;
+    let editOfThreadRoot: MatrixEvent;
 
     beforeEach(() => {
         httpBackend = new MockHttpBackend();
@@ -57,6 +59,15 @@ describe("Read receipt", () => {
         client.isGuest = () => false;
         client.supportsThreads = () => true;
 
+        threadRoot = utils.mkEvent({
+            event: true,
+            type: EventType.RoomMessage,
+            user: "@bob:matrix.org",
+            room: ROOM_ID,
+            content: { body: "This is the thread root" },
+        });
+        threadRoot.event.event_id = THREAD_ID;
+
         threadEvent = utils.mkEvent({
             event: true,
             type: EventType.RoomMessage,
@@ -82,6 +93,9 @@ describe("Read receipt", () => {
                 body: "Hello from a room",
             },
         });
+
+        editOfThreadRoot = utils.mkEdit(threadRoot, client, "@bob:matrix.org", ROOM_ID);
+        editOfThreadRoot.setThreadId(THREAD_ID);
     });
 
     describe("sendReceipt", () => {
@@ -208,6 +222,7 @@ describe("Read receipt", () => {
         it.each([
             { getEvent: () => roomEvent, destinationId: MAIN_ROOM_TIMELINE },
             { getEvent: () => threadEvent, destinationId: THREAD_ID },
+            { getEvent: () => editOfThreadRoot, destinationId: MAIN_ROOM_TIMELINE },
         ])("adds the receipt to $destinationId", ({ getEvent, destinationId }) => {
             const event = getEvent();
             const userId = "@bob:example.org";
@@ -225,6 +240,7 @@ describe("Read receipt", () => {
         it("should not allow an older unthreaded receipt to clobber a `main` threaded one", () => {
             const userId = client.getSafeUserId();
             const room = new Room(ROOM_ID, client, userId);
+            room.findEventById = jest.fn().mockReturnValue({} as MatrixEvent);
 
             const unthreadedReceipt: WrappedReceipt = {
                 eventId: "$olderEvent",

spec/unit/rendezvous/ecdhv2.spec.ts

@@ -17,7 +17,7 @@ limitations under the License.
 import "../../olm-loader";
 import { RendezvousFailureReason, RendezvousIntent } from "../../../src/rendezvous";
 import { MSC3903ECDHPayload, MSC3903ECDHv2RendezvousChannel } from "../../../src/rendezvous/channels";
-import { decodeBase64 } from "../../../src/crypto/olmlib";
+import { decodeBase64 } from "../../../src/base64";
 import { DummyTransport } from "./DummyTransport";
 
 function makeTransport(name: string) {

spec/unit/rendezvous/rendezvous.spec.ts

@@ -29,7 +29,7 @@ import {
     MSC3886SimpleHttpRendezvousTransportDetails,
 } from "../../../src/rendezvous/transports";
 import { DummyTransport } from "./DummyTransport";
-import { decodeBase64 } from "../../../src/crypto/olmlib";
+import { decodeBase64 } from "../../../src/base64";
 import { logger } from "../../../src/logger";
 import { DeviceInfo } from "../../../src/crypto/deviceinfo";
 

spec/unit/room-state.spec.ts

@@ -27,6 +27,7 @@ import { M_BEACON } from "../../src/@types/beacon";
 import { MatrixClient } from "../../src/client";
 import { DecryptionError } from "../../src/crypto/algorithms";
 import { defer } from "../../src/utils";
+import { Room } from "../../src/models/room";
 
 describe("RoomState", function () {
     const roomId = "!foo:bar";
@@ -362,9 +363,11 @@ describe("RoomState", function () {
         });
 
         it("does not add redacted beacon info events to state", () => {
+            const mockClient = {} as unknown as MockedObject<MatrixClient>;
             const redactedBeaconEvent = makeBeaconInfoEvent(userA, roomId);
             const redactionEvent = new MatrixEvent({ type: "m.room.redaction" });
-            redactedBeaconEvent.makeRedacted(redactionEvent);
+            const room = new Room(roomId, mockClient, userA);
+            redactedBeaconEvent.makeRedacted(redactionEvent, room);
             const emitSpy = jest.spyOn(state, "emit");
 
             state.setStateEvents([redactedBeaconEvent]);
@@ -394,11 +397,13 @@ describe("RoomState", function () {
         });
 
         it("destroys and removes redacted beacon events", () => {
+            const mockClient = {} as unknown as MockedObject<MatrixClient>;
             const beaconId = "$beacon1";
             const beaconEvent = makeBeaconInfoEvent(userA, roomId, { isLive: true }, beaconId);
             const redactedBeaconEvent = makeBeaconInfoEvent(userA, roomId, { isLive: true }, beaconId);
             const redactionEvent = new MatrixEvent({ type: "m.room.redaction", redacts: beaconEvent.getId() });
-            redactedBeaconEvent.makeRedacted(redactionEvent);
+            const room = new Room(roomId, mockClient, userA);
+            redactedBeaconEvent.makeRedacted(redactionEvent, room);
 
             state.setStateEvents([beaconEvent]);
             const beaconInstance = state.beacons.get(getBeaconInfoIdentifier(beaconEvent));

spec/unit/room.spec.ts

@@ -1743,12 +1743,70 @@ describe("Room", function () {
         });
 
         describe("hasUserReadUpTo", function () {
-            it("should acknowledge if an event has been read", function () {
+            it("returns true if there is a receipt for this event (main timeline)", function () {
                 const ts = 13787898424;
+                room.addLiveEvents([eventToAck]);
                 room.addReceipt(mkReceipt(roomId, [mkRecord(eventToAck.getId()!, "m.read", userB, ts)]));
+                room.findEventById = jest.fn().mockReturnValue({ getThread: jest.fn() } as unknown as MatrixEvent);
                 expect(room.hasUserReadEvent(userB, eventToAck.getId()!)).toEqual(true);
             });
-            it("return false for an unknown event", function () {
+
+            it("returns true if there is a receipt for a later event (main timeline)", async function () {
+                // Given some events exist in the room
+                const events: MatrixEvent[] = [
+                    utils.mkMessage({
+                        room: roomId,
+                        user: userA,
+                        msg: "1111",
+                        event: true,
+                    }),
+                    utils.mkMessage({
+                        room: roomId,
+                        user: userA,
+                        msg: "2222",
+                        event: true,
+                    }),
+                    utils.mkMessage({
+                        room: roomId,
+                        user: userA,
+                        msg: "3333",
+                        event: true,
+                    }),
+                ];
+                await room.addLiveEvents(events);
+
+                // When I add a receipt for the latest one
+                room.addReceipt(mkReceipt(roomId, [mkRecord(events[2].getId()!, "m.read", userB, 102)]));
+
+                // Then the older ones are read too
+                expect(room.hasUserReadEvent(userB, events[0].getId()!)).toEqual(true);
+                expect(room.hasUserReadEvent(userB, events[1].getId()!)).toEqual(true);
+            });
+
+            describe("threads enabled", () => {
+                beforeEach(() => {
+                    jest.spyOn(room.client, "supportsThreads").mockReturnValue(true);
+                });
+
+                afterEach(() => {
+                    jest.restoreAllMocks();
+                });
+
+                it("returns true if there is an unthreaded receipt for a later event in a thread", async () => {
+                    // Given a thread exists in the room
+                    const { thread, events } = mkThread({ room, length: 3 });
+                    thread.initialEventsFetched = true;
+                    await room.addLiveEvents(events);
+
+                    // When I add an unthreaded receipt for the latest thread message
+                    room.addReceipt(mkReceipt(roomId, [mkRecord(events[2].getId()!, "m.read", userB, 102)]));
+
+                    // Then the main timeline message is read
+                    expect(room.hasUserReadEvent(userB, events[0].getId()!)).toEqual(true);
+                });
+            });
+
+            it("returns false for an unknown event", function () {
                 expect(room.hasUserReadEvent(userB, "unknown_event")).toEqual(false);
             });
         });
@@ -3147,106 +3205,194 @@ describe("Room", function () {
         const client = new TestClient(userA).client;
         const room = new Room(roomId, client, userA);
 
-        it("handles missing receipt type", () => {
-            room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
-                return receiptType === ReceiptType.ReadPrivate ? ({ eventId: "eventId" } as WrappedReceipt) : null;
-            };
-
-            expect(room.getEventReadUpTo(userA)).toEqual("eventId");
-        });
-
-        describe("prefers newer receipt", () => {
-            it("should compare correctly using timelines", () => {
-                room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
-                    if (receiptType === ReceiptType.ReadPrivate) {
-                        return { eventId: "eventId1" } as WrappedReceipt;
-                    }
-                    if (receiptType === ReceiptType.Read) {
-                        return { eventId: "eventId2" } as WrappedReceipt;
-                    }
-                    return null;
-                };
-
-                for (let i = 1; i <= 2; i++) {
-                    room.getUnfilteredTimelineSet = () =>
-                        ({
-                            compareEventOrdering: (event1, event2) => {
-                                return event1 === `eventId${i}` ? 1 : -1;
-                            },
-                        } as EventTimelineSet);
-
-                    expect(room.getEventReadUpTo(userA)).toEqual(`eventId${i}`);
-                }
+        describe("invalid receipts", () => {
+            beforeEach(() => {
+                // Clear the spies on logger.warn
+                jest.clearAllMocks();
             });
 
-            describe("correctly compares by timestamp", () => {
-                it("should correctly compare, if we have all receipts", () => {
+            it("ignores receipts pointing at missing events", () => {
+                // Given a receipt exists
+                room.getReadReceiptForUserId = (): WrappedReceipt | null => {
+                    return { eventId: "missingEventId" } as WrappedReceipt;
+                };
+                // But the event ID it contains does not refer to an event we have
+                room.findEventById = jest.fn().mockReturnValue(null);
+
+                // When we ask what they have read
+                // Then we say "nothing"
+                expect(room.getEventReadUpTo(userA)).toBeNull();
+            });
+
+            it("ignores receipts pointing at the wrong thread", () => {
+                // Given a threaded receipt exists
+                room.getReadReceiptForUserId = (): WrappedReceipt | null => {
+                    return { eventId: "wrongThreadEventId", data: { ts: 0, thread_id: "thread1" } } as WrappedReceipt;
+                };
+                // But the event it refers to is in a thread
+                room.findEventById = jest.fn().mockReturnValue({ threadRootId: "thread2" } as MatrixEvent);
+
+                // When we ask what they have read
+                // Then we say "nothing"
+                expect(room.getEventReadUpTo(userA)).toBeNull();
+                expect(logger.warn).toHaveBeenCalledWith(
+                    "Ignoring receipt because its thread_id (thread1) disagrees with the thread root (thread2) " +
+                        "of the referenced event (event ID = wrongThreadEventId)",
+                );
+            });
+
+            it("accepts unthreaded receipts pointing at an event in a thread", () => {
+                // Given an unthreaded receipt exists
+                room.getReadReceiptForUserId = (): WrappedReceipt | null => {
+                    return { eventId: "inThreadEventId" } as WrappedReceipt;
+                };
+                // And the event it refers to is in a thread
+                room.findEventById = jest.fn().mockReturnValue({ threadRootId: "thread2" } as MatrixEvent);
+
+                // When we ask what they have read
+                // Then we say the event
+                expect(room.getEventReadUpTo(userA)).toEqual("inThreadEventId");
+            });
+
+            it("accepts main thread receipts pointing at an event in main timeline", () => {
+                // Given a threaded receipt exists, in main thread
+                room.getReadReceiptForUserId = (): WrappedReceipt | null => {
+                    return { eventId: "mainThreadEventId", data: { ts: 12, thread_id: "main" } } as WrappedReceipt;
+                };
+                // And the event it refers to is in a thread
+                room.findEventById = jest.fn().mockReturnValue({ threadRootId: undefined } as MatrixEvent);
+
+                // When we ask what they have read
+                // Then we say the event
+                expect(room.getEventReadUpTo(userA)).toEqual("mainThreadEventId");
+            });
+
+            it("accepts main thread receipts pointing at a thread root", () => {
+                // Given a threaded receipt exists, in main thread
+                room.getReadReceiptForUserId = (): WrappedReceipt | null => {
+                    return { eventId: "rootId", data: { ts: 12, thread_id: "main" } } as WrappedReceipt;
+                };
+                // And the event it refers to is in a thread, because it is a thread root
+                room.findEventById = jest
+                    .fn()
+                    .mockReturnValue({ isThreadRoot: true, threadRootId: "thread1" } as MatrixEvent);
+
+                // When we ask what they have read
+                // Then we say the event
+                expect(room.getEventReadUpTo(userA)).toEqual("rootId");
+            });
+        });
+
+        describe("valid receipts", () => {
+            beforeEach(() => {
+                // When we look up the event referred to by the receipt, it exists
+                room.findEventById = jest.fn().mockReturnValue({} as MatrixEvent);
+            });
+
+            it("handles missing receipt type", () => {
+                room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
+                    return receiptType === ReceiptType.ReadPrivate ? ({ eventId: "eventId" } as WrappedReceipt) : null;
+                };
+                expect(room.getEventReadUpTo(userA)).toEqual("eventId");
+            });
+
+            describe("prefers newer receipt", () => {
+                it("should compare correctly using timelines", () => {
+                    room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
+                        if (receiptType === ReceiptType.ReadPrivate) {
+                            return { eventId: "eventId1" } as WrappedReceipt;
+                        }
+                        if (receiptType === ReceiptType.Read) {
+                            return { eventId: "eventId2" } as WrappedReceipt;
+                        }
+                        return null;
+                    };
+
                     for (let i = 1; i <= 2; i++) {
                         room.getUnfilteredTimelineSet = () =>
                             ({
-                                compareEventOrdering: (_1, _2) => null,
-                            } as EventTimelineSet);
-                        room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
-                            if (receiptType === ReceiptType.ReadPrivate) {
-                                return { eventId: "eventId1", data: { ts: i === 1 ? 2 : 1 } } as WrappedReceipt;
-                            }
-                            if (receiptType === ReceiptType.Read) {
-                                return { eventId: "eventId2", data: { ts: i === 2 ? 2 : 1 } } as WrappedReceipt;
-                            }
-                            return null;
-                        };
+                                compareEventOrdering: (event1: string, _event2: string) => {
+                                    return event1 === `eventId${i}` ? 1 : -1;
+                                },
+                                findEventById: jest.fn().mockReturnValue({} as MatrixEvent),
+                            } as unknown as EventTimelineSet);
 
                         expect(room.getEventReadUpTo(userA)).toEqual(`eventId${i}`);
                     }
                 });
 
-                it("should correctly compare, if private read receipt is missing", () => {
-                    room.getUnfilteredTimelineSet = () =>
-                        ({
-                            compareEventOrdering: (_1, _2) => null,
-                        } as EventTimelineSet);
-                    room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
-                        if (receiptType === ReceiptType.Read) {
-                            return { eventId: "eventId2", data: { ts: 1 } } as WrappedReceipt;
+                describe("correctly compares by timestamp", () => {
+                    it("should correctly compare, if we have all receipts", () => {
+                        for (let i = 1; i <= 2; i++) {
+                            room.getUnfilteredTimelineSet = () =>
+                                ({
+                                    compareEventOrdering: () => null,
+                                    findEventById: jest.fn().mockReturnValue({} as MatrixEvent),
+                                } as unknown as EventTimelineSet);
+                            room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
+                                if (receiptType === ReceiptType.ReadPrivate) {
+                                    return { eventId: "eventId1", data: { ts: i === 1 ? 2 : 1 } } as WrappedReceipt;
+                                }
+                                if (receiptType === ReceiptType.Read) {
+                                    return { eventId: "eventId2", data: { ts: i === 2 ? 2 : 1 } } as WrappedReceipt;
+                                }
+                                return null;
+                            };
+
+                            expect(room.getEventReadUpTo(userA)).toEqual(`eventId${i}`);
                         }
-                        return null;
-                    };
+                    });
 
-                    expect(room.getEventReadUpTo(userA)).toEqual(`eventId2`);
-                });
-            });
+                    it("should correctly compare, if private read receipt is missing", () => {
+                        room.getUnfilteredTimelineSet = () =>
+                            ({
+                                compareEventOrdering: () => null,
+                                findEventById: jest.fn().mockReturnValue({} as MatrixEvent),
+                            } as unknown as EventTimelineSet);
+                        room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
+                            if (receiptType === ReceiptType.Read) {
+                                return { eventId: "eventId2", data: { ts: 1 } } as WrappedReceipt;
+                            }
+                            return null;
+                        };
 
-            describe("fallback precedence", () => {
-                beforeAll(() => {
-                    room.getUnfilteredTimelineSet = () =>
-                        ({
-                            compareEventOrdering: (_1, _2) => null,
-                        } as EventTimelineSet);
+                        expect(room.getEventReadUpTo(userA)).toEqual(`eventId2`);
+                    });
                 });
 
-                it("should give precedence to m.read.private", () => {
-                    room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
-                        if (receiptType === ReceiptType.ReadPrivate) {
-                            return { eventId: "eventId1", data: { ts: 123 } };
-                        }
-                        if (receiptType === ReceiptType.Read) {
-                            return { eventId: "eventId2", data: { ts: 123 } };
-                        }
-                        return null;
-                    };
+                describe("fallback precedence", () => {
+                    beforeAll(() => {
+                        room.getUnfilteredTimelineSet = () =>
+                            ({
+                                compareEventOrdering: () => null,
+                                findEventById: jest.fn().mockReturnValue({} as MatrixEvent),
+                            } as unknown as EventTimelineSet);
+                    });
 
-                    expect(room.getEventReadUpTo(userA)).toEqual(`eventId1`);
-                });
+                    it("should give precedence to m.read.private", () => {
+                        room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
+                            if (receiptType === ReceiptType.ReadPrivate) {
+                                return { eventId: "eventId1", data: { ts: 123 } };
+                            }
+                            if (receiptType === ReceiptType.Read) {
+                                return { eventId: "eventId2", data: { ts: 123 } };
+                            }
+                            return null;
+                        };
 
-                it("should give precedence to m.read", () => {
-                    room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
-                        if (receiptType === ReceiptType.Read) {
-                            return { eventId: "eventId3" } as WrappedReceipt;
-                        }
-                        return null;
-                    };
+                        expect(room.getEventReadUpTo(userA)).toEqual(`eventId1`);
+                    });
 
-                    expect(room.getEventReadUpTo(userA)).toEqual(`eventId3`);
+                    it("should give precedence to m.read", () => {
+                        room.getReadReceiptForUserId = (userId, ignore, receiptType): WrappedReceipt | null => {
+                            if (receiptType === ReceiptType.Read) {
+                                return { eventId: "eventId3" } as WrappedReceipt;
+                            }
+                            return null;
+                        };
+
+                        expect(room.getEventReadUpTo(userA)).toEqual(`eventId3`);
+                    });
                 });
             });
         });
@@ -3564,7 +3710,7 @@ describe("Room", function () {
             expect(room.polls.get(pollStartEvent.getId()!)).toBeTruthy();
 
             const redactedEvent = new MatrixEvent({ type: "m.room.redaction" });
-            pollStartEvent.makeRedacted(redactedEvent);
+            pollStartEvent.makeRedacted(redactedEvent, room);
 
             await flushPromises();
 

spec/unit/rust-crypto/KeyClaimManager.spec.ts

@@ -23,6 +23,7 @@ import { OutgoingRequestProcessor } from "../../../src/rust-crypto/OutgoingReque
 import { KeyClaimManager } from "../../../src/rust-crypto/KeyClaimManager";
 import { TypedEventEmitter } from "../../../src/models/typed-event-emitter";
 import { HttpApiEvent, HttpApiEventHandlerMap, MatrixHttpApi } from "../../../src";
+import { logger, LogSpan } from "../../../src/logger";
 
 afterEach(() => {
     fetchMock.mockReset();
@@ -93,7 +94,7 @@ describe("KeyClaimManager", () => {
         olmMachine.markRequestAsSent.mockResolvedValueOnce(undefined);
 
         // fire off the request
-        await keyClaimManager.ensureSessionsForUsers([u1, u2]);
+        await keyClaimManager.ensureSessionsForUsers(new LogSpan(logger, "test"), [u1, u2]);
 
         // check that all the calls were made
         expect(olmMachine.getMissingSessions).toHaveBeenCalledWith([u1, u2]);
@@ -119,12 +120,13 @@ describe("KeyClaimManager", () => {
         let markRequestAsSentPromise = awaitCallToMarkRequestAsSent();
 
         // fire off two requests, and keep track of whether their promises resolve
+        const span = new LogSpan(logger, "test");
         let req1Resolved = false;
-        keyClaimManager.ensureSessionsForUsers([u1]).then(() => {
+        keyClaimManager.ensureSessionsForUsers(span, [u1]).then(() => {
             req1Resolved = true;
         });
         let req2Resolved = false;
-        const req2 = keyClaimManager.ensureSessionsForUsers([u2]).then(() => {
+        const req2 = keyClaimManager.ensureSessionsForUsers(span, [u2]).then(() => {
             req2Resolved = true;
         });
 

spec/unit/rust-crypto/OutgoingRequestProcessor.spec.ts

@@ -24,7 +24,7 @@ import {
     KeysUploadRequest,
     RoomMessageRequest,
     SignatureUploadRequest,
-    SigningKeysUploadRequest,
+    UploadSigningKeysRequest,
     ToDeviceRequest,
 } from "@matrix-org/matrix-sdk-crypto-wasm";
 
@@ -82,12 +82,6 @@ describe("OutgoingRequestProcessor", () => {
             "https://example.com/_matrix/client/v3/keys/signatures/upload",
         ],
         ["KeysBackupRequest", KeysBackupRequest, "PUT", "https://example.com/_matrix/client/v3/room_keys/keys"],
-        [
-            "SigningKeysUploadRequest",
-            SigningKeysUploadRequest,
-            "POST",
-            "https://example.com/_matrix/client/v3/keys/device_signing/upload",
-        ],
     ];
 
     test.each(tests)(`should handle %ss`, async (_, RequestClass, expectedMethod, expectedPath) => {
@@ -121,7 +115,7 @@ describe("OutgoingRequestProcessor", () => {
 
     it("should handle ToDeviceRequests", async () => {
         // first, mock up the ToDeviceRequest as we might expect to receive it from the Rust layer ...
-        const testBody = '{ "foo": "bar" }';
+        const testBody = '{ "messages": { "user": {"device": "bar" }}}';
         const outgoingRequest = new ToDeviceRequest("1234", "test/type", "test/txnid", testBody);
 
         // ... then poke it into the OutgoingRequestProcessor under test.
@@ -179,10 +173,37 @@ describe("OutgoingRequestProcessor", () => {
         httpBackend.verifyNoOutstandingRequests();
     });
 
-    it("should handle SigningKeysUploadRequests with UIA", async () => {
+    it("should handle UploadSigningKeysRequest without UIA", async () => {
         // first, mock up a request as we might expect to receive it from the Rust layer ...
         const testReq = { foo: "bar" };
-        const outgoingRequest = new SigningKeysUploadRequest("1234", JSON.stringify(testReq));
+        const outgoingRequest = new UploadSigningKeysRequest(JSON.stringify(testReq));
+
+        // ... then poke the request into the OutgoingRequestProcessor under test
+        const reqProm = processor.makeOutgoingRequest(outgoingRequest);
+
+        // Now: check that it makes a matching HTTP request.
+        const testResponse = '{"result":1}';
+        httpBackend
+            .when("POST", "/_matrix")
+            .check((req) => {
+                expect(req.path).toEqual("https://example.com/_matrix/client/v3/keys/device_signing/upload");
+                expect(JSON.parse(req.rawData)).toEqual(testReq);
+                expect(req.headers["Accept"]).toEqual("application/json");
+                expect(req.headers["Content-Type"]).toEqual("application/json");
+            })
+            .respond(200, testResponse, true);
+
+        // SigningKeysUploadRequest does not need to be marked as sent, so no call to OlmMachine.markAsSent is expected.
+
+        await httpBackend.flushAllExpected();
+        await reqProm;
+        httpBackend.verifyNoOutstandingRequests();
+    });
+
+    it("should handle UploadSigningKeysRequest with UIA", async () => {
+        // first, mock up a request as we might expect to receive it from the Rust layer ...
+        const testReq = { foo: "bar" };
+        const outgoingRequest = new UploadSigningKeysRequest(JSON.stringify(testReq));
 
         // also create a UIA callback
         const authCallback: UIAuthCallback<Object> = async (makeRequest) => {
@@ -192,7 +213,7 @@ describe("OutgoingRequestProcessor", () => {
         // ... then poke the request into the OutgoingRequestProcessor under test
         const reqProm = processor.makeOutgoingRequest(outgoingRequest, authCallback);
 
-        // Now: check that it makes a matching HTTP request ...
+        // Now: check that it makes a matching HTTP request.
         const testResponse = '{"result":1}';
         httpBackend
             .when("POST", "/_matrix")
@@ -204,12 +225,10 @@ describe("OutgoingRequestProcessor", () => {
             })
             .respond(200, testResponse, true);
 
-        // ... and that it calls OlmMachine.markAsSent.
-        const markSentCallPromise = awaitCallToMarkAsSent();
-        await httpBackend.flushAllExpected();
+        // SigningKeysUploadRequest does not need to be marked as sent, so no call to OlmMachine.markAsSent is expected.
 
-        await Promise.all([reqProm, markSentCallPromise]);
-        expect(olmMachine.markRequestAsSent).toHaveBeenCalledWith("1234", outgoingRequest.type, testResponse);
+        await httpBackend.flushAllExpected();
+        await reqProm;
         httpBackend.verifyNoOutstandingRequests();
     });
 

spec/unit/rust-crypto/OutgoingRequestsManager.spec.ts

@@ -0,0 +1,237 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { Mocked } from "jest-mock";
+import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
+
+import { OutgoingRequest, OutgoingRequestProcessor } from "../../../src/rust-crypto/OutgoingRequestProcessor";
+import { OutgoingRequestsManager } from "../../../src/rust-crypto/OutgoingRequestsManager";
+import { defer, IDeferred } from "../../../src/utils";
+import { logger } from "../../../src/logger";
+
+describe("OutgoingRequestsManager", () => {
+    /** the OutgoingRequestsManager implementation under test */
+    let manager: OutgoingRequestsManager;
+
+    /** a mock OutgoingRequestProcessor */
+    let processor: Mocked<OutgoingRequestProcessor>;
+
+    /** a mocked-up OlmMachine which manager is connected to */
+    let olmMachine: Mocked<RustSdkCryptoJs.OlmMachine>;
+
+    beforeEach(async () => {
+        olmMachine = {
+            outgoingRequests: jest.fn(),
+        } as unknown as Mocked<RustSdkCryptoJs.OlmMachine>;
+
+        processor = {
+            makeOutgoingRequest: jest.fn(),
+        } as unknown as Mocked<OutgoingRequestProcessor>;
+
+        manager = new OutgoingRequestsManager(logger, olmMachine, processor);
+    });
+
+    describe("Call doProcessOutgoingRequests", () => {
+        it("The call triggers handling of the machine outgoing requests", async () => {
+            const request1 = new RustSdkCryptoJs.KeysQueryRequest("foo", "{}");
+            const request2 = new RustSdkCryptoJs.KeysUploadRequest("foo2", "{}");
+            olmMachine.outgoingRequests.mockImplementationOnce(async () => {
+                return [request1, request2];
+            });
+
+            processor.makeOutgoingRequest.mockImplementationOnce(async () => {
+                return;
+            });
+
+            await manager.doProcessOutgoingRequests();
+
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(1);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(2);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request1);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request2);
+        });
+
+        it("Stack and batch calls to doProcessOutgoingRequests while one is already running", async () => {
+            const request1 = new RustSdkCryptoJs.KeysQueryRequest("foo", "{}");
+            const request2 = new RustSdkCryptoJs.KeysUploadRequest("foo2", "{}");
+            const request3 = new RustSdkCryptoJs.KeysBackupRequest("foo3", "{}", "1");
+
+            const firstOutgoingRequestDefer = defer<OutgoingRequest[]>();
+
+            olmMachine.outgoingRequests
+                .mockImplementationOnce(async (): Promise<OutgoingRequest[]> => {
+                    return firstOutgoingRequestDefer.promise;
+                })
+                .mockImplementationOnce(async () => {
+                    return [request3];
+                });
+
+            const firstRequest = manager.doProcessOutgoingRequests();
+
+            // stack 2 additional requests while the first one is still running
+            const secondRequest = manager.doProcessOutgoingRequests();
+            const thirdRequest = manager.doProcessOutgoingRequests();
+
+            // let the first request complete
+            firstOutgoingRequestDefer.resolve([request1, request2]);
+
+            await firstRequest;
+            await secondRequest;
+            await thirdRequest;
+
+            // outgoingRequests should be called twice in total, as the second and third requests are
+            // processed in the same loop.
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(2);
+
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(3);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request1);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request2);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request3);
+        });
+
+        it("Process 3 consecutive calls to doProcessOutgoingRequests while not blocking previous ones", async () => {
+            const request1 = new RustSdkCryptoJs.KeysQueryRequest("foo", "{}");
+            const request2 = new RustSdkCryptoJs.KeysUploadRequest("foo2", "{}");
+            const request3 = new RustSdkCryptoJs.KeysBackupRequest("foo3", "{}", "1");
+
+            // promises which will resolve when OlmMachine.outgoingRequests is called
+            const outgoingRequestCalledPromises: Promise<void>[] = [];
+
+            // deferreds which will provide the results of OlmMachine.outgoingRequests
+            const outgoingRequestResultDeferreds: IDeferred<OutgoingRequest[]>[] = [];
+
+            for (let i = 0; i < 3; i++) {
+                const resultDeferred = defer<OutgoingRequest[]>();
+                const calledPromise = new Promise<void>((resolve) => {
+                    olmMachine.outgoingRequests.mockImplementationOnce(() => {
+                        resolve();
+                        return resultDeferred.promise;
+                    });
+                });
+                outgoingRequestCalledPromises.push(calledPromise);
+                outgoingRequestResultDeferreds.push(resultDeferred);
+            }
+
+            const call1 = manager.doProcessOutgoingRequests();
+
+            // First call will start an iteration and for now is awaiting on outgoingRequests
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(1);
+
+            // Make a new call now: this will request a new iteration
+            const call2 = manager.doProcessOutgoingRequests();
+
+            // let the first iteration complete
+            outgoingRequestResultDeferreds[0].resolve([request1]);
+
+            // The first call should now complete
+            await call1;
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(1);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request1);
+
+            // Wait for the second iteration to fire and be waiting on `outgoingRequests`
+            await outgoingRequestCalledPromises[1];
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(2);
+
+            // Stack a new call that should be processed in an additional iteration.
+            const call3 = manager.doProcessOutgoingRequests();
+
+            outgoingRequestResultDeferreds[1].resolve([request2]);
+            await call2;
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(2);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request2);
+
+            // Wait for the third iteration to fire and be waiting on `outgoingRequests`
+            await outgoingRequestCalledPromises[2];
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(3);
+            outgoingRequestResultDeferreds[2].resolve([request3]);
+            await call3;
+
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(3);
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledWith(request3);
+
+            // ensure that no other iteration is going on
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(3);
+        });
+
+        it("Should not bubble exceptions if server request is rejected", async () => {
+            const request = new RustSdkCryptoJs.KeysQueryRequest("foo", "{}");
+            olmMachine.outgoingRequests.mockImplementationOnce(async () => {
+                return [request];
+            });
+
+            processor.makeOutgoingRequest.mockImplementationOnce(async () => {
+                throw new Error("Some network error");
+            });
+
+            await manager.doProcessOutgoingRequests();
+
+            expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(1);
+        });
+    });
+
+    describe("Calling stop on the manager should stop ongoing work", () => {
+        it("When the manager is stopped after outgoingRequests() call, do not make sever requests", async () => {
+            const request1 = new RustSdkCryptoJs.KeysQueryRequest("foo", "{}");
+
+            const firstOutgoingRequestDefer = defer<OutgoingRequest[]>();
+
+            olmMachine.outgoingRequests.mockImplementationOnce(async (): Promise<OutgoingRequest[]> => {
+                return firstOutgoingRequestDefer.promise;
+            });
+
+            const firstRequest = manager.doProcessOutgoingRequests();
+
+            // stop
+            manager.stop();
+
+            // let the first request complete
+            firstOutgoingRequestDefer.resolve([request1]);
+
+            await firstRequest;
+
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(0);
+        });
+
+        it("When the manager is stopped while doing server calls, it should stop before the next sever call", async () => {
+            const request1 = new RustSdkCryptoJs.KeysQueryRequest("11", "{}");
+            const request2 = new RustSdkCryptoJs.KeysUploadRequest("12", "{}");
+
+            const firstRequestDefer = defer<void>();
+
+            olmMachine.outgoingRequests.mockImplementationOnce(async (): Promise<OutgoingRequest[]> => {
+                return [request1, request2];
+            });
+
+            processor.makeOutgoingRequest
+                .mockImplementationOnce(async () => {
+                    manager.stop();
+                    return firstRequestDefer.promise;
+                })
+                .mockImplementationOnce(async () => {
+                    return;
+                });
+
+            const firstRequest = manager.doProcessOutgoingRequests();
+
+            firstRequestDefer.resolve();
+
+            await firstRequest;
+
+            // should have been called once but not twice
+            expect(processor.makeOutgoingRequest).toHaveBeenCalledTimes(1);
+        });
+    });
+});

spec/unit/rust-crypto/PerSessionKeyBackupDownloader.spec.ts

@@ -0,0 +1,598 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { Mocked, SpyInstance } from "jest-mock";
+import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
+import { OlmMachine } from "@matrix-org/matrix-sdk-crypto-wasm";
+import fetchMock from "fetch-mock-jest";
+
+import { PerSessionKeyBackupDownloader } from "../../../src/rust-crypto/PerSessionKeyBackupDownloader";
+import { logger } from "../../../src/logger";
+import { defer, IDeferred } from "../../../src/utils";
+import { RustBackupCryptoEventMap, RustBackupCryptoEvents, RustBackupManager } from "../../../src/rust-crypto/backup";
+import * as TestData from "../../test-utils/test-data";
+import {
+    ConnectionError,
+    CryptoEvent,
+    HttpApiEvent,
+    HttpApiEventHandlerMap,
+    IHttpOpts,
+    IMegolmSessionData,
+    MatrixHttpApi,
+    TypedEventEmitter,
+} from "../../../src";
+import * as testData from "../../test-utils/test-data";
+import { BackupDecryptor } from "../../../src/common-crypto/CryptoBackend";
+import { KeyBackupSession } from "../../../src/crypto-api/keybackup";
+
+describe("PerSessionKeyBackupDownloader", () => {
+    /** The downloader under test */
+    let downloader: PerSessionKeyBackupDownloader;
+
+    const mockCipherKey: Mocked<KeyBackupSession> = {} as unknown as Mocked<KeyBackupSession>;
+
+    // matches the const in PerSessionKeyBackupDownloader
+    const BACKOFF_TIME = 5000;
+
+    let mockEmitter: TypedEventEmitter<RustBackupCryptoEvents, RustBackupCryptoEventMap>;
+    let mockHttp: MatrixHttpApi<IHttpOpts & { onlyData: true }>;
+    let mockRustBackupManager: Mocked<RustBackupManager>;
+    let mockOlmMachine: Mocked<OlmMachine>;
+    let mockBackupDecryptor: Mocked<BackupDecryptor>;
+
+    let expectedSession: { [roomId: string]: { [sessionId: string]: IDeferred<void> } };
+
+    function expectSessionImported(roomId: string, sessionId: string) {
+        const deferred = defer<void>();
+        if (!expectedSession[roomId]) {
+            expectedSession[roomId] = {};
+        }
+        expectedSession[roomId][sessionId] = deferred;
+        return deferred.promise;
+    }
+
+    function mockClearSession(sessionId: string): Mocked<IMegolmSessionData> {
+        return {
+            session_id: sessionId,
+        } as unknown as Mocked<IMegolmSessionData>;
+    }
+
+    beforeEach(async () => {
+        mockEmitter = new TypedEventEmitter() as TypedEventEmitter<RustBackupCryptoEvents, RustBackupCryptoEventMap>;
+
+        mockHttp = new MatrixHttpApi(new TypedEventEmitter<HttpApiEvent, HttpApiEventHandlerMap>(), {
+            baseUrl: "http://server/",
+            prefix: "",
+            onlyData: true,
+        });
+
+        mockBackupDecryptor = {
+            decryptSessions: jest.fn(),
+        } as unknown as Mocked<BackupDecryptor>;
+
+        mockBackupDecryptor.decryptSessions.mockImplementation(async (ciphertexts) => {
+            const sessionId = Object.keys(ciphertexts)[0];
+            return [mockClearSession(sessionId)];
+        });
+
+        mockRustBackupManager = {
+            getActiveBackupVersion: jest.fn(),
+            requestKeyBackupVersion: jest.fn(),
+            importBackedUpRoomKeys: jest.fn(),
+            createBackupDecryptor: jest.fn().mockReturnValue(mockBackupDecryptor),
+            on: jest.fn().mockImplementation((event, listener) => {
+                mockEmitter.on(event, listener);
+            }),
+            off: jest.fn().mockImplementation((event, listener) => {
+                mockEmitter.off(event, listener);
+            }),
+        } as unknown as Mocked<RustBackupManager>;
+
+        mockOlmMachine = {
+            getBackupKeys: jest.fn(),
+        } as unknown as Mocked<OlmMachine>;
+
+        downloader = new PerSessionKeyBackupDownloader(logger, mockOlmMachine, mockHttp, mockRustBackupManager);
+
+        expectedSession = {};
+        mockRustBackupManager.importBackedUpRoomKeys.mockImplementation(async (keys) => {
+            const roomId = keys[0].room_id;
+            const sessionId = keys[0].session_id;
+            const deferred = expectedSession[roomId] && expectedSession[roomId][sessionId];
+            if (deferred) {
+                deferred.resolve();
+            }
+        });
+
+        jest.useFakeTimers();
+    });
+
+    afterEach(() => {
+        expectedSession = {};
+        downloader.stop();
+        fetchMock.mockReset();
+        jest.useRealTimers();
+    });
+
+    describe("Given valid backup available", () => {
+        beforeEach(async () => {
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA.version!);
+            mockOlmMachine.getBackupKeys.mockResolvedValue({
+                backupVersion: TestData.SIGNED_BACKUP_DATA.version!,
+                decryptionKey: RustSdkCryptoJs.BackupDecryptionKey.fromBase64(TestData.BACKUP_DECRYPTION_KEY_BASE64),
+            } as unknown as RustSdkCryptoJs.BackupKeys);
+
+            mockRustBackupManager.requestKeyBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA);
+        });
+
+        it("Should download and import a missing key from backup", async () => {
+            const awaitKeyImported = defer<void>();
+            const roomId = "!roomId";
+            const sessionId = "sessionId";
+            const expectAPICall = new Promise<void>((resolve) => {
+                fetchMock.get(`path:/_matrix/client/v3/room_keys/keys/${roomId}/${sessionId}`, (url, request) => {
+                    resolve();
+                    return TestData.CURVE25519_KEY_BACKUP_DATA;
+                });
+            });
+            mockRustBackupManager.importBackedUpRoomKeys.mockImplementation(async (keys) => {
+                awaitKeyImported.resolve();
+            });
+            mockBackupDecryptor.decryptSessions.mockResolvedValue([TestData.MEGOLM_SESSION_DATA]);
+
+            downloader.onDecryptionKeyMissingError(roomId, sessionId);
+
+            await expectAPICall;
+            await awaitKeyImported.promise;
+            expect(mockRustBackupManager.createBackupDecryptor).toHaveBeenCalledTimes(1);
+        });
+
+        it("Should not hammer the backup if the key is requested repeatedly", async () => {
+            const blockOnServerRequest = defer<void>();
+
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/!roomId/:session_id`, async (url, request) => {
+                await blockOnServerRequest.promise;
+                return [mockCipherKey];
+            });
+
+            const awaitKey2Imported = defer<void>();
+
+            mockRustBackupManager.importBackedUpRoomKeys.mockImplementation(async (keys) => {
+                if (keys[0].session_id === "sessionId2") {
+                    awaitKey2Imported.resolve();
+                }
+            });
+
+            // @ts-ignore access to private function
+            const spy = jest.spyOn(downloader, "queryKeyBackup");
+
+            // Call 3 times for same key
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+
+            // Call again for a different key
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId2");
+
+            // Allow the first server request to complete
+            blockOnServerRequest.resolve();
+
+            await awaitKey2Imported.promise;
+            expect(spy).toHaveBeenCalledTimes(2);
+        });
+
+        it("should continue to next key if current not in backup", async () => {
+            fetchMock.get(`path:/_matrix/client/v3/room_keys/keys/!roomA/sessionA0`, {
+                status: 404,
+                body: {
+                    errcode: "M_NOT_FOUND",
+                    error: "No backup found",
+                },
+            });
+            fetchMock.get(`path:/_matrix/client/v3/room_keys/keys/!roomA/sessionA1`, mockCipherKey);
+
+            // @ts-ignore access to private function
+            const spy: SpyInstance = jest.spyOn(downloader, "queryKeyBackup");
+
+            const expectImported = expectSessionImported("!roomA", "sessionA1");
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            await jest.runAllTimersAsync();
+            expect(spy).toHaveBeenCalledTimes(1);
+            expect(spy).toHaveLastReturnedWith(Promise.resolve({ ok: false, error: "MISSING_DECRYPTION_KEY" }));
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA1");
+            await jest.runAllTimersAsync();
+            expect(spy).toHaveBeenCalledTimes(2);
+
+            await expectImported;
+        });
+
+        it("Should not query repeatedly for a key not in backup", async () => {
+            fetchMock.get(`path:/_matrix/client/v3/room_keys/keys/!roomA/sessionA0`, {
+                status: 404,
+                body: {
+                    errcode: "M_NOT_FOUND",
+                    error: "No backup found",
+                },
+            });
+
+            // @ts-ignore access to private function
+            const spy: SpyInstance = jest.spyOn(downloader, "queryKeyBackup");
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            await jest.runAllTimersAsync();
+
+            expect(spy).toHaveBeenCalledTimes(1);
+            const returnedPromise = spy.mock.results[0].value;
+            await expect(returnedPromise).rejects.toThrow("Failed to get key from backup: MISSING_DECRYPTION_KEY");
+
+            // Should not query again for a key not in backup
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            await jest.runAllTimersAsync();
+
+            expect(spy).toHaveBeenCalledTimes(1);
+
+            // advance time to retry
+            jest.advanceTimersByTime(BACKOFF_TIME + 10);
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            await jest.runAllTimersAsync();
+
+            expect(spy).toHaveBeenCalledTimes(2);
+            await expect(spy.mock.results[1].value).rejects.toThrow(
+                "Failed to get key from backup: MISSING_DECRYPTION_KEY",
+            );
+        });
+
+        it("Should stop properly", async () => {
+            // Simulate a call to stop while request is in flight
+            const blockOnServerRequest = defer<void>();
+            const requestRoomKeyCalled = defer<void>();
+
+            // Mock the request to block
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`, async (url, request) => {
+                requestRoomKeyCalled.resolve();
+                await blockOnServerRequest.promise;
+                return mockCipherKey;
+            });
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA1");
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA2");
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA3");
+
+            await requestRoomKeyCalled.promise;
+            downloader.stop();
+
+            blockOnServerRequest.resolve();
+
+            // let the first request complete
+            await jest.runAllTimersAsync();
+
+            expect(mockRustBackupManager.importBackedUpRoomKeys).not.toHaveBeenCalled();
+            expect(
+                fetchMock.calls(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`).length,
+            ).toStrictEqual(1);
+        });
+    });
+
+    describe("Given no usable backup available", () => {
+        let getConfigSpy: SpyInstance;
+
+        beforeEach(async () => {
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(null);
+            mockOlmMachine.getBackupKeys.mockResolvedValue(null);
+
+            // @ts-ignore access to private function
+            getConfigSpy = jest.spyOn(downloader, "getOrCreateBackupConfiguration");
+        });
+
+        it("Should not query server if no backup", async () => {
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", {
+                status: 404,
+                body: { errcode: "M_NOT_FOUND", error: "No current backup version." },
+            });
+
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+
+            await jest.runAllTimersAsync();
+
+            expect(getConfigSpy).toHaveBeenCalledTimes(1);
+            expect(getConfigSpy).toHaveReturnedWith(Promise.resolve(null));
+        });
+
+        it("Should not query server if backup not active", async () => {
+            // there is a backup
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+
+            // but it's not trusted
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(null);
+
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+
+            await jest.runAllTimersAsync();
+
+            expect(getConfigSpy).toHaveBeenCalledTimes(1);
+            expect(getConfigSpy).toHaveReturnedWith(Promise.resolve(null));
+        });
+
+        it("Should stop if backup key is not cached", async () => {
+            // there is a backup
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+            // it is trusted
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA.version!);
+            // but the key is not cached
+            mockOlmMachine.getBackupKeys.mockResolvedValue(null);
+
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+
+            await jest.runAllTimersAsync();
+
+            expect(getConfigSpy).toHaveBeenCalledTimes(1);
+            expect(getConfigSpy).toHaveReturnedWith(Promise.resolve(null));
+        });
+
+        it("Should stop if backup key cached as wrong version", async () => {
+            // there is a backup
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+            // it is trusted
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA.version!);
+            // but the cached key has the wrong version
+            mockOlmMachine.getBackupKeys.mockResolvedValue({
+                backupVersion: "0",
+                decryptionKey: RustSdkCryptoJs.BackupDecryptionKey.fromBase64(TestData.BACKUP_DECRYPTION_KEY_BASE64),
+            } as unknown as RustSdkCryptoJs.BackupKeys);
+
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+
+            await jest.runAllTimersAsync();
+
+            expect(getConfigSpy).toHaveBeenCalledTimes(1);
+            expect(getConfigSpy).toHaveReturnedWith(Promise.resolve(null));
+        });
+
+        it("Should stop if backup key version does not match the active one", async () => {
+            // there is a backup
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+            // The sdk is out of sync, the trusted version is the old one
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue("0");
+            // key for old backup cached
+            mockOlmMachine.getBackupKeys.mockResolvedValue({
+                backupVersion: "0",
+                decryptionKey: RustSdkCryptoJs.BackupDecryptionKey.fromBase64(TestData.BACKUP_DECRYPTION_KEY_BASE64),
+            } as unknown as RustSdkCryptoJs.BackupKeys);
+
+            downloader.onDecryptionKeyMissingError("!roomId", "sessionId");
+
+            await jest.runAllTimersAsync();
+
+            expect(getConfigSpy).toHaveBeenCalledTimes(1);
+            expect(getConfigSpy).toHaveReturnedWith(Promise.resolve(null));
+        });
+    });
+
+    describe("Given Backup state update", () => {
+        it("After initial sync, when backup becomes trusted it should request keys for past requests", async () => {
+            // there is a backup
+            mockRustBackupManager.requestKeyBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA);
+
+            // but at this point it's not trusted and we don't have the key
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(null);
+            mockOlmMachine.getBackupKeys.mockResolvedValue(null);
+
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`, mockCipherKey);
+
+            const a0Imported = expectSessionImported("!roomA", "sessionA0");
+            const a1Imported = expectSessionImported("!roomA", "sessionA1");
+            const b1Imported = expectSessionImported("!roomB", "sessionB1");
+            const c1Imported = expectSessionImported("!roomC", "sessionC1");
+
+            // During initial sync several keys are requested
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA1");
+            downloader.onDecryptionKeyMissingError("!roomB", "sessionB1");
+            downloader.onDecryptionKeyMissingError("!roomC", "sessionC1");
+            await jest.runAllTimersAsync();
+
+            // @ts-ignore access to private property
+            expect(downloader.hasConfigurationProblem).toEqual(true);
+
+            // Now the backup becomes trusted
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA.version!);
+            // And we have the key in cache
+            mockOlmMachine.getBackupKeys.mockResolvedValue({
+                backupVersion: TestData.SIGNED_BACKUP_DATA.version!,
+                decryptionKey: RustSdkCryptoJs.BackupDecryptionKey.fromBase64(TestData.BACKUP_DECRYPTION_KEY_BASE64),
+            } as unknown as RustSdkCryptoJs.BackupKeys);
+
+            // In that case the sdk would fire a backup status update
+            mockEmitter.emit(CryptoEvent.KeyBackupStatus, true);
+
+            await jest.runAllTimersAsync();
+
+            await a0Imported;
+            await a1Imported;
+            await b1Imported;
+            await c1Imported;
+        });
+    });
+
+    describe("Error cases", () => {
+        beforeEach(async () => {
+            // there is a backup
+            mockRustBackupManager.requestKeyBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA);
+            // It's trusted
+            mockRustBackupManager.getActiveBackupVersion.mockResolvedValue(TestData.SIGNED_BACKUP_DATA.version!);
+            // And we have the key in cache
+            mockOlmMachine.getBackupKeys.mockResolvedValue({
+                backupVersion: TestData.SIGNED_BACKUP_DATA.version!,
+                decryptionKey: RustSdkCryptoJs.BackupDecryptionKey.fromBase64(TestData.BACKUP_DECRYPTION_KEY_BASE64),
+            } as unknown as RustSdkCryptoJs.BackupKeys);
+        });
+
+        it("Should wait on rate limit error", async () => {
+            // simulate rate limit error
+            fetchMock.get(
+                `express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`,
+                {
+                    status: 429,
+                    body: {
+                        errcode: "M_LIMIT_EXCEEDED",
+                        error: "Too many requests",
+                        retry_after_ms: 5000,
+                    },
+                },
+                { overwriteRoutes: true },
+            );
+
+            const keyImported = expectSessionImported("!roomA", "sessionA0");
+
+            // @ts-ignore
+            const originalImplementation = downloader.queryKeyBackup.bind(downloader);
+
+            // @ts-ignore access to private function
+            const keyQuerySpy: SpyInstance = jest.spyOn(downloader, "queryKeyBackup");
+            const rateDeferred = defer<void>();
+
+            keyQuerySpy.mockImplementation(
+                // @ts-ignore
+                async (targetRoomId: string, targetSessionId: string, configuration: any) => {
+                    try {
+                        return await originalImplementation(targetRoomId, targetSessionId, configuration);
+                    } catch (err: any) {
+                        if (err.name === "KeyDownloadRateLimitError") {
+                            rateDeferred.resolve();
+                        }
+                        throw err;
+                    }
+                },
+            );
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+
+            await rateDeferred.promise;
+            expect(keyQuerySpy).toHaveBeenCalledTimes(1);
+            await expect(keyQuerySpy.mock.results[0].value).rejects.toThrow(
+                "Failed to get key from backup: rate limited",
+            );
+
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`, mockCipherKey, {
+                overwriteRoutes: true,
+            });
+
+            // Advance less than the retry_after_ms
+            jest.advanceTimersByTime(100);
+            // let any pending callbacks in PromiseJobs run
+            await Promise.resolve();
+            // no additional call should have been made
+            expect(keyQuerySpy).toHaveBeenCalledTimes(1);
+
+            // The loop should resume after the retry_after_ms
+            jest.advanceTimersByTime(5000);
+            // let any pending callbacks in PromiseJobs run
+            await Promise.resolve();
+
+            await keyImported;
+            expect(keyQuerySpy).toHaveBeenCalledTimes(2);
+        });
+
+        it("After a network error the same key is retried", async () => {
+            // simulate connectivity error
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`, () => {
+                throw new ConnectionError("fetch failed", new Error("fetch failed"));
+            });
+
+            // @ts-ignore
+            const originalImplementation = downloader.queryKeyBackup.bind(downloader);
+
+            // @ts-ignore
+            const keyQuerySpy: SpyInstance = jest.spyOn(downloader, "queryKeyBackup");
+            const errorDeferred = defer<void>();
+
+            keyQuerySpy.mockImplementation(
+                // @ts-ignore
+                async (targetRoomId: string, targetSessionId: string, configuration: any) => {
+                    try {
+                        return await originalImplementation(targetRoomId, targetSessionId, configuration);
+                    } catch (err: any) {
+                        if (err.name === "KeyDownloadError") {
+                            errorDeferred.resolve();
+                        }
+                        throw err;
+                    }
+                },
+            );
+            const keyImported = expectSessionImported("!roomA", "sessionA0");
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            await errorDeferred.promise;
+            await Promise.resolve();
+
+            await expect(keyQuerySpy.mock.results[0].value).rejects.toThrow(
+                "Failed to get key from backup: NETWORK_ERROR",
+            );
+
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`, mockCipherKey, {
+                overwriteRoutes: true,
+            });
+
+            // Advance less than the retry_after_ms
+            jest.advanceTimersByTime(100);
+            // let any pending callbacks in PromiseJobs run
+            await Promise.resolve();
+            // no additional call should have been made
+            expect(keyQuerySpy).toHaveBeenCalledTimes(1);
+
+            // The loop should resume after the retry_after_ms
+            jest.advanceTimersByTime(BACKOFF_TIME + 100);
+            await Promise.resolve();
+
+            await keyImported;
+        });
+
+        it("On Unknown error on import skip the key and continue", async () => {
+            const keyImported = defer<void>();
+            mockRustBackupManager.importBackedUpRoomKeys
+                .mockImplementationOnce(async () => {
+                    throw new Error("Didn't work");
+                })
+                .mockImplementationOnce(async (sessions) => {
+                    const roomId = sessions[0].room_id;
+                    const sessionId = sessions[0].session_id;
+                    if (roomId === "!roomA" && sessionId === "sessionA1") {
+                        keyImported.resolve();
+                    }
+                    return;
+                });
+
+            fetchMock.get(`express:/_matrix/client/v3/room_keys/keys/:roomId/:sessionId`, mockCipherKey, {
+                overwriteRoutes: true,
+            });
+
+            // @ts-ignore access to private function
+            const keyQuerySpy: SpyInstance = jest.spyOn(downloader, "queryKeyBackup");
+
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA0");
+            downloader.onDecryptionKeyMissingError("!roomA", "sessionA1");
+            await jest.runAllTimersAsync();
+
+            await keyImported.promise;
+
+            expect(keyQuerySpy).toHaveBeenCalledTimes(2);
+            expect(mockRustBackupManager.importBackedUpRoomKeys).toHaveBeenCalledTimes(2);
+        });
+    });
+});

spec/unit/rust-crypto/RoomEncryptor.spec.ts

@@ -0,0 +1,31 @@
+/*
+ *
+ * Copyright 2023 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+import { HistoryVisibility as RustHistoryVisibility } from "@matrix-org/matrix-sdk-crypto-wasm";
+
+import { HistoryVisibility } from "../../../src";
+import { toRustHistoryVisibility } from "../../../src/rust-crypto/RoomEncryptor";
+
+it.each([
+    [HistoryVisibility.Invited, RustHistoryVisibility.Invited],
+    [HistoryVisibility.Joined, RustHistoryVisibility.Joined],
+    [HistoryVisibility.Shared, RustHistoryVisibility.Shared],
+    [HistoryVisibility.WorldReadable, RustHistoryVisibility.WorldReadable],
+])("JS HistoryVisibility to Rust HistoryVisibility: converts %s to %s", (historyVisibility, expected) => {
+    expect(toRustHistoryVisibility(historyVisibility)).toBe(expected);
+});

spec/unit/rust-crypto/rust-crypto.spec.ts

@@ -50,6 +50,8 @@ import {
 import * as testData from "../../test-utils/test-data";
 import { defer } from "../../../src/utils";
 import { logger } from "../../../src/logger";
+import { OutgoingRequestsManager } from "../../../src/rust-crypto/OutgoingRequestsManager";
+import { Curve25519AuthData } from "../../../src/crypto-api/keybackup";
 
 const TEST_USER = "@alice:example.com";
 const TEST_DEVICE_ID = "TEST_DEVICE";
@@ -64,6 +66,9 @@ describe("initRustCrypto", () => {
         return {
             registerRoomKeyUpdatedCallback: jest.fn(),
             registerUserIdentityUpdatedCallback: jest.fn(),
+            getSecretsFromInbox: jest.fn().mockResolvedValue(["dGhpc2lzYWZha2VzZWNyZXQ="]),
+            deleteSecretsFromInbox: jest.fn(),
+            registerReceiveSecretCallback: jest.fn(),
             outgoingRequests: jest.fn(),
         } as unknown as Mocked<OlmMachine>;
     }
@@ -108,9 +113,35 @@ describe("initRustCrypto", () => {
 
         expect(OlmMachine.initialize).toHaveBeenCalledWith(expect.anything(), expect.anything(), undefined, undefined);
     });
+
+    it("Should get secrets from inbox on start", async () => {
+        const testOlmMachine = makeTestOlmMachine() as OlmMachine;
+        jest.spyOn(OlmMachine, "initialize").mockResolvedValue(testOlmMachine);
+
+        await initRustCrypto(
+            logger,
+            {} as MatrixClient["http"],
+            TEST_USER,
+            TEST_DEVICE_ID,
+            {} as ServerSideSecretStorage,
+            {} as CryptoCallbacks,
+            "storePrefix",
+            "storePassphrase",
+        );
+
+        expect(testOlmMachine.getSecretsFromInbox).toHaveBeenCalledWith("m.megolm_backup.v1");
+    });
 });
 
 describe("RustCrypto", () => {
+    it("getVersion() should return the current version of the rust sdk and vodozemac", async () => {
+        const rustCrypto = await makeTestRustCrypto();
+        const versions = RustSdkCryptoJs.getVersions();
+        expect(rustCrypto.getVersion()).toBe(
+            `Rust SDK ${versions.matrix_sdk_crypto} (${versions.git_sha}), Vodozemac ${versions.vodozemac}`,
+        );
+    });
+
     describe(".importRoomKeys and .exportRoomKeys", () => {
         let rustCrypto: RustCrypto;
 
@@ -318,6 +349,8 @@ describe("RustCrypto", () => {
                 makeOutgoingRequest: jest.fn(),
             } as unknown as Mocked<OutgoingRequestProcessor>;
 
+            const outgoingRequestsManager = new OutgoingRequestsManager(logger, olmMachine, outgoingRequestProcessor);
+
             rustCrypto = new RustCrypto(
                 logger,
                 olmMachine,
@@ -328,6 +361,7 @@ describe("RustCrypto", () => {
                 {} as CryptoCallbacks,
             );
             rustCrypto["outgoingRequestProcessor"] = outgoingRequestProcessor;
+            rustCrypto["outgoingRequestsManager"] = outgoingRequestsManager;
         });
 
         it("should poll for outgoing messages and send them", async () => {
@@ -366,50 +400,6 @@ describe("RustCrypto", () => {
             await awaitCallToMakeOutgoingRequest();
             expect(olmMachine.outgoingRequests).toHaveBeenCalledTimes(2);
         });
-
-        it("stops looping when stop() is called", async () => {
-            for (let i = 0; i < 5; i++) {
-                outgoingRequestQueue.push([new KeysQueryRequest("1234", "{}")]);
-            }
-
-            let makeRequestPromise = awaitCallToMakeOutgoingRequest();
-
-            rustCrypto.onSyncCompleted({});
-
-            expect(rustCrypto["outgoingRequestLoopRunning"]).toBeTruthy();
-
-            // go a couple of times round the loop
-            let resolveMakeRequest = await makeRequestPromise;
-            makeRequestPromise = awaitCallToMakeOutgoingRequest();
-            resolveMakeRequest();
-
-            resolveMakeRequest = await makeRequestPromise;
-            makeRequestPromise = awaitCallToMakeOutgoingRequest();
-            resolveMakeRequest();
-
-            // a second sync while this is going on shouldn't make any difference
-            rustCrypto.onSyncCompleted({});
-
-            resolveMakeRequest = await makeRequestPromise;
-            outgoingRequestProcessor.makeOutgoingRequest.mockReset();
-            resolveMakeRequest();
-
-            // now stop...
-            rustCrypto.stop();
-
-            // which should (eventually) cause the loop to stop with no further calls to outgoingRequests
-            olmMachine.outgoingRequests.mockReset();
-
-            await new Promise((resolve) => {
-                setTimeout(resolve, 100);
-            });
-            expect(rustCrypto["outgoingRequestLoopRunning"]).toBeFalsy();
-            expect(outgoingRequestProcessor.makeOutgoingRequest).not.toHaveBeenCalled();
-            expect(olmMachine.outgoingRequests).not.toHaveBeenCalled();
-
-            // we sent three, so there should be 2 left
-            expect(outgoingRequestQueue.length).toEqual(2);
-        });
     });
 
     describe(".getEventEncryptionInfo", () => {
@@ -484,6 +474,26 @@ describe("RustCrypto", () => {
             expect(olmMachine.getRoomEventEncryptionInfo).not.toHaveBeenCalled();
         });
 
+        it("should handle decryption failures", async () => {
+            const event = mkEvent({
+                event: true,
+                type: "m.room.encrypted",
+                content: { algorithm: "fake_alg" },
+                room: "!room:id",
+            });
+            event.event.event_id = "$event:id";
+            const mockCryptoBackend = {
+                decryptEvent: () => {
+                    throw new Error("UISI");
+                },
+            };
+            await event.attemptDecryption(mockCryptoBackend as unknown as CryptoBackend);
+
+            const res = await rustCrypto.getEncryptionInfoForEvent(event);
+            expect(res).toBe(null);
+            expect(olmMachine.getRoomEventEncryptionInfo).not.toHaveBeenCalled();
+        });
+
         it("passes the event into the OlmMachine", async () => {
             const encryptedEvent = await makeEncryptedEvent();
             const res = await rustCrypto.getEncryptionInfoForEvent(encryptedEvent);
@@ -636,6 +646,7 @@ describe("RustCrypto", () => {
 
         it("should call getDevice", async () => {
             olmMachine.getDevice.mockResolvedValue({
+                free: jest.fn(),
                 isCrossSigningTrusted: jest.fn().mockReturnValue(false),
                 isLocallyTrusted: jest.fn().mockReturnValue(false),
                 isCrossSignedByOwner: jest.fn().mockReturnValue(false),
@@ -862,7 +873,7 @@ describe("RustCrypto", () => {
         });
 
         it("returns a verified UserVerificationStatus when the UserIdentity is verified", async () => {
-            olmMachine.getIdentity.mockResolvedValue({ isVerified: jest.fn().mockReturnValue(true) });
+            olmMachine.getIdentity.mockResolvedValue({ free: jest.fn(), isVerified: jest.fn().mockReturnValue(true) });
 
             const userVerificationStatus = await rustCrypto.getUserVerificationStatus(testData.TEST_USER_ID);
             expect(userVerificationStatus.isVerified()).toBeTruthy();
@@ -921,6 +932,48 @@ describe("RustCrypto", () => {
             await rustCrypto.onUserIdentityUpdated(new RustSdkCryptoJs.UserId(testData.TEST_USER_ID));
             expect(await keyBackupStatusPromise).toBe(true);
         });
+
+        it("does not back up keys that came from backup", async () => {
+            const rustCrypto = await makeTestRustCrypto();
+            const olmMachine: OlmMachine = rustCrypto["olmMachine"];
+
+            await olmMachine.enableBackupV1(
+                (testData.SIGNED_BACKUP_DATA.auth_data as Curve25519AuthData).public_key,
+                testData.SIGNED_BACKUP_DATA.version!,
+            );
+
+            // we import two keys: one "from backup", and one "from export"
+            const [backedUpRoomKey, exportedRoomKey] = testData.MEGOLM_SESSION_DATA_ARRAY;
+            await rustCrypto.importBackedUpRoomKeys([backedUpRoomKey]);
+            await rustCrypto.importRoomKeys([exportedRoomKey]);
+
+            // we ask for the keys that should be backed up
+            const roomKeysRequest = await olmMachine.backupRoomKeys();
+            expect(roomKeysRequest).toBeTruthy();
+            const roomKeys = JSON.parse(roomKeysRequest!.body);
+
+            // we expect that the key "from export" is present
+            expect(roomKeys).toMatchObject({
+                rooms: {
+                    [exportedRoomKey.room_id]: {
+                        sessions: {
+                            [exportedRoomKey.session_id]: {},
+                        },
+                    },
+                },
+            });
+
+            // we expect that the key "from backup" is not present
+            expect(roomKeys).not.toMatchObject({
+                rooms: {
+                    [backedUpRoomKey.room_id]: {
+                        sessions: {
+                            [backedUpRoomKey.session_id]: {},
+                        },
+                    },
+                },
+            });
+        });
     });
 });
 

spec/unit/rust-crypto/verification.spec.ts

@@ -62,9 +62,13 @@ describe("VerificationRequest", () => {
 
     describe("startVerification", () => {
         let request: RustVerificationRequest;
+        let machine: Mocked<RustSdkCryptoJs.OlmMachine>;
+        let inner: Mocked<RustSdkCryptoJs.VerificationRequest>;
 
         beforeEach(() => {
-            request = makeTestRequest();
+            inner = makeMockedInner();
+            machine = { getDevice: jest.fn() } as unknown as Mocked<RustSdkCryptoJs.OlmMachine>;
+            request = makeTestRequest(inner, machine);
         });
 
         it("does not permit methods other than SAS", async () => {
@@ -73,7 +77,15 @@ describe("VerificationRequest", () => {
             );
         });
 
+        it("raises an error if the other device is unknown", async () => {
+            await expect(request.startVerification("m.sas.v1")).rejects.toThrow(
+                "startVerification(): other device is unknown",
+            );
+        });
+
         it("raises an error if starting verification does not produce a verifier", async () => {
+            jest.spyOn(inner, "otherDeviceId", "get").mockReturnValue(new RustSdkCryptoJs.DeviceId("other_device"));
+            machine.getDevice.mockResolvedValue({} as RustSdkCryptoJs.Device);
             await expect(request.startVerification("m.sas.v1")).rejects.toThrow(
                 "Still no verifier after startSas() call",
             );
@@ -118,11 +130,13 @@ describe("isVerificationEvent", () => {
 /** build a RustVerificationRequest with default parameters */
 function makeTestRequest(
     inner?: RustSdkCryptoJs.VerificationRequest,
+    olmMachine?: RustSdkCryptoJs.OlmMachine,
     outgoingRequestProcessor?: OutgoingRequestProcessor,
 ): RustVerificationRequest {
     inner ??= makeMockedInner();
+    olmMachine ??= {} as RustSdkCryptoJs.OlmMachine;
     outgoingRequestProcessor ??= {} as OutgoingRequestProcessor;
-    return new RustVerificationRequest(inner, outgoingRequestProcessor, []);
+    return new RustVerificationRequest(olmMachine, inner, outgoingRequestProcessor, []);
 }
 
 /** Mock up a rust-side VerificationRequest */
@@ -133,5 +147,8 @@ function makeMockedInner(): Mocked<RustSdkCryptoJs.VerificationRequest> {
         phase: jest.fn().mockReturnValue(RustSdkCryptoJs.VerificationRequestPhase.Created),
         isPassive: jest.fn().mockReturnValue(false),
         timeRemainingMillis: jest.fn(),
+        get otherDeviceId() {
+            return undefined;
+        },
     } as unknown as Mocked<RustSdkCryptoJs.VerificationRequest>;
 }

spec/unit/stores/indexeddb.spec.ts

@@ -16,8 +16,9 @@ limitations under the License.
 
 import "fake-indexeddb/auto";
 import "jest-localstorage-mock";
+import { IDBFactory } from "fake-indexeddb";
 
-import { IndexedDBStore, IStateEventWithRoomId, MemoryStore } from "../../../src";
+import { IndexedDBStore, IStateEventWithRoomId, MemoryStore, User, UserEvent } from "../../../src";
 import { emitPromise } from "../../test-utils/test-utils";
 import { LocalIndexedDBStoreBackend } from "../../../src/store/indexeddb-local-backend";
 import { defer } from "../../../src/utils";
@@ -76,6 +77,62 @@ describe("IndexedDBStore", () => {
         expect(await store.getOutOfBandMembers(roomId)).toHaveLength(2);
     });
 
+    it("Should load presence events on startup", async () => {
+        // 1. Create idb database
+        const indexedDB = new IDBFactory();
+        const setupDefer = defer<Event>();
+        const req = indexedDB.open("matrix-js-sdk:db3", 1);
+        let db: IDBDatabase;
+        req.onupgradeneeded = () => {
+            db = req.result;
+            db.createObjectStore("users", { keyPath: ["userId"] });
+            db.createObjectStore("accountData", { keyPath: ["type"] });
+            db.createObjectStore("sync", { keyPath: ["clobber"] });
+        };
+        req.onsuccess = setupDefer.resolve;
+        await setupDefer.promise;
+
+        // 2. Fill in user presence data
+        const writeDefer = defer<Event>();
+        const transaction = db!.transaction(["users"], "readwrite");
+        const objectStore = transaction.objectStore("users");
+        const request = objectStore.put({
+            userId: "@alice:matrix.org",
+            event: {
+                content: {
+                    presence: "online",
+                },
+                sender: "@alice:matrix.org",
+                type: "m.presence",
+            },
+        });
+        request.onsuccess = writeDefer.resolve;
+        await writeDefer.promise;
+
+        // 3. Close database
+        req.result.close();
+
+        // 2. Check if the code loads presence events
+        const store = new IndexedDBStore({
+            indexedDB: indexedDB,
+            dbName: "db3",
+            localStorage,
+        });
+        let userCreated = false;
+        let presenceEventEmitted = false;
+        store.setUserCreator((id: string) => {
+            userCreated = true;
+            const user = new User(id);
+            user.on(UserEvent.Presence, () => {
+                presenceEventEmitted = true;
+            });
+            return user;
+        });
+        await store.startup();
+        expect(userCreated).toBe(true);
+        expect(presenceEventEmitted).toBe(true);
+    });
+
     it("should use MemoryStore methods for pending events if no localStorage", async () => {
         jest.spyOn(MemoryStore.prototype, "setPendingEvents");
         jest.spyOn(MemoryStore.prototype, "getPendingEvents");

spec/unit/timeline-window.spec.ts

@@ -22,12 +22,15 @@ import { Room } from "../../src/models/room";
 import { EventTimeline } from "../../src/models/event-timeline";
 import { TimelineIndex, TimelineWindow } from "../../src/timeline-window";
 import { mkMessage } from "../test-utils/test-utils";
+import { MatrixEvent } from "../../src/models/event";
 
 const ROOM_ID = "roomId";
 const USER_ID = "userId";
 const mockClient = {
     getEventTimeline: jest.fn(),
     paginateEventTimeline: jest.fn(),
+    supportsThreads: jest.fn(),
+    getUserId: jest.fn().mockReturnValue(USER_ID),
 } as unknown as MockedObject<MatrixClient>;
 
 /*
@@ -64,6 +67,23 @@ function addEventsToTimeline(timeline: EventTimeline, numEvents: number, toStart
     }
 }
 
+function createEvents(numEvents: number): Array<MatrixEvent> {
+    const ret = [];
+
+    for (let i = 0; i < numEvents; i++) {
+        ret.push(
+            mkMessage({
+                room: ROOM_ID,
+                user: USER_ID,
+                event: true,
+                unsigned: { age: 1 },
+            }),
+        );
+    }
+
+    return ret;
+}
+
 /*
  * create a pair of linked timelines
  */
@@ -412,4 +432,46 @@ describe("TimelineWindow", function () {
             expect(timelineWindow.canPaginate(EventTimeline.FORWARDS)).toBe(true);
         });
     });
+
+    function idsOf(events: Array<MatrixEvent>): Array<string> {
+        return events.map((e) => (e ? e.getId() ?? "MISSING_ID" : "MISSING_EVENT"));
+    }
+
+    describe("removing events", () => {
+        it("should shorten if removing an event within the window makes it overflow", function () {
+            // Given a room with events in two timelines
+            const room = new Room(ROOM_ID, mockClient, USER_ID, { timelineSupport: true });
+            const timelineSet = room.getUnfilteredTimelineSet();
+            const liveTimeline = room.getLiveTimeline();
+            const oldTimeline = room.addTimeline();
+            liveTimeline.setNeighbouringTimeline(oldTimeline, EventTimeline.BACKWARDS);
+            oldTimeline.setNeighbouringTimeline(liveTimeline, EventTimeline.FORWARDS);
+
+            const oldEvents = createEvents(5);
+            const liveEvents = createEvents(5);
+            const [, , e3, e4, e5] = oldEvents;
+            const [, e7, e8, e9, e10] = liveEvents;
+            room.addLiveEvents(liveEvents);
+            room.addEventsToTimeline(oldEvents, true, oldTimeline);
+
+            // And 2 windows over the timelines in this room
+            const oldWindow = new TimelineWindow(mockClient, timelineSet);
+            oldWindow.load(e5.getId(), 6);
+            expect(idsOf(oldWindow.getEvents())).toEqual(idsOf([e5, e4, e3]));
+
+            const newWindow = new TimelineWindow(mockClient, timelineSet);
+            newWindow.load(e9.getId(), 4);
+            expect(idsOf(newWindow.getEvents())).toEqual(idsOf([e7, e8, e9, e10]));
+
+            // When I remove an event
+            room.removeEvent(e8.getId()!);
+
+            // Then the affected timeline is shortened (because it would have
+            // been too long with the removed event gone)
+            expect(idsOf(newWindow.getEvents())).toEqual(idsOf([e7, e9, e10]));
+
+            // And the unaffected one is not
+            expect(idsOf(oldWindow.getEvents())).toEqual(idsOf([e5, e4, e3]));
+        });
+    });
 });

src/@types/auth.ts

@@ -250,12 +250,6 @@ export interface LoginTokenPostResponse {
      * The token to use with `m.login.token` to authenticate.
      */
     login_token: string;
-    /**
-     * Expiration in seconds.
-     *
-     * @deprecated this is only provided for compatibility with original revision of [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882).
-     */
-    expires_in: number;
     /**
      * Expiration in milliseconds.
      */

src/@types/event.ts

@@ -55,6 +55,7 @@ export enum EventType {
     CallReplaces = "m.call.replaces",
     CallAssertedIdentity = "m.call.asserted_identity",
     CallAssertedIdentityPrefix = "org.matrix.call.asserted_identity",
+    CallEncryptionKeysPrefix = "io.element.call.encryption_keys",
     KeyVerificationRequest = "m.key.verification.request",
     KeyVerificationStart = "m.key.verification.start",
     KeyVerificationCancel = "m.key.verification.cancel",
@@ -93,6 +94,9 @@ export enum EventType {
     // Group call events
     GroupCallPrefix = "org.matrix.msc3401.call",
     GroupCallMemberPrefix = "org.matrix.msc3401.call.member",
+
+    // MatrixRTC events
+    CallNotify = "org.matrix.msc4075.call.notify",
 }
 
 export enum RelationType {

src/base64.ts

@@ -0,0 +1,88 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Base64 encoding and decoding utilities
+ */
+
+/**
+ * Encode a typed array of uint8 as base64.
+ * @param uint8Array - The data to encode.
+ * @returns The base64.
+ */
+export function encodeBase64(uint8Array: ArrayBuffer | Uint8Array): string {
+    // A brief note on the state of base64 encoding in Javascript.
+    // As of 2023, there is still no common native impl between both browsers and
+    // node. Older Webpack provides an impl for Buffer and there is a polyfill class
+    // for it. There are also plenty of pure js impls, eg. base64-js which has 2336
+    // dependents at current count. Using this would probably be fine although it's
+    // a little under-docced and run by an individual. The node impl works fine,
+    // the browser impl works but predates Uint8Array and so only uses strings.
+    // Right now, switching between native (or polyfilled) impls like this feels
+    // like the least bad option, but... *shrugs*.
+    if (typeof Buffer === "function") {
+        return Buffer.from(uint8Array).toString("base64");
+    } else if (typeof btoa === "function" && uint8Array instanceof Uint8Array) {
+        // ArrayBuffer is a node concept so the param should always be a Uint8Array on
+        // the browser. We need to check because ArrayBuffers don't have reduce.
+        return btoa(uint8Array.reduce((acc, current) => acc + String.fromCharCode(current), ""));
+    } else {
+        throw new Error("No base64 impl found!");
+    }
+}
+
+/**
+ * Encode a typed array of uint8 as unpadded base64.
+ * @param uint8Array - The data to encode.
+ * @returns The unpadded base64.
+ */
+export function encodeUnpaddedBase64(uint8Array: ArrayBuffer | Uint8Array): string {
+    return encodeBase64(uint8Array).replace(/={1,2}$/, "");
+}
+
+/**
+ * Encode a typed array of uint8 as unpadded base64 using the URL-safe encoding.
+ * @param uint8Array - The data to encode.
+ * @returns The unpadded base64.
+ */
+export function encodeUnpaddedBase64Url(uint8Array: ArrayBuffer | Uint8Array): string {
+    return encodeUnpaddedBase64(uint8Array).replace("+", "-").replace("/", "_");
+}
+
+/**
+ * Decode a base64 (or base64url) string to a typed array of uint8.
+ * @param base64 - The base64 to decode.
+ * @returns The decoded data.
+ */
+export function decodeBase64(base64: string): Uint8Array {
+    // See encodeBase64 for a short treatise on base64 en/decoding in JS
+    if (typeof Buffer === "function") {
+        return Buffer.from(base64, "base64");
+    } else if (typeof atob === "function") {
+        const itFunc = function* (): Generator<number> {
+            const decoded = atob(
+                // built-in atob doesn't support base64url: convert so we support either
+                base64.replace("-", "+").replace("_", "/"),
+            );
+            for (let i = 0; i < decoded.length; ++i) {
+                yield decoded.charCodeAt(i);
+            }
+        };
+        return Uint8Array.from(itFunc());
+    } else {
+        throw new Error("No base64 impl found!");
+    }
+}

src/client.ts

@@ -47,11 +47,11 @@ import { Direction, EventTimeline } from "./models/event-timeline";
 import { IActionsObject, PushProcessor } from "./pushprocessor";
 import { AutoDiscovery, AutoDiscoveryAction } from "./autodiscovery";
 import * as olmlib from "./crypto/olmlib";
-import { decodeBase64, encodeBase64 } from "./crypto/olmlib";
+import { decodeBase64, encodeBase64 } from "./base64";
 import { IExportedDevice as IExportedOlmDevice } from "./crypto/OlmDevice";
 import { IOlmDevice } from "./crypto/algorithms/megolm";
 import { TypedReEmitter } from "./ReEmitter";
-import { IRoomEncryption, RoomList } from "./crypto/RoomList";
+import { IRoomEncryption } from "./crypto/RoomList";
 import { logger, Logger } from "./logger";
 import { SERVICE_TYPES } from "./service-types";
 import {
@@ -536,21 +536,11 @@ export interface IThreadsCapability extends ICapability {}
 
 export interface IGetLoginTokenCapability extends ICapability {}
 
-/**
- * @deprecated use {@link IGetLoginTokenCapability} instead
- */
-export type IMSC3882GetLoginTokenCapability = IGetLoginTokenCapability;
-
 export const GET_LOGIN_TOKEN_CAPABILITY = new NamespacedValue(
     "m.get_login_token",
     "org.matrix.msc3882.get_login_token",
 );
 
-/**
- * @deprecated use {@link GET_LOGIN_TOKEN_CAPABILITY} instead
- */
-export const UNSTABLE_MSC3882_CAPABILITY = GET_LOGIN_TOKEN_CAPABILITY;
-
 export const UNSTABLE_MSC2666_SHARED_ROOMS = "uk.half-shot.msc2666";
 export const UNSTABLE_MSC2666_MUTUAL_ROOMS = "uk.half-shot.msc2666.mutual_rooms";
 export const UNSTABLE_MSC2666_QUERY_MUTUAL_ROOMS = "uk.half-shot.msc2666.query_mutual_rooms";
@@ -905,7 +895,7 @@ interface IRoomHierarchy {
 
 export interface TimestampToEventResponse {
     event_id: string;
-    origin_server_ts: string;
+    origin_server_ts: number;
 }
 
 interface IWhoamiResponse {
@@ -961,6 +951,7 @@ type CryptoEvents =
     | CryptoEvent.KeyBackupStatus
     | CryptoEvent.KeyBackupFailed
     | CryptoEvent.KeyBackupSessionsRemaining
+    | CryptoEvent.KeyBackupDecryptionKeyCached
     | CryptoEvent.RoomKeyRequest
     | CryptoEvent.RoomKeyRequestCancellation
     | CryptoEvent.VerificationRequest
@@ -1227,7 +1218,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     public reEmitter = new TypedReEmitter<EmittedEvents, ClientEventHandlerMap>(this);
     public olmVersion: [number, number, number] | null = null; // populated after initCrypto
     public usingExternalCrypto = false;
-    public store: Store;
+    private _store!: Store;
     public deviceId: string | null;
     public credentials: { userId: string | null };
 
@@ -1281,7 +1272,6 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     protected cryptoStore?: CryptoStore;
     protected verificationMethods?: VerificationMethod[];
     protected fallbackICEServerAllowed = false;
-    protected roomList: RoomList;
     protected syncApi?: SlidingSyncSdk | SyncApi;
     public roomNameGenerator?: ICreateClientOpts["roomNameGenerator"];
     public pushRules?: IPushRules;
@@ -1437,10 +1427,6 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         this.livekitServiceURL = opts.livekitServiceURL;
 
-        // List of which rooms have encryption enabled: separate from crypto because
-        // we still want to know which rooms are encrypted even if crypto is disabled:
-        // we don't want to start sending unencrypted events to them.
-        this.roomList = new RoomList(this.cryptoStore);
         this.roomNameGenerator = opts.roomNameGenerator;
 
         this.toDeviceMessageQueue = new ToDeviceMessageQueue(this);
@@ -1504,6 +1490,18 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         this.ignoredInvites = new IgnoredInvites(this);
         this._secretStorage = new ServerSideSecretStorageImpl(this, opts.cryptoCallbacks ?? {});
+
+        // having lots of event listeners is not unusual. 0 means "unlimited".
+        this.setMaxListeners(0);
+    }
+
+    public set store(newStore: Store) {
+        this._store = newStore;
+        this._store.setUserCreator((userId) => User.createUser(userId, this));
+    }
+
+    public get store(): Store {
+        return this._store;
     }
 
     /**
@@ -2230,10 +2228,6 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         this.logger.debug("Crypto: Starting up crypto store...");
         await this.cryptoStore.startup();
 
-        // initialise the list of encrypted rooms (whether or not crypto is enabled)
-        this.logger.debug("Crypto: initialising roomlist...");
-        await this.roomList.init();
-
         const userId = this.getUserId();
         if (userId === null) {
             throw new Error(
@@ -2248,15 +2242,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             );
         }
 
-        const crypto = new Crypto(
-            this,
-            userId,
-            this.deviceId,
-            this.store,
-            this.cryptoStore,
-            this.roomList,
-            this.verificationMethods!,
-        );
+        const crypto = new Crypto(this, userId, this.deviceId, this.store, this.cryptoStore, this.verificationMethods!);
 
         this.reEmitter.reEmit(crypto, [
             CryptoEvent.KeyBackupFailed,
@@ -2357,6 +2343,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             CryptoEvent.KeyBackupStatus,
             CryptoEvent.KeyBackupSessionsRemaining,
             CryptoEvent.KeyBackupFailed,
+            CryptoEvent.KeyBackupDecryptionKeyCached,
         ]);
     }
 
@@ -2391,6 +2378,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      *
      * @returns base64-encoded ed25519 key. Null if crypto is
      *    disabled.
+     *
+     * @deprecated Prefer {@link CryptoApi.getOwnDeviceKeys}
      */
     public getDeviceEd25519Key(): string | null {
         return this.crypto?.getDeviceEd25519Key() ?? null;
@@ -2401,6 +2390,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      *
      * @returns base64-encoded curve25519 key. Null if crypto is
      *    disabled.
+     *
+     * @deprecated Use {@link CryptoApi.getOwnDeviceKeys}
      */
     public getDeviceCurve25519Key(): string | null {
         return this.crypto?.getDeviceCurve25519Key() ?? null;
@@ -2679,12 +2670,12 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      *
      * This API is currently UNSTABLE and may change or be removed without notice.
      *
+     * It has no effect with the Rust crypto implementation.
+     *
      * @param value - whether error on unknown devices
      *
-     * @deprecated Prefer direct access to {@link CryptoApi.globalBlacklistUnverifiedDevices}:
-     *
      * ```ts
-     * client.getCrypto().globalBlacklistUnverifiedDevices = value;
+     * client.getCrypto().globalErrorOnUnknownDevices = value;
      * ```
      */
     public setGlobalErrorOnUnknownDevices(value: boolean): void {
@@ -3275,7 +3266,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         // we don't have an m.room.encrypted event, but that might be because
         // the server is hiding it from us. Check the store to see if it was
         // previously encrypted.
-        return this.roomList.isRoomEncrypted(roomId);
+        return this.crypto?.isRoomEncrypted(roomId) ?? false;
     }
 
     /**
@@ -3638,6 +3629,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     /**
      * Marks all group sessions as needing to be backed up and schedules them to
      * upload in the background as soon as possible.
+     *
+     * (This is done automatically as part of {@link CryptoApi.resetKeyBackup},
+     * so there is probably no need to call this manually.)
      */
     public async scheduleAllGroupSessionsForBackup(): Promise<void> {
         if (!this.crypto) {
@@ -3650,6 +3644,10 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     /**
      * Marks all group sessions as needing to be backed up without scheduling
      * them to upload in the background.
+     *
+     * (This is done automatically as part of {@link CryptoApi.resetKeyBackup},
+     * so there is probably no need to call this manually.)
+     *
      * @returns Promise which resolves to the number of sessions requiring a backup.
      */
     public flagAllGroupSessionsForBackup(): Promise<number> {
@@ -3969,10 +3967,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             backupDecryptor.free();
         }
 
-        await this.cryptoBackend.importRoomKeys(keys, {
+        await this.cryptoBackend.importBackedUpRoomKeys(keys, {
             progressCallback,
             untrusted,
-            source: "backup",
         });
 
         /// in case entering the passphrase would add a new signature?
@@ -4005,7 +4002,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             throw new Error("End-to-end encryption disabled");
         }
 
-        const roomEncryption = this.roomList.getRoomEncryption(roomId);
+        const roomEncryption = this.crypto?.getRoomEncryption(roomId);
         if (!roomEncryption) {
             // unknown room, or unencrypted room
             this.logger.error("Unknown room.  Not sharing decryption keys");
@@ -5166,7 +5163,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         const room = this.getRoom(event.getRoomId());
         if (room && this.credentials.userId) {
-            room.addLocalEchoReceipt(this.credentials.userId, event, receiptType);
+            room.addLocalEchoReceipt(this.credentials.userId, event, receiptType, unthreaded);
         }
         return promise;
     }
@@ -8044,50 +8041,23 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
     /**
      * Make a request for an `m.login.token` to be issued as per
-     * [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882).
-     * The server may require User-Interactive auth.
+     * https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv1loginget_token
      *
-     * Compatibility with unstable implementations of MSC3882 is deprecated and will be removed in a future release.
+     * The server may require User-Interactive auth.
      *
      * @param auth - Optional. Auth data to supply for User-Interactive auth.
      * @returns Promise which resolves: On success, the token response
      * or UIA auth data.
      */
     public async requestLoginToken(auth?: AuthDict): Promise<UIAResponse<LoginTokenPostResponse>> {
-        // use capabilities to determine which revision of the MSC is being used
-        const capabilities = await this.getCapabilities();
-
-        let endpoint: string;
-        if (capabilities[GET_LOGIN_TOKEN_CAPABILITY.name]) {
-            // use the stable endpoint
-            endpoint = `${ClientPrefix.V1}/login/get_token`;
-        } else if (capabilities[GET_LOGIN_TOKEN_CAPABILITY.altName!]) {
-            // newer unstable r1 endpoint
-            endpoint = `${ClientPrefix.Unstable}/org.matrix.msc3882/login/get_token`;
-        } else {
-            // old unstable r0 endpoint
-            endpoint = `${ClientPrefix.Unstable}/org.matrix.msc3882/login/token`;
-        }
-
         const body: UIARequest<{}> = { auth };
-        const res = await this.http.authedRequest<UIAResponse<LoginTokenPostResponse>>(
+        return this.http.authedRequest<UIAResponse<LoginTokenPostResponse>>(
             Method.Post,
-            endpoint,
+            "/login/get_token",
             undefined, // no query params
             body,
-            { prefix: "" },
+            { prefix: ClientPrefix.V1 },
         );
-
-        // the representation of expires_in changed from unstable revision 0 to unstable revision 1 so we cross populate
-        if ("login_token" in res) {
-            if (typeof res.expires_in_ms === "number") {
-                res.expires_in = Math.floor(res.expires_in_ms / 1000);
-            } else if (typeof res.expires_in === "number") {
-                res.expires_in_ms = res.expires_in * 1000;
-            }
-        }
-
-        return res;
     }
 
     /**
@@ -9857,6 +9827,19 @@ export function fixNotificationCountOnDecryption(cli: MatrixClient, event: Matri
     const room = cli.getRoom(event.getRoomId());
     if (!room || !ourUserId || !eventId) return;
 
+    // Due to threads, we can get relation events (eg. edits & reactions) that never get
+    // added to a timeline and so cannot be found in their own room (their edit / reaction
+    // still applies to the event it needs to, so it doesn't matter too much). However, if
+    // we try to process notification about this event, we'll get very confused because we
+    // won't be able to find the event in the room, so will assume it must be unread, even
+    // if it's actually read. We therefore skip anything that isn't in the room. This isn't
+    // *great*, so if we can fix the homeless events (eg. with MSC4023) then we should probably
+    // remove this workaround.
+    if (!room.findEventById(eventId)) {
+        logger.info(`Decrypted event ${event.getId()} is not in room ${room.roomId}: ignoring`);
+        return;
+    }
+
     const isThreadEvent = !!event.threadRootId && !event.isThreadRoot;
 
     let hasReadEvent;
@@ -9941,7 +9924,7 @@ export function threadIdForReceipt(event: MatrixEvent): string {
  * @returns true if this event is considered to be in the main timeline as far
  *               as receipts are concerned.
  */
-function inMainTimelineForReceipt(event: MatrixEvent): boolean {
+export function inMainTimelineForReceipt(event: MatrixEvent): boolean {
     if (!event.threadRootId) {
         // Not in a thread: then it is in the main timeline
         return true;

src/common-crypto/CryptoBackend.ts

@@ -17,7 +17,7 @@ limitations under the License.
 import type { IDeviceLists, IToDeviceEvent } from "../sync-accumulator";
 import { IClearEvent, MatrixEvent } from "../models/event";
 import { Room } from "../models/room";
-import { CryptoApi } from "../crypto-api";
+import { CryptoApi, ImportRoomKeysOpts } from "../crypto-api";
 import { CrossSigningInfo, UserTrustLevel } from "../crypto/CrossSigning";
 import { IEncryptedEventInfo } from "../crypto/api";
 import { KeyBackupInfo, KeyBackupSession } from "../crypto-api/keybackup";
@@ -35,8 +35,8 @@ export interface CryptoBackend extends SyncCryptoCallbacks, CryptoApi {
      * symmetry with setGlobalBlacklistUnverifiedDevices but there is currently
      * no room-level equivalent for this setting.
      *
-     * @remarks this is here, rather than in `CryptoApi`, because I don't think we're
-     * going to support it in the rust crypto implementation.
+     * @remarks This has no effect in Rust Crypto; it exists only for the sake of
+     * the accessors in MatrixClient.
      */
     globalErrorOnUnknownDevices: boolean;
 
@@ -108,6 +108,15 @@ export interface CryptoBackend extends SyncCryptoCallbacks, CryptoApi {
      * @param privKey - The private decryption key.
      */
     getBackupDecryptor(backupInfo: KeyBackupInfo, privKey: ArrayLike<number>): Promise<BackupDecryptor>;
+
+    /**
+     * Import a list of room keys restored from backup
+     *
+     * @param keys - a list of session export objects
+     * @param opts - options object
+     * @returns a promise which resolves once the keys have been imported
+     */
+    importBackedUpRoomKeys(keys: IMegolmSessionData[], opts?: ImportRoomKeysOpts): Promise<void>;
 }
 
 /** The methods which crypto implementations should expose to the Sync api

src/common-crypto/base64.ts

@@ -1,46 +0,0 @@
-/*
-Copyright 2023 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/**
- * Base64 encoding and decoding utility for crypo.
- */
-
-/**
- * Encode a typed array of uint8 as base64.
- * @param uint8Array - The data to encode.
- * @returns The base64.
- */
-export function encodeBase64(uint8Array: ArrayBuffer | Uint8Array): string {
-    return Buffer.from(uint8Array).toString("base64");
-}
-
-/**
- * Encode a typed array of uint8 as unpadded base64.
- * @param uint8Array - The data to encode.
- * @returns The unpadded base64.
- */
-export function encodeUnpaddedBase64(uint8Array: ArrayBuffer | Uint8Array): string {
-    return encodeBase64(uint8Array).replace(/={1,2}$/, "");
-}
-
-/**
- * Decode a base64 string to a typed array of uint8.
- * @param base64 - The base64 to decode.
- * @returns The decoded data.
- */
-export function decodeBase64(base64: string): Uint8Array {
-    return Buffer.from(base64, "base64");
-}

src/crypto-api.ts

@@ -39,6 +39,20 @@ export interface CryptoApi {
      */
     globalBlacklistUnverifiedDevices: boolean;
 
+    /**
+     * Return the current version of the crypto module.
+     * For example: `Rust SDK ${versions.matrix_sdk_crypto} (${versions.git_sha}), Vodozemac ${versions.vodozemac}`.
+     * @returns the formatted version
+     */
+    getVersion(): string;
+
+    /**
+     * Get the public part of the device keys for the current device.
+     *
+     * @returns The public device keys.
+     */
+    getOwnDeviceKeys(): Promise<OwnDeviceKeys>;
+
     /**
      * Perform any background tasks that can be done before a message is ready to
      * send, in order to speed up sending of the message.
@@ -155,7 +169,7 @@ export interface CryptoApi {
     /**
      * Mark the given device as locally verified.
      *
-     * Marking a devices as locally verified has much the same effect as completing the verification dance, or receiving
+     * Marking a device as locally verified has much the same effect as completing the verification dance, or receiving
      * a cross-signing signature for it.
      *
      * @param userId - owner of the device
@@ -168,6 +182,21 @@ export interface CryptoApi {
      */
     setDeviceVerified(userId: string, deviceId: string, verified?: boolean): Promise<void>;
 
+    /**
+     * Cross-sign one of our own devices.
+     *
+     * This will create a signature for the device using our self-signing key, and publish that signature.
+     * Cross-signing a device indicates, to our other devices and to other users, that we have verified that it really
+     * belongs to us.
+     *
+     * Requires that cross-signing has been set up on this device (normally by calling {@link bootstrapCrossSigning}.
+     *
+     * *Note*: Do not call this unless you have verified, somehow, that the device is genuine!
+     *
+     * @param deviceId - ID of the device to be signed.
+     */
+    crossSignDevice(deviceId: string): Promise<void>;
+
     /**
      * Checks whether cross signing:
      * - is enabled on this account and trusted by this device
@@ -568,9 +597,10 @@ export interface ImportRoomKeyProgressData {
 export interface ImportRoomKeysOpts {
     /** Reports ongoing progress of the import process. Can be used for feedback. */
     progressCallback?: (stage: ImportRoomKeyProgressData) => void;
-    // TODO, the rust SDK will always such imported keys as untrusted
+    /** @deprecated the rust SDK will always such imported keys as untrusted */
     untrusted?: boolean;
-    source?: String; // TODO: Enum (backup, file, ??)
+    /** @deprecated not useful externally */
+    source?: string;
 }
 
 /**
@@ -742,5 +772,13 @@ export enum EventShieldReason {
     MISMATCHED_SENDER_KEY,
 }
 
+/** The result of a call to {@link CryptoApi.getOwnDeviceKeys} */
+export interface OwnDeviceKeys {
+    /** Public part of the Ed25519 fingerprint key for the current device, base64 encoded. */
+    ed25519: string;
+    /** Public part of the Curve25519 identity key for the current device, base64 encoded. */
+    curve25519: string;
+}
+
 export * from "./crypto-api/verification";
 export * from "./crypto-api/keybackup";

src/crypto/CrossSigning.ts

@@ -19,7 +19,7 @@ limitations under the License.
  */
 
 import type { PkSigning } from "@matrix-org/olm";
-import { decodeBase64, encodeBase64, IObject, pkSign, pkVerify } from "./olmlib";
+import { IObject, pkSign, pkVerify } from "./olmlib";
 import { logger } from "../logger";
 import { IndexedDBCryptoStore } from "../crypto/store/indexeddb-crypto-store";
 import { decryptAES, encryptAES } from "./aes";
@@ -31,6 +31,7 @@ import { ISignatures } from "../@types/signed";
 import { CryptoStore, SecretStorePrivateKeys } from "./store/base";
 import { ServerSideSecretStorage, SecretStorageKeyDescription } from "../secret-storage";
 import { DeviceVerificationStatus, UserVerificationStatus as UserTrustLevel } from "../crypto-api";
+import { decodeBase64, encodeBase64 } from "../base64";
 
 // backwards-compatibility re-exports
 export { UserTrustLevel };

src/crypto/RoomList.ts

@@ -29,6 +29,13 @@ export interface IRoomEncryption {
 }
 /* eslint-enable camelcase */
 
+/**
+ * Information about the encryption settings of rooms. Loads this information
+ * from the supplied crypto store when `init()` is called, and saves it to the
+ * crypto store whenever it is updated via `setRoomEncryption()`. Can supply
+ * full information about a room's encryption via `getRoomEncryption()`, or just
+ * answer whether or not a room has encryption via `isRoomEncrypted`.
+ */
 export class RoomList {
     // Object of roomId -> room e2e info object (body of the m.room.encryption event)
     private roomEncryption: Record<string, IRoomEncryption> = {};
@@ -43,7 +50,7 @@ export class RoomList {
         });
     }
 
-    public getRoomEncryption(roomId: string): IRoomEncryption {
+    public getRoomEncryption(roomId: string): IRoomEncryption | null {
         return this.roomEncryption[roomId] || null;
     }
 

src/crypto/aes.ts

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { decodeBase64, encodeBase64 } from "./olmlib";
+import { decodeBase64, encodeBase64 } from "../base64";
 import { subtleCrypto, crypto, TextEncoder } from "./crypto";
 
 // salt for HKDF, with 8 bytes of zeros

src/crypto/dehydration.ts

@@ -17,7 +17,7 @@ limitations under the License.
 import anotherjson from "another-json";
 
 import type { IDeviceKeys, IOneTimeKey } from "../@types/crypto";
-import { decodeBase64, encodeBase64 } from "./olmlib";
+import { decodeBase64, encodeBase64 } from "../base64";
 import { IndexedDBCryptoStore } from "../crypto/store/indexeddb-crypto-store";
 import { decryptAES, encryptAES } from "./aes";
 import { logger } from "../logger";

src/crypto/index.ts

@@ -64,7 +64,7 @@ import {
     IUploadKeySignaturesResponse,
     MatrixClient,
 } from "../client";
-import type { IRoomEncryption, RoomList } from "./RoomList";
+import { IRoomEncryption, RoomList } from "./RoomList";
 import { IKeyBackupInfo } from "./keybackup";
 import { ISyncStateData } from "../sync";
 import { CryptoStore } from "./store/base";
@@ -98,10 +98,12 @@ import {
     KeyBackupCheck,
     KeyBackupInfo,
     VerificationRequest as CryptoApiVerificationRequest,
+    OwnDeviceKeys,
 } from "../crypto-api";
 import { Device, DeviceMap } from "../models/device";
 import { deviceInfoToDevice } from "./device-converter";
 import { ClientPrefix, MatrixError, Method } from "../http-api";
+import { decodeBase64, encodeBase64 } from "../base64";
 
 /* re-exports for backwards compatibility */
 export type {
@@ -230,6 +232,18 @@ export enum CryptoEvent {
     KeyBackupStatus = "crypto.keyBackupStatus",
     KeyBackupFailed = "crypto.keyBackupFailed",
     KeyBackupSessionsRemaining = "crypto.keyBackupSessionsRemaining",
+
+    /**
+     * Fires when a new valid backup decryption key is in cache.
+     * This will happen when a secret is received from another session, from secret storage,
+     * or when a new backup is created from this session.
+     *
+     * The payload is the version of the backup for which we have the key for.
+     *
+     * This event is only fired by the rust crypto backend.
+     */
+    KeyBackupDecryptionKeyCached = "crypto.keyBackupDecryptionKeyCached",
+
     KeySignatureUploadFailure = "crypto.keySignatureUploadFailure",
     /** @deprecated Use `VerificationRequestReceived`. */
     VerificationRequest = "crypto.verification.request",
@@ -295,6 +309,13 @@ export type CryptoEventHandlerMap = {
     [CryptoEvent.KeyBackupStatus]: (enabled: boolean) => void;
     [CryptoEvent.KeyBackupFailed]: (errcode: string) => void;
     [CryptoEvent.KeyBackupSessionsRemaining]: (remaining: number) => void;
+
+    /**
+     * Fires when the backup decryption key is received and cached.
+     *
+     * @param version - The version of the backup for which we have the key for.
+     */
+    [CryptoEvent.KeyBackupDecryptionKeyCached]: (version: string) => void;
     [CryptoEvent.KeySignatureUploadFailure]: (
         failures: IUploadKeySignaturesResponse["failures"],
         source: "checkOwnCrossSigningTrust" | "afterCrossSigningLocalKeyChange" | "setDeviceVerification",
@@ -364,6 +385,7 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
     public readonly dehydrationManager: DehydrationManager;
     public readonly secretStorage: LegacySecretStorage;
 
+    private readonly roomList: RoomList;
     private readonly reEmitter: TypedReEmitter<CryptoEvent, CryptoEventHandlerMap>;
     private readonly verificationMethods: Map<VerificationMethod, typeof VerificationBase>;
     public readonly supportedAlgorithms: string[];
@@ -452,10 +474,13 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
         private readonly deviceId: string,
         private readonly clientStore: IStore,
         public readonly cryptoStore: CryptoStore,
-        private readonly roomList: RoomList,
         verificationMethods: Array<VerificationMethod | (typeof VerificationBase & { NAME: string })>,
     ) {
         super();
+
+        logger.debug("Crypto: initialising roomlist...");
+        this.roomList = new RoomList(cryptoStore);
+
         this.reEmitter = new TypedReEmitter(this);
 
         if (verificationMethods) {
@@ -500,7 +525,7 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
                     await this.secretStorage.store("m.megolm_backup.v1", fixedKey, [keys![0]]);
                 }
 
-                return olmlib.decodeBase64(fixedKey || storedKey);
+                return decodeBase64(fixedKey || storedKey);
             }
 
             // try to get key from app
@@ -605,10 +630,21 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
         // (this is important for key backups & things)
         this.deviceList.startTrackingDeviceList(this.userId);
 
+        logger.debug("Crypto: initialising roomlist...");
+        await this.roomList.init();
+
         logger.log("Crypto: checking for key backup...");
         this.backupManager.checkAndStart();
     }
 
+    /**
+     * Implementation of {@link CryptoApi#getVersion}.
+     */
+    public getVersion(): string {
+        const olmVersionTuple = Crypto.getOlmVersion();
+        return `Olm ${olmVersionTuple[0]}.${olmVersionTuple[1]}.${olmVersionTuple[2]}`;
+    }
+
     /**
      * Whether to trust a others users signatures of their devices.
      * If false, devices will only be considered 'verified' if we have
@@ -1050,7 +1086,7 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
             newKeyId = await createSSSS(opts, backupKey);
 
             // store the backup key in secret storage
-            await secretStorage.store("m.megolm_backup.v1", olmlib.encodeBase64(backupKey!), [newKeyId]);
+            await secretStorage.store("m.megolm_backup.v1", encodeBase64(backupKey!), [newKeyId]);
 
             // The backup is trusted because the user provided the private key.
             // Sign the backup with the cross-signing key so the key backup can
@@ -1094,7 +1130,7 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
             );
             // write the key to 4S
             const privateKey = decodeRecoveryKey(info.recovery_key);
-            await secretStorage.store("m.megolm_backup.v1", olmlib.encodeBase64(privateKey));
+            await secretStorage.store("m.megolm_backup.v1", encodeBase64(privateKey));
 
             // create keyBackupInfo object to add to builder
             const data: IKeyBackupInfo = {
@@ -1122,7 +1158,7 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
                 const keyId = newKeyId || oldKeyId;
                 await secretStorage.store("m.megolm_backup.v1", fixedBackupKey, keyId ? [keyId] : null);
             }
-            const decodedBackupKey = new Uint8Array(olmlib.decodeBase64(fixedBackupKey || sessionBackupKey));
+            const decodedBackupKey = new Uint8Array(decodeBase64(fixedBackupKey || sessionBackupKey));
             builder.addSessionBackupPrivateKeyToCache(decodedBackupKey);
         } else if (this.backupManager.getKeyBackupEnabled()) {
             // key backup is enabled but we don't have a session backup key in SSSS: see if we have one in
@@ -1137,7 +1173,7 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
                 return;
             }
             logger.info("Got session backup key from cache/user that wasn't in SSSS: saving to SSSS");
-            await secretStorage.store("m.megolm_backup.v1", olmlib.encodeBase64(backupKey));
+            await secretStorage.store("m.megolm_backup.v1", encodeBase64(backupKey));
         }
 
         const operation = builder.buildOperation();
@@ -1178,10 +1214,11 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
 
         // write the key to 4S
         const privateKey = info.privateKey;
-        await this.secretStorage.store("m.megolm_backup.v1", olmlib.encodeBase64(privateKey));
+        await this.secretStorage.store("m.megolm_backup.v1", encodeBase64(privateKey));
         await this.storeSessionBackupPrivateKey(privateKey);
 
         await this.backupManager.checkAndStart();
+        await this.backupManager.scheduleAllGroupSessionsForBackup();
     }
 
     /**
@@ -1301,13 +1338,13 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
 
         // make sure we have a Uint8Array, rather than a string
         if (typeof encodedKey === "string") {
-            key = new Uint8Array(olmlib.decodeBase64(fixBackupKey(encodedKey) || encodedKey));
+            key = new Uint8Array(decodeBase64(fixBackupKey(encodedKey) || encodedKey));
             await this.storeSessionBackupPrivateKey(key);
         }
         if (encodedKey && typeof encodedKey === "object" && "ciphertext" in encodedKey) {
             const pickleKey = Buffer.from(this.olmDevice.pickleKey);
             const decrypted = await decryptAES(encodedKey, pickleKey, "m.megolm_backup.v1");
-            key = olmlib.decodeBase64(decrypted);
+            key = decodeBase64(decrypted);
         }
         return key;
     }
@@ -1323,7 +1360,7 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
             throw new Error(`storeSessionBackupPrivateKey expects Uint8Array, got ${key}`);
         }
         const pickleKey = Buffer.from(this.olmDevice.pickleKey);
-        const encryptedKey = await encryptAES(olmlib.encodeBase64(key), pickleKey, "m.megolm_backup.v1");
+        const encryptedKey = await encryptAES(encodeBase64(key), pickleKey, "m.megolm_backup.v1");
         return this.cryptoStore.doTxn("readwrite", [IndexedDBCryptoStore.STORE_ACCOUNT], (txn) => {
             this.cryptoStore.storeSecretStorePrivateKey(txn, "m.megolm_backup.v1", encryptedKey);
         });
@@ -1867,6 +1904,14 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
         return new LibOlmBackupDecryptor(algorithm);
     }
 
+    /**
+     * Implementation of {@link CryptoBackend#importBackedUpRoomKeys}.
+     */
+    public importBackedUpRoomKeys(keys: IMegolmSessionData[], opts: ImportRoomKeysOpts = {}): Promise<void> {
+        opts.source = "backup";
+        return this.importRoomKeys(keys, opts);
+    }
+
     /**
      * Store a set of keys as our own, trusted, cross-signing keys.
      *
@@ -1959,6 +2004,8 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
      * Get the Ed25519 key for this device
      *
      * @returns base64-encoded ed25519 key.
+     *
+     * @deprecated Use {@link CryptoApi#getOwnDeviceKeys}.
      */
     public getDeviceEd25519Key(): string | null {
         return this.olmDevice.deviceEd25519Key;
@@ -1968,11 +2015,29 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
      * Get the Curve25519 key for this device
      *
      * @returns base64-encoded curve25519 key.
+     *
+     * @deprecated Use {@link CryptoApi#getOwnDeviceKeys}
      */
     public getDeviceCurve25519Key(): string | null {
         return this.olmDevice.deviceCurve25519Key;
     }
 
+    /**
+     * Implementation of {@link CryptoApi#getOwnDeviceKeys}.
+     */
+    public async getOwnDeviceKeys(): Promise<OwnDeviceKeys> {
+        if (!this.olmDevice.deviceCurve25519Key) {
+            throw new Error("Curve25519 key not yet created");
+        }
+        if (!this.olmDevice.deviceEd25519Key) {
+            throw new Error("Ed25519 key not yet created");
+        }
+        return {
+            ed25519: this.olmDevice.deviceEd25519Key,
+            curve25519: this.olmDevice.deviceCurve25519Key,
+        };
+    }
+
     /**
      * Set the global override for whether the client should ever send encrypted
      * messages to unverified devices.  This provides the default for rooms which
@@ -2297,6 +2362,15 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
         await this.setDeviceVerification(userId, deviceId, verified);
     }
 
+    /**
+     * Blindly cross-sign one of our other devices.
+     *
+     * Implementation of {@link CryptoApi#crossSignDevice}.
+     */
+    public async crossSignDevice(deviceId: string): Promise<void> {
+        await this.setDeviceVerified(this.userId, deviceId, true);
+    }
+
     /**
      * Update the blocked/verified state of the given device
      *
@@ -4177,6 +4251,22 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
         obj.signatures = recursiveMapToObject(sigs);
         if (unsigned !== undefined) obj.unsigned = unsigned;
     }
+
+    /**
+     * @returns true if the room with the supplied ID is encrypted. False if the
+     * room is not encrypted, or is unknown to us.
+     */
+    public isRoomEncrypted(roomId: string): boolean {
+        return this.roomList.isRoomEncrypted(roomId);
+    }
+
+    /**
+     * @returns information about the encryption on the room with the supplied
+     * ID, or null if the room is not encrypted or unknown to us.
+     */
+    public getRoomEncryption(roomId: string): IRoomEncryption | null {
+        return this.roomList.getRoomEncryption(roomId);
+    }
 }
 
 /**
@@ -4196,7 +4286,7 @@ export function fixBackupKey(key?: string): string | null {
         return null;
     }
     const fixedKey = Uint8Array.from(key.split(","), (x) => parseInt(x));
-    return olmlib.encodeBase64(fixedKey);
+    return encodeBase64(fixedKey);
 }
 
 /**

src/crypto/olmlib.ts

@@ -537,30 +537,3 @@ export function isOlmEncrypted(event: MatrixEvent): boolean {
     }
     return true;
 }
-
-/**
- * Encode a typed array of uint8 as base64.
- * @param uint8Array - The data to encode.
- * @returns The base64.
- */
-export function encodeBase64(uint8Array: ArrayBuffer | Uint8Array): string {
-    return Buffer.from(uint8Array).toString("base64");
-}
-
-/**
- * Encode a typed array of uint8 as unpadded base64.
- * @param uint8Array - The data to encode.
- * @returns The unpadded base64.
- */
-export function encodeUnpaddedBase64(uint8Array: ArrayBuffer | Uint8Array): string {
-    return encodeBase64(uint8Array).replace(/=+$/g, "");
-}
-
-/**
- * Decode a base64 string to a typed array of uint8.
- * @param base64 - The base64 to decode.
- * @returns The decoded data.
- */
-export function decodeBase64(base64: string): Uint8Array {
-    return Buffer.from(base64, "base64");
-}

src/crypto/verification/QRCode.ts

@@ -21,7 +21,7 @@ limitations under the License.
 import { crypto } from "../crypto";
 import { VerificationBase as Base } from "./Base";
 import { newKeyMismatchError, newUserCancelledError } from "./Error";
-import { decodeBase64, encodeUnpaddedBase64 } from "../olmlib";
+import { decodeBase64, encodeUnpaddedBase64 } from "../../base64";
 import { logger } from "../../logger";
 import { VerificationRequest } from "./request/VerificationRequest";
 import { MatrixClient } from "../../client";

src/http-api/index.ts

@@ -50,7 +50,7 @@ export class MatrixHttpApi<O extends IHttpOpts> extends FetchHttpApi<O> {
         const abortController = opts.abortController ?? new AbortController();
 
         // If the file doesn't have a mime type, use a default since the HS errors if we don't supply one.
-        const contentType = opts.type ?? (file as File).type ?? "application/octet-stream";
+        const contentType = (opts.type ?? (file as File).type) || "application/octet-stream";
         const fileName = opts.name ?? (file as File).name;
 
         const upload = {

src/logger.ts

@@ -18,7 +18,18 @@ limitations under the License.
 import loglevel from "loglevel";
 
 /** Logger interface used within the js-sdk codebase */
-export interface Logger {
+export interface Logger extends BaseLogger {
+    /**
+     * Create a child logger.
+     *
+     * @param namespace - name to add to the current logger to generate the child. Some implementations of `Logger`
+     *    use this as a prefix; others use a different mechanism.
+     */
+    getChild(namespace: string): Logger;
+}
+
+/** The basic interface for a logger which doesn't support children */
+interface BaseLogger {
     /**
      * Output trace message to the logger, with stack trace.
      *
@@ -53,14 +64,6 @@ export interface Logger {
      * @param msg - Data to log.
      */
     error(...msg: any[]): void;
-
-    /**
-     * Create a child logger.
-     *
-     * @param namespace - name to add to the current logger to generate the child. Some implementations of `Logger`
-     *    use this as a prefix; others use a different mechanism.
-     */
-    getChild(namespace: string): Logger;
 }
 
 // This is to demonstrate, that you can use any namespace you want.
@@ -139,3 +142,41 @@ function getPrefixedLogger(prefix: string): PrefixedLogger {
 export const logger = loglevel.getLogger(DEFAULT_NAMESPACE) as PrefixedLogger;
 logger.setLevel(loglevel.levels.DEBUG, false);
 extendLogger(logger);
+
+/**
+ * A "span" for grouping related log lines together.
+ *
+ * The current implementation just adds the name at the start of each log line.
+ *
+ * This offers a lighter-weight alternative to 'child' loggers returned by {@link Logger#getChild}. In particular,
+ * it's not possible to apply individual filters to the LogSpan such as setting the verbosity level. On the other hand,
+ * no reference to the LogSpan is retained in the logging framework, so it is safe to make lots of them over the course
+ * of an application's life and just drop references to them when the job is done.
+ */
+export class LogSpan implements BaseLogger {
+    private readonly name;
+
+    public constructor(private readonly parent: BaseLogger, name: string) {
+        this.name = name + ":";
+    }
+
+    public trace(...msg: any[]): void {
+        this.parent.trace(this.name, ...msg);
+    }
+
+    public debug(...msg: any[]): void {
+        this.parent.debug(this.name, ...msg);
+    }
+
+    public info(...msg: any[]): void {
+        this.parent.info(this.name, ...msg);
+    }
+
+    public warn(...msg: any[]): void {
+        this.parent.warn(this.name, ...msg);
+    }
+
+    public error(...msg: any[]): void {
+        this.parent.error(this.name, ...msg);
+    }
+}

src/matrix.ts

@@ -29,6 +29,7 @@ export * from "./http-api";
 export * from "./autodiscovery";
 export * from "./sync-accumulator";
 export * from "./errors";
+export * from "./base64";
 export * from "./models/beacon";
 export * from "./models/event";
 export * from "./models/room";

src/matrixrtc/CallMembership.ts

@@ -91,7 +91,7 @@ export class CallMembership {
     }
 
     public isExpired(): boolean {
-        return this.getAbsoluteExpiry() < this.parentEvent.getTs() + this.parentEvent.getLocalAge();
+        return this.getMsUntilExpiry() <= 0;
     }
 
     public getActiveFoci(): Focus[] {

src/matrixrtc/MatrixRTCSession.ts

@@ -22,12 +22,32 @@ import { MatrixClient } from "../client";
 import { EventType } from "../@types/event";
 import { CallMembership, CallMembershipData } from "./CallMembership";
 import { Focus } from "./focus";
-import { MatrixEvent } from "../matrix";
-import { randomString } from "../randomstring";
+import { MatrixError, MatrixEvent } from "../matrix";
+import { randomString, secureRandomBase64Url } from "../randomstring";
+import { EncryptionKeysEventContent } from "./types";
+import { decodeBase64, encodeUnpaddedBase64 } from "../base64";
 
 const MEMBERSHIP_EXPIRY_TIME = 60 * 60 * 1000;
 const MEMBER_EVENT_CHECK_PERIOD = 2 * 60 * 1000; // How often we check to see if we need to re-send our member event
 const CALL_MEMBER_EVENT_RETRY_DELAY_MIN = 3000;
+const UPDATE_ENCRYPTION_KEY_THROTTLE = 3000;
+
+// A delay after a member leaves before we create and publish a new key, because people
+// tend to leave calls at the same time
+const MAKE_KEY_DELAY = 3000;
+// The delay between creating and sending a new key and starting to encrypt with it. This gives others
+// a chance to receive the new key to minimise the chance they don't get media they can't decrypt.
+// The total time between a member leaving and the call switching to new keys is therefore
+// MAKE_KEY_DELAY + SEND_KEY_DELAY
+const USE_KEY_DELAY = 5000;
+
+const getParticipantId = (userId: string, deviceId: string): string => `${userId}:${deviceId}`;
+const getParticipantIdFromMembership = (m: CallMembership): string => getParticipantId(m.sender!, m.deviceId);
+
+function keysEqual(a: Uint8Array, b: Uint8Array): boolean {
+    if (a === b) return true;
+    return a && b && a.length === b.length && a.every((x, i) => x === b[i]);
+}
 
 export enum MatrixRTCSessionEvent {
     // A member joined, left, or updated a property of their membership.
@@ -36,6 +56,8 @@ export enum MatrixRTCSessionEvent {
     // separate from MembershipsChanged, ie. independent of whether our member event
     // has succesfully gone through.
     JoinStateChanged = "join_state_changed",
+    // The key used to encrypt media has changed
+    EncryptionKeyChanged = "encryption_key_changed",
 }
 
 export type MatrixRTCSessionEventHandlerMap = {
@@ -44,6 +66,11 @@ export type MatrixRTCSessionEventHandlerMap = {
         newMemberships: CallMembership[],
     ) => void;
     [MatrixRTCSessionEvent.JoinStateChanged]: (isJoined: boolean) => void;
+    [MatrixRTCSessionEvent.EncryptionKeyChanged]: (
+        key: Uint8Array,
+        encryptionKeyIndex: number,
+        participantId: string,
+    ) => void;
 };
 
 /**
@@ -51,6 +78,9 @@ export type MatrixRTCSessionEventHandlerMap = {
  * This class doesn't deal with media at all, just membership & properties of a session.
  */
 export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, MatrixRTCSessionEventHandlerMap> {
+    // The session Id of the call, this is the call_id of the call Member event.
+    private _callId: string | undefined;
+
     // How many ms after we joined the call, that our membership should expire, or undefined
     // if we're not yet joined
     private relativeExpiry: number | undefined;
@@ -65,12 +95,29 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
 
     private memberEventTimeout?: ReturnType<typeof setTimeout>;
     private expiryTimeout?: ReturnType<typeof setTimeout>;
+    private keysEventUpdateTimeout?: ReturnType<typeof setTimeout>;
+    private makeNewKeyTimeout?: ReturnType<typeof setTimeout>;
+    private setNewKeyTimeouts = new Set<ReturnType<typeof setTimeout>>();
 
     private activeFoci: Focus[] | undefined;
 
     private updateCallMembershipRunning = false;
     private needCallMembershipUpdate = false;
 
+    private manageMediaKeys = false;
+    // userId:deviceId => array of keys
+    private encryptionKeys = new Map<string, Array<Uint8Array>>();
+    private lastEncryptionKeyUpdateRequest?: number;
+
+    /**
+     * The callId (sessionId) of the call.
+     *
+     * It can be undefined since the callId is only known once the first membership joins.
+     * The callId is the property that, per definition, groups memberships into one call.
+     */
+    public get callId(): string | undefined {
+        return this._callId;
+    }
     /**
      * Returns all the call memberships for a room, oldest first
      */
@@ -118,10 +165,12 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
         }
 
         callMemberships.sort((a, b) => a.createdTs() - b.createdTs());
-        logger.debug(
-            "Call memberships, in order: ",
-            callMemberships.map((m) => [m.createdTs(), m.sender]),
-        );
+        if (callMemberships.length > 1) {
+            logger.debug(
+                `Call memberships in room ${room.roomId}, in order: `,
+                callMemberships.map((m) => [m.createdTs(), m.sender]),
+            );
+        }
 
         return callMemberships;
     }
@@ -141,6 +190,7 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
         public memberships: CallMembership[],
     ) {
         super();
+        this._callId = memberships[0]?.callId;
         this.setExpiryTimer();
     }
 
@@ -173,18 +223,28 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
      * This will not subscribe to updates: remember to call subscribe() separately if
      * desired.
      * This method will return immediately and the session will be joined in the background.
+     *
+     * @param activeFoci - The list of foci to set as currently active in the call member event
+     * @param manageMediaKeys - If true, generate and share a a media key for this participant,
+     *                          and emit MatrixRTCSessionEvent.EncryptionKeyChanged when
+     *                          media keys for other participants become available.
      */
-    public joinRoomSession(activeFoci: Focus[]): void {
+    public joinRoomSession(activeFoci: Focus[], manageMediaKeys?: boolean): void {
         if (this.isJoined()) {
             logger.info(`Already joined to session in room ${this.room.roomId}: ignoring join call`);
             return;
         }
 
-        logger.info(`Joining call session in room ${this.room.roomId}`);
+        logger.info(`Joining call session in room ${this.room.roomId} with manageMediaKeys=${manageMediaKeys}`);
         this.activeFoci = activeFoci;
         this.relativeExpiry = MEMBERSHIP_EXPIRY_TIME;
+        this.manageMediaKeys = manageMediaKeys ?? false;
         this.membershipId = randomString(5);
         this.emit(MatrixRTCSessionEvent.JoinStateChanged, true);
+        if (manageMediaKeys) {
+            this.makeNewSenderKey();
+            this.requestKeyEventSend();
+        }
         // We don't wait for this, mostly because it may fail and schedule a retry, so this
         // function returning doesn't really mean anything at all.
         this.triggerCallMembershipEventUpdate();
@@ -205,9 +265,30 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
             return new Promise((resolve) => resolve(false));
         }
 
+        const userId = this.client.getUserId();
+        const deviceId = this.client.getDeviceId();
+
+        if (!userId) throw new Error("No userId");
+        if (!deviceId) throw new Error("No deviceId");
+
+        // clear our encryption keys as we're done with them now (we'll
+        // make new keys if we rejoin). We leave keys for other participants
+        // as they may still be using the same ones.
+        this.encryptionKeys.set(getParticipantId(userId, deviceId), []);
+
+        if (this.makeNewKeyTimeout !== undefined) {
+            clearTimeout(this.makeNewKeyTimeout);
+            this.makeNewKeyTimeout = undefined;
+        }
+        for (const t of this.setNewKeyTimeouts) {
+            clearTimeout(t);
+        }
+        this.setNewKeyTimeouts.clear();
+
         logger.info(`Leaving call session in room ${this.room.roomId}`);
         this.relativeExpiry = undefined;
         this.activeFoci = undefined;
+        this.manageMediaKeys = false;
         this.membershipId = undefined;
         this.emit(MatrixRTCSessionEvent.JoinStateChanged, false);
 
@@ -226,6 +307,167 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
         });
     }
 
+    public getKeysForParticipant(userId: string, deviceId: string): Array<Uint8Array> | undefined {
+        return this.encryptionKeys.get(getParticipantId(userId, deviceId));
+    }
+
+    /**
+     * A map of keys used to encrypt and decrypt (we are using a symmetric
+     * cipher) given participant's media. This also includes our own key
+     */
+    public getEncryptionKeys(): IterableIterator<[string, Array<Uint8Array>]> {
+        return this.encryptionKeys.entries();
+    }
+
+    private getNewEncryptionKeyIndex(): number {
+        const userId = this.client.getUserId();
+        const deviceId = this.client.getDeviceId();
+
+        if (!userId) throw new Error("No userId!");
+        if (!deviceId) throw new Error("No deviceId!");
+
+        return (this.getKeysForParticipant(userId, deviceId)?.length ?? 0) % 16;
+    }
+
+    /**
+     * Sets an encryption key at a specified index for a participant.
+     * The encryption keys for the local participanmt are also stored here under the
+     * user and device ID of the local participant.
+     * @param userId - The user ID of the participant
+     * @param deviceId - Device ID of the participant
+     * @param encryptionKeyIndex - The index of the key to set
+     * @param encryptionKeyString - The string represenation of the key to set in base64
+     * @param delayBeforeuse - If true, delay before emitting a key changed event. Useful when setting
+     *                         encryption keys for the local participant to allow time for the key to
+     *                         be distributed.
+     */
+    private setEncryptionKey(
+        userId: string,
+        deviceId: string,
+        encryptionKeyIndex: number,
+        encryptionKeyString: string,
+        delayBeforeuse = false,
+    ): void {
+        const keyBin = decodeBase64(encryptionKeyString);
+
+        const participantId = getParticipantId(userId, deviceId);
+        const encryptionKeys = this.encryptionKeys.get(participantId) ?? [];
+
+        if (keysEqual(encryptionKeys[encryptionKeyIndex], keyBin)) return;
+
+        encryptionKeys[encryptionKeyIndex] = keyBin;
+        this.encryptionKeys.set(participantId, encryptionKeys);
+        if (delayBeforeuse) {
+            const useKeyTimeout = setTimeout(() => {
+                this.setNewKeyTimeouts.delete(useKeyTimeout);
+                logger.info(`Delayed-emitting key changed event for ${participantId} idx ${encryptionKeyIndex}`);
+                this.emit(MatrixRTCSessionEvent.EncryptionKeyChanged, keyBin, encryptionKeyIndex, participantId);
+            }, USE_KEY_DELAY);
+            this.setNewKeyTimeouts.add(useKeyTimeout);
+        } else {
+            this.emit(MatrixRTCSessionEvent.EncryptionKeyChanged, keyBin, encryptionKeyIndex, participantId);
+        }
+    }
+
+    /**
+     * Generate a new sender key and add it at the next available index
+     * @param delayBeforeUse - If true, wait for a short period before settign the key for the
+     *                         media encryptor to use. If false, set the key immediately.
+     */
+    private makeNewSenderKey(delayBeforeUse = false): void {
+        const userId = this.client.getUserId();
+        const deviceId = this.client.getDeviceId();
+
+        if (!userId) throw new Error("No userId");
+        if (!deviceId) throw new Error("No deviceId");
+
+        const encryptionKey = secureRandomBase64Url(16);
+        const encryptionKeyIndex = this.getNewEncryptionKeyIndex();
+        logger.info("Generated new key at index " + encryptionKeyIndex);
+        this.setEncryptionKey(userId, deviceId, encryptionKeyIndex, encryptionKey, delayBeforeUse);
+    }
+
+    /**
+     * Requests that we resend our keys to the room. May send a keys event immediately
+     * or queue for alter if one has already been sent recently.
+     */
+    private requestKeyEventSend(): void {
+        if (!this.manageMediaKeys) return;
+
+        if (
+            this.lastEncryptionKeyUpdateRequest &&
+            this.lastEncryptionKeyUpdateRequest + UPDATE_ENCRYPTION_KEY_THROTTLE > Date.now()
+        ) {
+            logger.info("Last encryption key event sent too recently: postponing");
+            if (this.keysEventUpdateTimeout === undefined) {
+                this.keysEventUpdateTimeout = setTimeout(this.sendEncryptionKeysEvent, UPDATE_ENCRYPTION_KEY_THROTTLE);
+            }
+            return;
+        }
+
+        this.sendEncryptionKeysEvent();
+    }
+
+    /**
+     * Re-sends the encryption keys room event
+     */
+    private sendEncryptionKeysEvent = async (): Promise<void> => {
+        if (this.keysEventUpdateTimeout !== undefined) {
+            clearTimeout(this.keysEventUpdateTimeout);
+            this.keysEventUpdateTimeout = undefined;
+        }
+        this.lastEncryptionKeyUpdateRequest = Date.now();
+
+        logger.info("Sending encryption keys event");
+
+        if (!this.isJoined()) return;
+
+        const userId = this.client.getUserId();
+        const deviceId = this.client.getDeviceId();
+
+        if (!userId) throw new Error("No userId");
+        if (!deviceId) throw new Error("No deviceId");
+
+        const myKeys = this.getKeysForParticipant(userId, deviceId);
+
+        if (!myKeys) {
+            logger.warn("Tried to send encryption keys event but no keys found!");
+            return;
+        }
+
+        try {
+            await this.client.sendEvent(this.room.roomId, EventType.CallEncryptionKeysPrefix, {
+                keys: myKeys.map((key, index) => {
+                    return {
+                        index,
+                        key: encodeUnpaddedBase64(key),
+                    };
+                }),
+                device_id: deviceId,
+                call_id: "",
+            } as EncryptionKeysEventContent);
+
+            logger.debug(
+                `Embedded-E2EE-LOG updateEncryptionKeyEvent participantId=${userId}:${deviceId} numSent=${myKeys.length}`,
+                this.encryptionKeys,
+            );
+        } catch (error) {
+            const matrixError = error as MatrixError;
+            if (matrixError.event) {
+                // cancel the pending event: we'll just generate a new one with our latest
+                // keys when we resend
+                this.client.cancelPendingEvent(matrixError.event);
+            }
+            if (this.keysEventUpdateTimeout === undefined) {
+                const resendDelay = matrixError.data?.retry_after_ms ?? 5000;
+                logger.warn(`Failed to send m.call.encryption_key, retrying in ${resendDelay}`, error);
+                this.keysEventUpdateTimeout = setTimeout(this.sendEncryptionKeysEvent, resendDelay);
+            } else {
+                logger.info("Not scheduling key resend as another re-send is already pending");
+            }
+        }
+    };
+
     /**
      * Sets a timer for the soonest membership expiry
      */
@@ -252,10 +494,78 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
         return this.memberships[0];
     }
 
+    public onCallEncryption = (event: MatrixEvent): void => {
+        const userId = event.getSender();
+        const content = event.getContent<EncryptionKeysEventContent>();
+
+        const deviceId = content["device_id"];
+        const callId = content["call_id"];
+
+        if (!userId) {
+            logger.warn(`Received m.call.encryption_keys with no userId: callId=${callId}`);
+            return;
+        }
+
+        // We currently only handle callId = ""
+        if (callId !== "") {
+            logger.warn(
+                `Received m.call.encryption_keys with unsupported callId: userId=${userId}, deviceId=${deviceId}, callId=${callId}`,
+            );
+            return;
+        }
+
+        if (!Array.isArray(content.keys)) {
+            logger.warn(`Received m.call.encryption_keys where keys wasn't an array: callId=${callId}`);
+            return;
+        }
+
+        if (userId === this.client.getUserId() && deviceId === this.client.getDeviceId()) {
+            // We store our own sender key in the same set along with keys from others, so it's
+            // important we don't allow our own keys to be set by one of these events (apart from
+            // the fact that we don't need it anyway because we already know our own keys).
+            logger.info("Ignoring our own keys event");
+            return;
+        }
+
+        for (const key of content.keys) {
+            if (!key) {
+                logger.info("Ignoring false-y key in keys event");
+                continue;
+            }
+
+            const encryptionKey = key.key;
+            const encryptionKeyIndex = key.index;
+
+            if (
+                !encryptionKey ||
+                encryptionKeyIndex === undefined ||
+                encryptionKeyIndex === null ||
+                callId === undefined ||
+                callId === null ||
+                typeof deviceId !== "string" ||
+                typeof callId !== "string" ||
+                typeof encryptionKey !== "string" ||
+                typeof encryptionKeyIndex !== "number"
+            ) {
+                logger.warn(
+                    `Malformed call encryption_key: userId=${userId}, deviceId=${deviceId}, encryptionKeyIndex=${encryptionKeyIndex} callId=${callId}`,
+                );
+            } else {
+                logger.debug(
+                    `Embedded-E2EE-LOG onCallEncryption userId=${userId}:${deviceId} encryptionKeyIndex=${encryptionKeyIndex}`,
+                    this.encryptionKeys,
+                );
+                this.setEncryptionKey(userId, deviceId, encryptionKeyIndex, encryptionKey);
+            }
+        }
+    };
+
     public onMembershipUpdate = (): void => {
         const oldMemberships = this.memberships;
         this.memberships = MatrixRTCSession.callMembershipsForRoom(this.room);
 
+        this._callId = this._callId ?? this.memberships[0]?.callId;
+
         const changed =
             oldMemberships.length != this.memberships.length ||
             oldMemberships.some((m, i) => !CallMembership.equal(m, this.memberships[i]));
@@ -265,6 +575,29 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
             this.emit(MatrixRTCSessionEvent.MembershipsChanged, oldMemberships, this.memberships);
         }
 
+        const isMyMembership = (m: CallMembership): boolean =>
+            m.sender === this.client.getUserId() && m.deviceId === this.client.getDeviceId();
+
+        if (this.manageMediaKeys && this.isJoined() && this.makeNewKeyTimeout === undefined) {
+            const oldMebershipIds = new Set(
+                oldMemberships.filter((m) => !isMyMembership(m)).map(getParticipantIdFromMembership),
+            );
+            const newMebershipIds = new Set(
+                this.memberships.filter((m) => !isMyMembership(m)).map(getParticipantIdFromMembership),
+            );
+
+            const anyLeft = Array.from(oldMebershipIds).some((x) => !newMebershipIds.has(x));
+            const anyJoined = Array.from(newMebershipIds).some((x) => !oldMebershipIds.has(x));
+
+            if (anyLeft) {
+                logger.debug(`Member(s) have left: queueing sender key rotation`);
+                this.makeNewKeyTimeout = setTimeout(this.onRotateKeyTimeout, MAKE_KEY_DELAY);
+            } else if (anyJoined) {
+                logger.debug(`New member(s) have joined: re-sending keys`);
+                this.requestKeyEventSend();
+            }
+        }
+
         this.setExpiryTimer();
     };
 
@@ -447,4 +780,15 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
             await this.triggerCallMembershipEventUpdate();
         }
     }
+
+    private onRotateKeyTimeout = (): void => {
+        if (!this.manageMediaKeys) return;
+
+        this.makeNewKeyTimeout = undefined;
+        logger.info("Making new sender key for key rotation");
+        this.makeNewSenderKey(true);
+        // send immediately: if we're about to start sending with a new key, it's
+        // important we get it out to others as soon as we can.
+        this.sendEncryptionKeysEvent();
+    };
 }

src/matrixrtc/MatrixRTCSessionManager.ts

@@ -17,10 +17,11 @@ limitations under the License.
 import { logger } from "../logger";
 import { MatrixClient, ClientEvent } from "../client";
 import { TypedEventEmitter } from "../models/typed-event-emitter";
-import { Room } from "../models/room";
+import { Room, RoomEvent } from "../models/room";
 import { RoomState, RoomStateEvent } from "../models/room-state";
 import { MatrixEvent } from "../models/event";
 import { MatrixRTCSession } from "./MatrixRTCSession";
+import { EventType } from "../@types/event";
 
 export enum MatrixRTCSessionManagerEvents {
     // A member has joined the MatrixRTC session, creating an active session in a room where there wasn't previously
@@ -62,6 +63,7 @@ export class MatrixRTCSessionManager extends TypedEventEmitter<MatrixRTCSessionM
         }
 
         this.client.on(ClientEvent.Room, this.onRoom);
+        this.client.on(RoomEvent.Timeline, this.onTimeline);
         this.client.on(RoomStateEvent.Events, this.onRoomState);
     }
 
@@ -72,6 +74,7 @@ export class MatrixRTCSessionManager extends TypedEventEmitter<MatrixRTCSessionM
         this.roomSessions.clear();
 
         this.client.removeListener(ClientEvent.Room, this.onRoom);
+        this.client.removeListener(RoomEvent.Timeline, this.onTimeline);
         this.client.removeListener(RoomStateEvent.Events, this.onRoomState);
     }
 
@@ -95,6 +98,18 @@ export class MatrixRTCSessionManager extends TypedEventEmitter<MatrixRTCSessionM
         return this.roomSessions.get(room.roomId)!;
     }
 
+    private onTimeline = (event: MatrixEvent): void => {
+        if (event.getType() !== EventType.CallEncryptionKeysPrefix) return;
+
+        const room = this.client.getRoom(event.getRoomId());
+        if (!room) {
+            logger.error(`Got room state event for unknown room ${event.getRoomId()}!`);
+            return;
+        }
+
+        this.getRoomSession(room).onCallEncryption(event);
+    };
+
     private onRoom = (room: Room): void => {
         this.refreshRoom(room);
     };
@@ -106,7 +121,9 @@ export class MatrixRTCSessionManager extends TypedEventEmitter<MatrixRTCSessionM
             return;
         }
 
-        this.refreshRoom(room);
+        if (event.getType() == EventType.GroupCallMemberPrefix) {
+            this.refreshRoom(room);
+        }
     };
 
     private refreshRoom(room: Room): void {

src/matrixrtc/types.ts

@@ -0,0 +1,35 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import { IMentions } from "../matrix";
+export interface EncryptionKeyEntry {
+    index: number;
+    key: string;
+}
+
+export interface EncryptionKeysEventContent {
+    keys: EncryptionKeyEntry[];
+    device_id: string;
+    call_id: string;
+}
+
+export type CallNotifyType = "ring" | "notify";
+
+export interface ICallNotifyContent {
+    "application": string;
+    "m.mentions": IMentions;
+    "notify_type": CallNotifyType;
+    "call_id": string;
+}

src/models/MSC3089Branch.ts

@@ -162,9 +162,8 @@ export class MSC3089Branch {
 
         if (!event) throw new Error("Failed to find event");
 
-        // Sometimes the event isn't decrypted for us, so do that. We specifically set `emit: true`
-        // to ensure that the relations system in the sdk will function.
-        await this.client.decryptEventIfNeeded(event, { emit: true, isRetry: true });
+        // Sometimes the event isn't decrypted for us, so do that.
+        await this.client.decryptEventIfNeeded(event);
 
         return event;
     }

src/models/compare-event-ordering.ts

@@ -0,0 +1,139 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { MatrixEvent } from "./event";
+import { Room } from "./room";
+import { inMainTimelineForReceipt, threadIdForReceipt } from "../client";
+
+/**
+ * Determine the order of two events in a room.
+ *
+ * In principle this should use the same order as the server, but in practice
+ * this is difficult for events that were not received over the Sync API. See
+ * MSC4033 for details.
+ *
+ * This implementation leans on the order of events within their timelines, and
+ * falls back to comparing event timestamps when they are in different
+ * timelines.
+ *
+ * See https://github.com/matrix-org/matrix-js-sdk/issues/3325 for where we are
+ * tracking the work to fix this.
+ *
+ * @param room - the room we are looking in
+ * @param leftEventId - the id of the first event
+ * @param rightEventId - the id of the second event
+
+ * @returns -1 if left \< right, 1 if left \> right, 0 if left == right, null if
+ *          we can't tell (because we can't find the events).
+ */
+export function compareEventOrdering(room: Room, leftEventId: string, rightEventId: string): number | null {
+    const leftEvent = room.findEventById(leftEventId);
+    const rightEvent = room.findEventById(rightEventId);
+
+    if (!leftEvent || !rightEvent) {
+        // Without the events themselves, we can't find their thread or
+        // timeline, or guess based on timestamp, so we just don't know.
+        return null;
+    }
+
+    // Check whether the events are in the main timeline
+    const isLeftEventInMainTimeline = inMainTimelineForReceipt(leftEvent);
+    const isRightEventInMainTimeline = inMainTimelineForReceipt(rightEvent);
+
+    if (isLeftEventInMainTimeline && isRightEventInMainTimeline) {
+        return compareEventsInMainTimeline(room, leftEventId, rightEventId, leftEvent, rightEvent);
+    } else {
+        // At least one event is not in the timeline, so we can't use the room's
+        // unfiltered timeline set.
+        return compareEventsInThreads(leftEventId, rightEventId, leftEvent, rightEvent);
+    }
+}
+
+function compareEventsInMainTimeline(
+    room: Room,
+    leftEventId: string,
+    rightEventId: string,
+    leftEvent: MatrixEvent,
+    rightEvent: MatrixEvent,
+): number | null {
+    // Get the timeline set that contains all the events.
+    const timelineSet = room.getUnfilteredTimelineSet();
+
+    // If they are in the same timeline, compareEventOrdering does what we need
+    const compareSameTimeline = timelineSet.compareEventOrdering(leftEventId, rightEventId);
+    if (compareSameTimeline !== null) {
+        return compareSameTimeline;
+    }
+
+    // Find which timeline each event is in. Refuse to provide an ordering if we
+    // can't find either of the events.
+
+    const leftTimeline = timelineSet.getTimelineForEvent(leftEventId);
+    if (leftTimeline === timelineSet.getLiveTimeline()) {
+        // The left event is part of the live timeline, so it must be after the
+        // right event (since they are not in the same timeline or we would have
+        // returned after compareEventOrdering.
+        return 1;
+    }
+
+    const rightTimeline = timelineSet.getTimelineForEvent(rightEventId);
+    if (rightTimeline === timelineSet.getLiveTimeline()) {
+        // The right event is part of the live timeline, so it must be after the
+        // left event.
+        return -1;
+    }
+
+    // They are in older timeline sets (because they were fetched by paging up).
+    return guessOrderBasedOnTimestamp(leftEvent, rightEvent);
+}
+
+function compareEventsInThreads(
+    leftEventId: string,
+    rightEventId: string,
+    leftEvent: MatrixEvent,
+    rightEvent: MatrixEvent,
+): number | null {
+    const leftEventThreadId = threadIdForReceipt(leftEvent);
+    const rightEventThreadId = threadIdForReceipt(rightEvent);
+
+    const leftThread = leftEvent.getThread();
+
+    if (leftThread && leftEventThreadId === rightEventThreadId) {
+        // They are in the same thread, so we can ask the thread's timeline to
+        // figure it out for us
+        return leftThread.timelineSet.compareEventOrdering(leftEventId, rightEventId);
+    } else {
+        return guessOrderBasedOnTimestamp(leftEvent, rightEvent);
+    }
+}
+
+/**
+ * Guess the order of events based on server timestamp. This is not good, but
+ * difficult to avoid without MSC4033.
+ *
+ * See https://github.com/matrix-org/matrix-js-sdk/issues/3325
+ */
+function guessOrderBasedOnTimestamp(leftEvent: MatrixEvent, rightEvent: MatrixEvent): number {
+    const leftTs = leftEvent.getTs();
+    const rightTs = rightEvent.getTs();
+    if (leftTs < rightTs) {
+        return -1;
+    } else if (leftTs > rightTs) {
+        return 1;
+    } else {
+        return 0;
+    }
+}

src/models/event-timeline-set.ts

@@ -839,7 +839,10 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
 
         const data: IRoomTimelineData = {
             timeline: timeline,
-            liveEvent: timeline == this.liveTimeline,
+            // The purpose of this method is inserting events in the middle of the
+            // timeline, so the events are, by definition, not live (whether or not
+            // we're adding them to the live timeline).
+            liveEvent: false,
         };
         this.emit(RoomEvent.Timeline, event, this.room, false, false, data);
     }
@@ -899,11 +902,10 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
      * @param eventId1 -   The id of the first event
      * @param eventId2 -   The id of the second event
 
-     * @returns a number less than zero if eventId1 precedes eventId2, and
-     *    greater than zero if eventId1 succeeds eventId2. zero if they are the
-     *    same event; null if we can't tell (either because we don't know about one
-     *    of the events, or because they are in separate timelines which don't join
-     *    up).
+     * @returns -1 if eventId1 precedes eventId2, and +1 eventId1 succeeds
+     * eventId2. 0 if they are the same event; null if we can't tell (either
+     * because we don't know about one of the events, or because they are in
+     * separate timelines which don't join up).
      */
     public compareEventOrdering(eventId1: string, eventId2: string): number | null {
         if (eventId1 == eventId2) {
@@ -935,7 +937,16 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
                     idx2 = idx;
                 }
             }
-            return idx1! - idx2!;
+            const difference = idx1! - idx2!;
+
+            // Return the sign of difference.
+            if (difference < 0) {
+                return -1;
+            } else if (difference > 0) {
+                return 1;
+            } else {
+                return 0;
+            }
         }
 
         // the events are in different timelines. Iterate through the

src/models/event.ts

@@ -45,6 +45,8 @@ import { DecryptionError } from "../crypto/algorithms";
 import { CryptoBackend } from "../common-crypto/CryptoBackend";
 import { WITHHELD_MESSAGES } from "../crypto/OlmDevice";
 import { IAnnotatedPushRule } from "../@types/PushRules";
+import { Room } from "./room";
+import { EventTimeline } from "./event-timeline";
 
 export { EventStatus } from "./event-status";
 
@@ -175,11 +177,23 @@ interface IKeyRequestRecipient {
 }
 
 export interface IDecryptOptions {
-    // Emits "event.decrypted" if set to true
+    /** Whether to emit {@link MatrixEventEvent.Decrypted} events on successful decryption. Defaults to true.
+     */
     emit?: boolean;
-    // True if this is a retry (enables more logging)
+
+    /**
+     * True if this is a retry, after receiving an update to the session key. (Enables more logging.)
+     *
+     * This is only intended for use within the js-sdk.
+     *
+     * @internal
+     */
     isRetry?: boolean;
-    // whether the message should be re-decrypted if it was previously successfully decrypted with an untrusted key
+
+    /**
+     * Whether the message should be re-decrypted if it was previously successfully decrypted with an untrusted key.
+     * Defaults to `false`.
+     */
     forceRedecryptIfUntrusted?: boolean;
 }
 
@@ -390,7 +404,13 @@ export class MatrixEvent extends TypedEventEmitter<MatrixEventEmittedEvents, Mat
         });
 
         this.txnId = event.txn_id;
-        this.localTimestamp = Date.now() - (this.getAge() ?? 0);
+        // The localTimestamp is calculated using the age.
+        // Some events lack an `age` property, either because they are EDUs such as typing events,
+        // or due to server-side bugs such as https://github.com/matrix-org/synapse/issues/8429.
+        // The fallback in these cases will be to use the origin_server_ts.
+        // For EDUs, the origin_server_ts also is not defined so we use Date.now().
+        const age = this.getAge();
+        this.localTimestamp = age !== undefined ? Date.now() - age : this.getTs() ?? Date.now();
         this.reEmitter = new TypedReEmitter(this);
     }
 
@@ -414,9 +434,12 @@ export class MatrixEvent extends TypedEventEmitter<MatrixEventEmittedEvents, Mat
     }
 
     /**
-     * Gets the event as though it would appear unencrypted. If the event is already not
-     * encrypted, it is simply returned as-is.
-     * @returns The event in wire format.
+     * Gets the event as it would appear if it had been sent unencrypted.
+     *
+     * If the event is encrypted, we attempt to mock up an event as it would have looked had the sender not encrypted it.
+     * If the event is not encrypted, a copy of it is simply returned as-is.
+     *
+     * @returns A shallow copy of the event, in wire format, as it would have been had it not been encrypted.
      */
     public getEffectiveEvent(): IEvent {
         const content = Object.assign({}, this.getContent()); // clone for mutation
@@ -1132,13 +1155,19 @@ export class MatrixEvent extends TypedEventEmitter<MatrixEventEmittedEvents, Mat
         return this.visibility;
     }
 
+    /**
+     * @deprecated In favor of the overload that includes a Room argument
+     */
+    public makeRedacted(redactionEvent: MatrixEvent): void;
     /**
      * Update the content of an event in the same way it would be by the server
      * if it were redacted before it was sent to us
      *
      * @param redactionEvent - event causing the redaction
+     * @param room - the room in which the event exists
      */
-    public makeRedacted(redactionEvent: MatrixEvent): void {
+    public makeRedacted(redactionEvent: MatrixEvent, room: Room): void;
+    public makeRedacted(redactionEvent: MatrixEvent, room?: Room): void {
         // quick sanity-check
         if (!redactionEvent.event) {
             throw new Error("invalid redactionEvent in makeRedacted");
@@ -1182,9 +1211,43 @@ export class MatrixEvent extends TypedEventEmitter<MatrixEventEmittedEvents, Mat
             }
         }
 
+        // If the redacted event was in a thread (but not thread root), move it
+        // to the main timeline. This will change if MSC3389 is merged.
+        if (room && !this.isThreadRoot && this.threadRootId && this.threadRootId !== this.getId()) {
+            this.moveAllRelatedToMainTimeline(room);
+            redactionEvent.moveToMainTimeline(room);
+        }
+
         this.invalidateExtensibleEvent();
     }
 
+    private moveAllRelatedToMainTimeline(room: Room): void {
+        const thread = this.thread;
+        this.moveToMainTimeline(room);
+
+        // If we dont have access to the thread, we can only move this
+        // event, not things related to it.
+        if (thread) {
+            for (const event of thread.events) {
+                if (event.getRelation()?.event_id === this.getId()) {
+                    event.moveAllRelatedToMainTimeline(room);
+                }
+            }
+        }
+    }
+
+    private moveToMainTimeline(room: Room): void {
+        // Remove it from its thread
+        this.thread?.timelineSet.removeEvent(this.getId()!);
+        this.setThread(undefined);
+
+        // And insert it into the main timeline
+        const timeline = room.getLiveTimeline();
+        // We use insertEventIntoTimeline to insert it in timestamp order,
+        // because we don't know where it should go (until we have MSC4033).
+        timeline.getTimelineSet().insertEventIntoTimeline(this, timeline, timeline.getState(EventTimeline.FORWARDS)!);
+    }
+
     /**
      * Check if this event has been redacted
      *
@@ -1593,15 +1656,21 @@ export class MatrixEvent extends TypedEventEmitter<MatrixEventEmittedEvents, Mat
     }
 
     /**
-     * Summarise the event as JSON. This is currently used by React SDK's view
-     * event source feature and Seshat's event indexing, so take care when
-     * adjusting the output here.
+     * Summarise the event as JSON.
      *
      * If encrypted, include both the decrypted and encrypted view of the event.
      *
      * This is named `toJSON` for use with `JSON.stringify` which checks objects
      * for functions named `toJSON` and will call them to customise the output
      * if they are defined.
+     *
+     * **WARNING** Do not log the result of this method; otherwise, it will end up
+     * in rageshakes, leading to a privacy violation.
+     *
+     * @deprecated Prefer to use {@link MatrixEvent#getEffectiveEvent} or similar.
+     * This method will be removed soon; it is too easy to use it accidentally
+     * and cause a privacy violation (cf https://github.com/vector-im/element-web/issues/26380).
+     * In any case, the value it returns is not a faithful serialization of the object.
      */
     public toJSON(): object {
         const event = this.getEffectiveEvent();

src/models/read-receipt.ts

@@ -26,15 +26,30 @@ import { EventType } from "../@types/event";
 import { EventTimelineSet } from "./event-timeline-set";
 import { MapWithDefault } from "../utils";
 import { NotificationCountType } from "./room";
+import { logger } from "../logger";
+import { inMainTimelineForReceipt, threadIdForReceipt } from "../client";
 
-export function synthesizeReceipt(userId: string, event: MatrixEvent, receiptType: ReceiptType): MatrixEvent {
+/**
+ * Create a synthetic receipt for the given event
+ * @param userId - The user ID if the receipt sender
+ * @param event - The event that is to be acknowledged
+ * @param receiptType - The type of receipt
+ * @param unthreaded - the receipt is unthreaded
+ * @returns a new event with the synthetic receipt in it
+ */
+export function synthesizeReceipt(
+    userId: string,
+    event: MatrixEvent,
+    receiptType: ReceiptType,
+    unthreaded = false,
+): MatrixEvent {
     return new MatrixEvent({
         content: {
             [event.getId()!]: {
                 [receiptType]: {
                     [userId]: {
                         ts: event.getTs(),
-                        thread_id: event.threadRootId ?? MAIN_ROOM_TIMELINE,
+                        ...(!unthreaded && { thread_id: threadIdForReceipt(event) }),
                     },
                 },
             },
@@ -94,15 +109,115 @@ export abstract class ReadReceipt<
     }
 
     /**
-     * Get the ID of the event that a given user has read up to, or null if we
-     * have received no read receipts from them.
+     * Get the ID of the event that a given user has read up to, or null if:
+     * - we have received no read receipts for them, or
+     * - the receipt we have points at an event we don't have, or
+     * - the thread ID in the receipt does not match the thread root of the
+     *   referenced event.
+     *
+     * (The event might not exist if it is not loaded, and the thread ID might
+     * not match if the event has moved thread because it was redacted.)
+     *
      * @param userId - The user ID to get read receipt event ID for
      * @param ignoreSynthesized - If true, return only receipts that have been
-     *                                    sent by the server, not implicit ones generated
-     *                                    by the JS SDK.
-     * @returns ID of the latest event that the given user has read, or null.
+     *                            sent by the server, not implicit ones generated
+     *                            by the JS SDK.
+     * @returns ID of the latest existing event that the given user has read, or null.
      */
     public getEventReadUpTo(userId: string, ignoreSynthesized = false): string | null {
+        // Find what the latest receipt says is the latest event we have read
+        const latestReceipt = this.getLatestReceipt(userId, ignoreSynthesized);
+
+        if (!latestReceipt) {
+            return null;
+        }
+
+        return this.receiptPointsAtConsistentEvent(latestReceipt) ? latestReceipt.eventId : null;
+    }
+
+    /**
+     * Returns true if the event pointed at by this receipt exists, and its
+     * threadRootId is consistent with the thread information in the receipt.
+     */
+    private receiptPointsAtConsistentEvent(receipt: WrappedReceipt): boolean {
+        const event = this.findEventById(receipt.eventId);
+        if (!event) {
+            // If the receipt points at a non-existent event, we have multiple
+            // possibilities:
+            //
+            // 1. We don't have the event because it's not loaded yet - probably
+            //    it's old and we're best off ignoring the receipt - we can just
+            //    send a new one when we read a new event.
+            //
+            // 2. We have a bug e.g. we misclassified this event into the wrong
+            //    thread.
+            //
+            // 3. The referenced event moved out of this thread (e.g. because it
+            //    was deleted.)
+            //
+            // 4. The receipt had the incorrect thread ID (due to a bug in a
+            // client, or malicious behaviour).
+
+            // This receipt is not "valid" because it doesn't point at an event
+            // we have. We want to pretend it doesn't exist.
+            return false;
+        }
+
+        if (!receipt.data?.thread_id) {
+            // If this is an unthreaded receipt, it could point at any event, so
+            // there is no need to validate further - this receipt is valid.
+            return true;
+        }
+        // Otherwise it is a threaded receipt...
+
+        if (receipt.data.thread_id === MAIN_ROOM_TIMELINE) {
+            // The receipt is for the main timeline: we check that the event is
+            // in the main timeline.
+
+            // Check if the event is in the main timeline
+            const eventIsInMainTimeline = inMainTimelineForReceipt(event);
+
+            if (eventIsInMainTimeline) {
+                // The receipt is for the main timeline, and so is the event, so
+                // the receipt is valid.
+                return true;
+            }
+        } else {
+            // The receipt is for a different thread (not the main timeline)
+
+            if (event.threadRootId === receipt.data.thread_id) {
+                // If the receipt and event agree on the thread ID, the receipt
+                // is valid.
+                return true;
+            }
+        }
+
+        // The receipt thread ID disagrees with the event thread ID. There are 2
+        // possibilities:
+        //
+        // 1. The event moved to a different thread after the receipt was
+        //    created. This can happen if the event was redacted because that
+        //    moves it to the main timeline.
+        //
+        // 2. There is a bug somewhere - either we put the event into the wrong
+        //    thread, or someone sent an incorrect receipt.
+        //
+        // In many cases, we won't get here because the call to findEventById
+        // would have already returned null. We include this check to cover
+        // cases when `this` is a  room, meaning findEventById will find events
+        // in any thread, and to be defensive against unforeseen code paths.
+        logger.warn(
+            `Ignoring receipt because its thread_id (${receipt.data.thread_id}) disagrees ` +
+                `with the thread root (${event.threadRootId}) of the referenced event ` +
+                `(event ID = ${receipt.eventId})`,
+        );
+
+        // This receipt is not "valid" because it disagrees with us about what
+        // thread the event is in. We want to pretend it doesn't exist.
+        return false;
+    }
+
+    private getLatestReceipt(userId: string, ignoreSynthesized: boolean): WrappedReceipt | null {
         // XXX: This is very very ugly and I hope I won't have to ever add a new
         // receipt type here again. IMHO this should be done by the server in
         // some more intelligent manner or the client should just use timestamps
@@ -118,10 +233,10 @@ export abstract class ReadReceipt<
 
         // The public receipt is more likely to drift out of date so the private
         // one has precedence
-        if (!comparison) return privateReadReceipt?.eventId ?? publicReadReceipt?.eventId ?? null;
+        if (!comparison) return privateReadReceipt ?? publicReadReceipt ?? null;
 
         // If public read receipt is older, return the private one
-        return (comparison < 0 ? privateReadReceipt?.eventId : publicReadReceipt?.eventId) ?? null;
+        return (comparison < 0 ? privateReadReceipt : publicReadReceipt) ?? null;
     }
 
     public addReceiptToStructure(
@@ -229,6 +344,13 @@ export abstract class ReadReceipt<
 
     public abstract setUnread(type: NotificationCountType, count: number): void;
 
+    /**
+     * Look in this room/thread's timeline to find an event. If `this` is a
+     * room, we look in all threads, but if `this` is a thread, we look only
+     * inside this thread.
+     */
+    public abstract findEventById(eventId: string): MatrixEvent | undefined;
+
     /**
      * This issue should also be addressed on synapse's side and is tracked as part
      * of https://github.com/matrix-org/synapse/issues/14837
@@ -256,9 +378,10 @@ export abstract class ReadReceipt<
      * @param userId - The user ID if the receipt sender
      * @param e - The event that is to be acknowledged
      * @param receiptType - The type of receipt
+     * @param unthreaded - the receipt is unthreaded
      */
-    public addLocalEchoReceipt(userId: string, e: MatrixEvent, receiptType: ReceiptType): void {
-        this.addReceipt(synthesizeReceipt(userId, e, receiptType), true);
+    public addLocalEchoReceipt(userId: string, e: MatrixEvent, receiptType: ReceiptType, unthreaded = false): void {
+        this.addReceipt(synthesizeReceipt(userId, e, receiptType, unthreaded), true);
     }
 
     /**
@@ -284,33 +407,7 @@ export abstract class ReadReceipt<
      * @param eventId - The event ID to check if the user read.
      * @returns True if the user has read the event, false otherwise.
      */
-    public hasUserReadEvent(userId: string, eventId: string): boolean {
-        const readUpToId = this.getEventReadUpTo(userId, false);
-        if (readUpToId === eventId) return true;
-
-        if (
-            this.timeline?.length &&
-            this.timeline[this.timeline.length - 1].getSender() &&
-            this.timeline[this.timeline.length - 1].getSender() === userId
-        ) {
-            // It doesn't matter where the event is in the timeline, the user has read
-            // it because they've sent the latest event.
-            return true;
-        }
-
-        for (let i = this.timeline?.length - 1; i >= 0; --i) {
-            const ev = this.timeline[i];
-
-            // If we encounter the target event first, the user hasn't read it
-            // however if we encounter the readUpToId first then the user has read
-            // it. These rules apply because we're iterating bottom-up.
-            if (ev.getId() === eventId) return false;
-            if (ev.getId() === readUpToId) return true;
-        }
-
-        // We don't know if the user has read it, so assume not.
-        return false;
-    }
+    public abstract hasUserReadEvent(userId: string, eventId: string): boolean;
 
     /**
      * Returns the most recent unthreaded receipt for a given user
@@ -318,6 +415,8 @@ export abstract class ReadReceipt<
      * @returns an unthreaded Receipt. Can be undefined if receipts have been disabled
      * or a user chooses to use private read receipts (or we have simply not received
      * a receipt from this user yet).
+     *
+     * @deprecated use `hasUserReadEvent` or `getEventReadUpTo` instead
      */
     public abstract getLastUnthreadedReceiptFor(userId: string): Receipt | undefined;
 }

src/models/room-receipts.ts

@@ -0,0 +1,435 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { MAIN_ROOM_TIMELINE, Receipt, ReceiptContent } from "../@types/read_receipts";
+import { threadIdForReceipt } from "../client";
+import { Room, RoomEvent } from "./room";
+import { MatrixEvent } from "./event";
+import { logger } from "../logger";
+
+/**
+ * The latest receipts we have for a room.
+ */
+export class RoomReceipts {
+    private room: Room;
+    private threadedReceipts: ThreadedReceipts;
+    private unthreadedReceipts: ReceiptsByUser;
+    private danglingReceipts: DanglingReceipts;
+
+    public constructor(room: Room) {
+        this.room = room;
+        this.threadedReceipts = new ThreadedReceipts(room);
+        this.unthreadedReceipts = new ReceiptsByUser(room);
+        this.danglingReceipts = new DanglingReceipts();
+        // We listen for timeline events so we can process dangling receipts
+        room.on(RoomEvent.Timeline, this.onTimelineEvent);
+    }
+
+    /**
+     * Remember the receipt information supplied. For each receipt:
+     *
+     * If we don't have the event for this receipt, store it as "dangling" so we
+     * can process it later.
+     *
+     * Otherwise store it per-user in either the threaded store for its
+     * thread_id, or the unthreaded store if there is no thread_id.
+     *
+     * Ignores any receipt that is before an existing receipt for the same user
+     * (in the same thread, if applicable). "Before" is defined by the
+     * unfilteredTimelineSet of the room.
+     */
+    public add(receiptContent: ReceiptContent, synthetic: boolean): void {
+        /*
+            Transform this structure:
+            {
+              "$EVENTID": {
+                "m.read|m.read.private": {
+                  "@user:example.org": {
+                    "ts": 1661,
+                    "thread_id": "main|$THREAD_ROOT_ID" // or missing/undefined for an unthreaded receipt
+                  }
+                }
+              },
+              ...
+            }
+            into maps of:
+            threaded :: threadid :: userId :: ReceiptInfo
+            unthreaded :: userId :: ReceiptInfo
+            dangling :: eventId :: DanglingReceipt
+        */
+        for (const [eventId, eventReceipt] of Object.entries(receiptContent)) {
+            for (const [receiptType, receiptsByUser] of Object.entries(eventReceipt)) {
+                for (const [userId, receipt] of Object.entries(receiptsByUser)) {
+                    const referencedEvent = this.room.findEventById(eventId);
+                    if (!referencedEvent) {
+                        this.danglingReceipts.add(
+                            new DanglingReceipt(eventId, receiptType, userId, receipt, synthetic),
+                        );
+                    } else if (receipt.thread_id) {
+                        this.threadedReceipts.set(
+                            receipt.thread_id,
+                            eventId,
+                            receiptType,
+                            userId,
+                            receipt.ts,
+                            synthetic,
+                        );
+                    } else {
+                        this.unthreadedReceipts.set(eventId, receiptType, userId, receipt.ts, synthetic);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Look for dangling receipts for the given event ID,
+     * and add them to the thread of unthread receipts if found.
+     * @param eventId - the event ID to look for
+     */
+    private onTimelineEvent = (event: MatrixEvent): void => {
+        const eventId = event.getId();
+        if (!eventId) return;
+
+        const danglingReceipts = this.danglingReceipts.remove(eventId);
+
+        danglingReceipts?.forEach((danglingReceipt) => {
+            // The receipt is a thread receipt
+            if (danglingReceipt.receipt.thread_id) {
+                this.threadedReceipts.set(
+                    danglingReceipt.receipt.thread_id,
+                    danglingReceipt.eventId,
+                    danglingReceipt.receiptType,
+                    danglingReceipt.userId,
+                    danglingReceipt.receipt.ts,
+                    danglingReceipt.synthetic,
+                );
+            } else {
+                this.unthreadedReceipts.set(
+                    eventId,
+                    danglingReceipt.receiptType,
+                    danglingReceipt.userId,
+                    danglingReceipt.receipt.ts,
+                    danglingReceipt.synthetic,
+                );
+            }
+        });
+    };
+
+    public hasUserReadEvent(userId: string, eventId: string): boolean {
+        const unthreaded = this.unthreadedReceipts.get(userId);
+        if (unthreaded) {
+            if (isAfterOrSame(unthreaded.eventId, eventId, this.room)) {
+                // The unthreaded receipt is after this event, so we have read it.
+                return true;
+            }
+        }
+
+        const event = this.room.findEventById(eventId);
+        if (!event) {
+            // We don't know whether the user has read it - default to caution and say no.
+            // This shouldn't really happen and feels like it ought to be an exception: let's
+            // log a warn for now.
+            logger.warn(
+                `hasUserReadEvent event ID ${eventId} not found in room ${this.room.roomId}: this shouldn't happen!`,
+            );
+            return false;
+        }
+
+        const threadId = threadIdForReceipt(event);
+        const threaded = this.threadedReceipts.get(threadId, userId);
+        if (threaded) {
+            if (isAfterOrSame(threaded.eventId, eventId, this.room)) {
+                // The threaded receipt is after this event, so we have read it.
+                return true;
+            }
+        }
+
+        // TODO: what if they sent the second-last event in the thread?
+        if (this.userSentLatestEventInThread(threadId, userId)) {
+            // The user sent the latest message in this event's thread, so we
+            // consider everything in the thread to be read.
+            //
+            // Note: maybe we don't need this because synthetic receipts should
+            // do this job for us?
+            return true;
+        }
+
+        // Neither of the receipts were after the event, so it's unread.
+        return false;
+    }
+
+    /**
+     * @returns true if the thread with this ID can be found, and the supplied
+     *          user sent the latest message in it.
+     */
+    private userSentLatestEventInThread(threadId: string, userId: String): boolean {
+        const timeline =
+            threadId === MAIN_ROOM_TIMELINE
+                ? this.room.getLiveTimeline().getEvents()
+                : this.room.getThread(threadId)?.timeline;
+
+        return !!(timeline && timeline.length > 0 && timeline[timeline.length - 1].getSender() === userId);
+    }
+}
+
+// --- implementation details ---
+
+/**
+ * The information "inside" a receipt once it has been stored inside
+ * RoomReceipts - what eventId it refers to, its type, and its ts.
+ *
+ * Does not contain userId or threadId since these are stored as keys of the
+ * maps in RoomReceipts.
+ */
+class ReceiptInfo {
+    public constructor(public eventId: string, public receiptType: string, public ts: number) {}
+}
+
+/**
+ * Everything we know about a receipt that is "dangling" because we can't find
+ * the event to which it refers.
+ */
+class DanglingReceipt {
+    public constructor(
+        public eventId: string,
+        public receiptType: string,
+        public userId: string,
+        public receipt: Receipt,
+        public synthetic: boolean,
+    ) {}
+}
+
+class UserReceipts {
+    private room: Room;
+
+    /**
+     * The real receipt for this user.
+     */
+    private real: ReceiptInfo | undefined;
+
+    /**
+     * The synthetic receipt for this user. If this is defined, it is later than real.
+     */
+    private synthetic: ReceiptInfo | undefined;
+
+    public constructor(room: Room) {
+        this.room = room;
+        this.real = undefined;
+        this.synthetic = undefined;
+    }
+
+    public set(synthetic: boolean, receiptInfo: ReceiptInfo): void {
+        if (synthetic) {
+            this.synthetic = receiptInfo;
+        } else {
+            this.real = receiptInfo;
+        }
+
+        // Preserve the invariant: synthetic is only defined if it's later than real
+        if (this.synthetic && this.real) {
+            if (isAfterOrSame(this.real.eventId, this.synthetic.eventId, this.room)) {
+                this.synthetic = undefined;
+            }
+        }
+    }
+
+    /**
+     * Return the latest receipt we have - synthetic if we have one (and it's
+     * later), otherwise real.
+     */
+    public get(): ReceiptInfo | undefined {
+        // Relies on the invariant that synthetic is only defined if it's later than real.
+        return this.synthetic ?? this.real;
+    }
+
+    /**
+     * Return the latest receipt we have of the specified type (synthetic or not).
+     */
+    public getByType(synthetic: boolean): ReceiptInfo | undefined {
+        return synthetic ? this.synthetic : this.real;
+    }
+}
+
+/**
+ * The latest receipt info we have, either for a single thread, or all the
+ * unthreaded receipts for a room.
+ *
+ * userId: ReceiptInfo
+ */
+class ReceiptsByUser {
+    private room: Room;
+
+    /** map of userId: UserReceipts */
+    private data: Map<String, UserReceipts>;
+
+    public constructor(room: Room) {
+        this.room = room;
+        this.data = new Map<string, UserReceipts>();
+    }
+
+    /**
+     * Add the supplied receipt to our structure, if it is not earlier than the
+     * one we already hold for this user.
+     */
+    public set(eventId: string, receiptType: string, userId: string, ts: number, synthetic: boolean): void {
+        const userReceipts = getOrCreate(this.data, userId, () => new UserReceipts(this.room));
+
+        const existingReceipt = userReceipts.getByType(synthetic);
+        if (existingReceipt && isAfter(existingReceipt.eventId, eventId, this.room)) {
+            // The new receipt is before the existing one - don't store it.
+            return;
+        }
+
+        // Possibilities:
+        //
+        // 1. there was no existing receipt, or
+        // 2. the existing receipt was before this one, or
+        // 3. we were unable to compare the receipts.
+        //
+        // In the case of 3 it's difficult to decide what to do, so the
+        // most-recently-received receipt wins.
+        //
+        // Case 3 can only happen if the events for these receipts have
+        // disappeared, which is quite unlikely since the new one has just been
+        // checked, and the old one was checked before it was inserted here.
+        //
+        // We go ahead and store this receipt (replacing the other if it exists)
+        userReceipts.set(synthetic, new ReceiptInfo(eventId, receiptType, ts));
+    }
+
+    /**
+     * Find the latest receipt we have for this user. (Note - there is only one
+     * receipt per user, because we are already inside a specific thread or
+     * unthreaded list.)
+     *
+     * If there is a later synthetic receipt for this user, return that.
+     * Otherwise, return the real receipt.
+     *
+     * @returns the found receipt info, or undefined if we have no receipt for this user.
+     */
+    public get(userId: string): ReceiptInfo | undefined {
+        return this.data.get(userId)?.get();
+    }
+}
+
+/**
+ * The latest threaded receipts we have for a room.
+ */
+class ThreadedReceipts {
+    private room: Room;
+
+    /** map of threadId: ReceiptsByUser */
+    private data: Map<string, ReceiptsByUser>;
+
+    public constructor(room: Room) {
+        this.room = room;
+        this.data = new Map<string, ReceiptsByUser>();
+    }
+
+    /**
+     * Add the supplied receipt to our structure, if it is not earlier than one
+     * we already hold for this user in this thread.
+     */
+    public set(
+        threadId: string,
+        eventId: string,
+        receiptType: string,
+        userId: string,
+        ts: number,
+        synthetic: boolean,
+    ): void {
+        const receiptsByUser = getOrCreate(this.data, threadId, () => new ReceiptsByUser(this.room));
+        receiptsByUser.set(eventId, receiptType, userId, ts, synthetic);
+    }
+
+    /**
+     * Find the latest threaded receipt for the supplied user in the supplied thread.
+     *
+     * @returns the found receipt info or undefined if we don't have one.
+     */
+    public get(threadId: string, userId: string): ReceiptInfo | undefined {
+        return this.data.get(threadId)?.get(userId);
+    }
+}
+
+/**
+ * All the receipts that we have received but can't process because we can't
+ * find the event they refer to.
+ *
+ * We hold on to them so we can process them if their event arrives later.
+ */
+class DanglingReceipts {
+    /**
+     * eventId: DanglingReceipt[]
+     */
+    private data = new Map<string, Array<DanglingReceipt>>();
+
+    /**
+     * Remember the supplied dangling receipt.
+     */
+    public add(danglingReceipt: DanglingReceipt): void {
+        const danglingReceipts = getOrCreate(this.data, danglingReceipt.eventId, () => []);
+        danglingReceipts.push(danglingReceipt);
+    }
+
+    /**
+     * Remove and return the dangling receipts for the given event ID.
+     * @param eventId - the event ID to look for
+     * @returns the found dangling receipts, or undefined if we don't have one.
+     */
+    public remove(eventId: string): Array<DanglingReceipt> | undefined {
+        const danglingReceipts = this.data.get(eventId);
+        this.data.delete(eventId);
+        return danglingReceipts;
+    }
+}
+
+function getOrCreate<K, V>(m: Map<K, V>, key: K, createFn: () => V): V {
+    const found = m.get(key);
+    if (found) {
+        return found;
+    } else {
+        const created = createFn();
+        m.set(key, created);
+        return created;
+    }
+}
+
+/**
+ * Is left after right (or the same)?
+ *
+ * Only returns true if both events can be found, and left is after or the same
+ * as right.
+ *
+ * @returns left \>= right
+ */
+function isAfterOrSame(leftEventId: string, rightEventId: string, room: Room): boolean {
+    const comparison = room.compareEventOrdering(leftEventId, rightEventId);
+    return comparison !== null && comparison >= 0;
+}
+
+/**
+ * Is left strictly after right?
+ *
+ * Only returns true if both events can be found, and left is strictly after right.
+ *
+ * @returns left \> right
+ */
+function isAfter(leftEventId: string, rightEventId: string, room: Room): boolean {
+    const comparison = room.compareEventOrdering(leftEventId, rightEventId);
+    return comparison !== null && comparison > 0;
+}

src/models/room.ts

@@ -66,6 +66,8 @@ import { IStateEventWithRoomId } from "../@types/search";
 import { RelationsContainer } from "./relations-container";
 import { ReadReceipt, synthesizeReceipt } from "./read-receipt";
 import { isPollEvent, Poll, PollEvent } from "./poll";
+import { RoomReceipts } from "./room-receipts";
+import { compareEventOrdering } from "./compare-event-ordering";
 
 // These constants are used as sane defaults when the homeserver doesn't support
 // the m.room_versions capability. In practice, KNOWN_SAFE_ROOM_VERSION should be
@@ -236,8 +238,9 @@ export type RoomEventHandlerMap = {
      *
      * @param event - The matrix redaction event
      * @param room - The room containing the redacted event
+     * @param threadId - The thread containing the redacted event (before it was redacted)
      */
-    [RoomEvent.Redaction]: (event: MatrixEvent, room: Room) => void;
+    [RoomEvent.Redaction]: (event: MatrixEvent, room: Room, threadId?: string) => void;
     /**
      * Fires when an event that was previously redacted isn't anymore.
      * This happens when the redaction couldn't be sent and
@@ -431,6 +434,12 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
      */
     private visibilityEvents = new Map<string, MatrixEvent[]>();
 
+    /**
+     * The latest receipts (synthetic and real) for each user in each thread
+     * (and unthreaded).
+     */
+    private roomReceipts = new RoomReceipts(this);
+
     /**
      * Construct a new Room.
      *
@@ -549,7 +558,7 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
         const decryptionPromises = events
             .slice(readReceiptTimelineIndex)
             .reverse()
-            .map((event) => this.client.decryptEventIfNeeded(event, { isRetry: true }));
+            .map((event) => this.client.decryptEventIfNeeded(event));
 
         await Promise.allSettled(decryptionPromises);
     }
@@ -567,7 +576,7 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
             .getEvents()
             .slice(0) // copy before reversing
             .reverse()
-            .map((event) => this.client.decryptEventIfNeeded(event, { isRetry: true }));
+            .map((event) => this.client.decryptEventIfNeeded(event));
 
         await Promise.allSettled(decryptionPromises);
     }
@@ -2113,6 +2122,12 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
      * Relations (other than m.thread), redactions, replies to a thread root live only in the main timeline
      * Relations, redactions, replies where the parent cannot be found live in no timelines but should be aggregated regardless.
      * Otherwise, the event lives in the main timeline only.
+     *
+     * Note: when a redaction is applied, the redacted event, events relating
+     * to it, and the redaction event itself, will all move to the main thread.
+     * This method classifies them as inside the thread of the redacted event.
+     * They are moved later as part of makeRedacted.
+     * This will change if MSC3389 is merged.
      */
     public eventShouldLiveIn(
         event: MatrixEvent,
@@ -2329,7 +2344,8 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
             // if we know about this event, redact its contents now.
             const redactedEvent = redactId ? this.findEventById(redactId) : undefined;
             if (redactedEvent) {
-                redactedEvent.makeRedacted(event);
+                const threadRootId = redactedEvent.threadRootId;
+                redactedEvent.makeRedacted(event, this);
 
                 // If this is in the current state, replace it with the redacted version
                 if (redactedEvent.isState()) {
@@ -2342,7 +2358,7 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
                     }
                 }
 
-                this.emit(RoomEvent.Redaction, event, this);
+                this.emit(RoomEvent.Redaction, event, this, threadRootId);
 
                 // TODO: we stash user displaynames (among other things) in
                 // RoomMember objects which are then attached to other events
@@ -2495,7 +2511,7 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
                 }
                 if (redactedEvent) {
                     redactedEvent.markLocallyRedacted(event);
-                    this.emit(RoomEvent.Redaction, event, this);
+                    this.emit(RoomEvent.Redaction, event, this, redactedEvent.threadRootId);
                 }
             }
         } else {
@@ -2927,6 +2943,10 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
      */
     public addReceipt(event: MatrixEvent, synthetic = false): void {
         const content = event.getContent<ReceiptContent>();
+
+        this.roomReceipts.add(content, synthetic);
+
+        // TODO: delete the following code when it has been replaced by RoomReceipts
         Object.keys(content).forEach((eventId: string) => {
             Object.keys(content[eventId]).forEach((receiptType: ReceiptType | string) => {
                 Object.keys(content[eventId][receiptType]).forEach((userId: string) => {
@@ -2988,6 +3008,7 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
                 });
             });
         });
+        // End of code to delete when replaced by RoomReceipts
 
         // send events after we've regenerated the structure & cache, otherwise things that
         // listened for the event would read stale data.
@@ -3574,6 +3595,19 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
         return this.oldestThreadedReceiptTs;
     }
 
+    /**
+     * Determines if the given user has read a particular event ID with the known
+     * history of the room. This is not a definitive check as it relies only on
+     * what is available to the room at the time of execution.
+     *
+     * @param userId - The user ID to check the read state of.
+     * @param eventId - The event ID to check if the user read.
+     * @returns true if the user has read the event, false otherwise.
+     */
+    public hasUserReadEvent(userId: string, eventId: string): boolean {
+        return this.roomReceipts.hasUserReadEvent(userId, eventId);
+    }
+
     /**
      * Returns the most recent unthreaded receipt for a given user
      * @param userId - the MxID of the User
@@ -3607,6 +3641,30 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
             thread.fixupNotifications(userId);
         }
     }
+
+    /**
+     * Determine the order of two events in this room.
+     *
+     * In principle this should use the same order as the server, but in practice
+     * this is difficult for events that were not received over the Sync API. See
+     * MSC4033 for details.
+     *
+     * This implementation leans on the order of events within their timelines, and
+     * falls back to comparing event timestamps when they are in different
+     * timelines.
+     *
+     * See https://github.com/matrix-org/matrix-js-sdk/issues/3325 for where we are
+     * tracking the work to fix this.
+     *
+     * @param leftEventId - the id of the first event
+     * @param rightEventId - the id of the second event
+
+     * @returns -1 if left \< right, 1 if left \> right, 0 if left == right, null if
+     *          we can't tell (because we can't find the events).
+     */
+    public compareEventOrdering(leftEventId: string, rightEventId: string): number | null {
+        return compareEventOrdering(this, leftEventId, rightEventId);
+    }
 }
 
 // a map from current event status to a list of allowed next statuses

src/models/thread.ts

@@ -143,6 +143,9 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
     public constructor(public readonly id: string, public rootEvent: MatrixEvent | undefined, opts: IThreadOpts) {
         super();
 
+        // each Event in the thread adds a reemitter, so we could hit the listener limit.
+        this.setMaxListeners(1000);
+
         if (!opts?.room) {
             // Logging/debugging for https://github.com/vector-im/element-web/issues/22141
             // Hope is that we end up with a more obvious stack trace.
@@ -228,8 +231,8 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
         }
     };
 
-    private onRedaction = async (event: MatrixEvent): Promise<void> => {
-        if (event.threadRootId !== this.id) return; // ignore redactions for other timelines
+    private onRedaction = async (event: MatrixEvent, room: Room, threadRootId?: string): Promise<void> => {
+        if (threadRootId !== this.id) return; // ignore redactions for other timelines
         if (this.replyCount <= 0) {
             for (const threadEvent of this.timeline) {
                 this.clearEventMetadata(threadEvent);
@@ -368,7 +371,7 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
         if (!Thread.hasServerSideSupport) {
             // When there's no server-side support, just add it to the end of the timeline.
             this.addEventToTimeline(event, toStartOfTimeline);
-            this.client.decryptEventIfNeeded(event, {});
+            this.client.decryptEventIfNeeded(event);
         } else if (!toStartOfTimeline && this.initialEventsFetched && isNewestReply) {
             // When we've asked for the event to be added to the end, and we're
             // not in the initial state, and this event belongs at the end, add it.
@@ -430,22 +433,19 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
             // If initial events have not been fetched, we are OK to throw away
             // this event, because we are about to fetch all the events for this
             // thread from the server.
-            //
-            // If not, this looks like a bug - we should always add the event to
-            // the thread. See https://github.com/vector-im/element-web/issues/26254
-            logger.warn(
-                `Not adding event ${event.getId()} to thread timeline!
-                isNewestReply=${isNewestReply}
-                event.localTimestamp=${event.localTimestamp}
-                !!lastReply=${!!lastReply}
-                lastReply?.getId()=${lastReply?.getId()}
-                lastReply?.localTimestamp=${lastReply?.localTimestamp}
-                toStartOfTimeline=${toStartOfTimeline}
-                Thread.hasServerSideSupport=${Thread.hasServerSideSupport}
-                event.isRelation(RelationType.Annotation)=${event.isRelation(RelationType.Annotation)}
-                event.isRelation(RelationType.Replace)=${event.isRelation(RelationType.Replace)}
-                `,
-            );
+
+            // Otherwise, we should add it, but we suspect it is out of order.
+            if (toStartOfTimeline) {
+                // If we're adding at the start of the timeline, it doesn't
+                // matter that it's out of order.
+                this.addEventToTimeline(event, toStartOfTimeline);
+            } else {
+                // We think this event might be out of order, because isNewestReply
+                // is false (otherwise we would have gone into the earlier if
+                // clause), so try to insert it in the right place based on
+                // timestamp.
+                this.insertEventIntoTimeline(event);
+            }
         }
 
         if (
@@ -748,6 +748,27 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
      * @returns ID of the latest event that the given user has read, or null.
      */
     public getEventReadUpTo(userId: string, ignoreSynthesized?: boolean): string | null {
+        // TODO: we think the implementation here is not right. Here is a sketch
+        // of the right answer:
+        //
+        // for event in timeline.events.reversed():
+        //     if room.hasUserReadEvent(event):
+        //         return event
+        // return null
+        //
+        // If this is too slow, we might be able to improve it by trying walking
+        // forward from the threaded receipt in this thread. We could alternate
+        // between backwards-from-front and forwards-from-threaded-receipt to
+        // improve our chances of hitting the right answer sooner.
+        //
+        // Either way, it's still fundamentally slow because we have to walk
+        // events.
+        //
+        // We also might just want to limit the time we spend on this by giving
+        // up after, say, 100 events.
+        //
+        // --- andyb
+
         const isCurrentUser = userId === this.client.getUserId();
         const lastReply = this.timeline[this.timeline.length - 1];
         if (isCurrentUser && lastReply) {
@@ -816,7 +837,7 @@ export class Thread extends ReadReceipt<ThreadEmittedEvents, ThreadEventHandlerM
             }
         }
 
-        return super.hasUserReadEvent(userId, eventId);
+        return this.room.hasUserReadEvent(userId, eventId);
     }
 
     public setUnread(type: NotificationCountType, count: number): void {

src/models/user.ts

@@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { MatrixClient } from "../matrix";
 import { MatrixEvent } from "./event";
 import { TypedEventEmitter } from "./typed-event-emitter";
 
@@ -152,6 +153,7 @@ export class User extends TypedEventEmitter<UserEvent, UserEventHandlerMap> {
     /**
      * Construct a new User. A User must have an ID and can optionally have extra information associated with it.
      * @param userId - Required. The ID of this user.
+     * @deprecated use `User.createUser`
      */
     public constructor(public readonly userId: string) {
         super();
@@ -160,6 +162,25 @@ export class User extends TypedEventEmitter<UserEvent, UserEventHandlerMap> {
         this.updateModifiedTime();
     }
 
+    /**
+     * Construct a new User whose events will also emit on MatrixClient.
+     * A User must have an ID and can optionally have extra information associated with it.
+     * @param userId - Required. The ID of this user.
+     * @param client - An instance of MatrixClient object
+     * @returns User object with reEmitter setup on client
+     */
+    public static createUser(userId: string, client: MatrixClient): User {
+        const user = new User(userId);
+        client.reEmitter.reEmit(user, [
+            UserEvent.AvatarUrl,
+            UserEvent.DisplayName,
+            UserEvent.Presence,
+            UserEvent.CurrentlyActive,
+            UserEvent.LastPresenceTs,
+        ]);
+        return user;
+    }
+
     /**
      * Update this User with the given presence event. May fire "User.presence",
      * "User.avatarUrl" and/or "User.displayName" if this event updates this user's

src/oidc/authorize.ts

@@ -198,7 +198,8 @@ const normalizeBearerTokenResponseTokenType = (response: SigninResponse): Bearer
  * @param code - authorization code as returned by OP during authorization
  * @param storedAuthorizationParams - stored params from start of oidc login flow
  * @returns valid bearer token response
- * @throws when request fails, or returned token response is invalid
+ * @throws An `Error` with `message` set to an entry in {@link OidcError},
+ *      when the request fails, or the returned token response is invalid.
  */
 export const completeAuthorizationCodeGrant = async (
     code: string,