v19.2.0

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

.eslintrc.js

@@ -52,6 +52,8 @@ module.exports = {
             "@typescript-eslint/no-explicit-any": "off",
             // We'd rather not do this but we do
             "@typescript-eslint/ban-ts-comment": "off",
+            // We're okay with assertion errors when we ask for them
+            "@typescript-eslint/no-non-null-assertion": "off",
 
             "quotes": "off",
             // We use a `logger` intermediary module

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,13 @@
-<!-- Please read https://github.com/matrix-org/matrix-js-sdk/blob/develop/CONTRIBUTING.md before submitting your pull request -->
+<!-- Thanks for submitting a PR! Please ensure the following requirements are met in order for us to review your PR -->
 
-<!-- Include a Sign-Off as described in https://github.com/matrix-org/matrix-js-sdk/blob/develop/CONTRIBUTING.md#sign-off -->
+## Checklist
 
-<!-- To specify text for the changelog entry (otherwise the PR title will be used):
-Notes:
+* [ ] Tests written for new code (and old code if feasible)
+* [ ] Linter and other CI checks pass
+* [ ] Sign-off given on the changes (see [CONTRIBUTING.md](https://github.com/matrix-org/matrix-js-sdk/blob/develop/CONTRIBUTING.md))
+
+<!--
+If you would like to specify text for the changelog entry other than your PR title, add the following:
+
+Notes: Add super cool feature
 -->

.github/workflows/jsdoc.yml

@@ -0,0 +1,53 @@
+name: Release Process
+on:
+  release:
+    types: [ published ]
+concurrency: ${{ github.workflow }}-${{ github.ref }}
+jobs:
+  jsdoc:
+    name: Publish Documentation
+    runs-on: ubuntu-latest
+    steps:
+      - name: 🧮 Checkout code
+        uses: actions/checkout@v3
+
+      - name: 🔧 Yarn cache
+        uses: actions/setup-node@v3
+        with:
+          cache: "yarn"
+
+      - name: 🔨 Install dependencies
+        run: "yarn install --pure-lockfile"
+
+      - name: 📖 Generate JSDoc
+        run: "yarn gendoc"
+
+      - name: 📋 Copy to temp
+        run: |
+          ls -lah
+          tag="${{ github.ref_name }}"
+          version="${tag#v}"
+          echo "VERSION=$version" >> $GITHUB_ENV
+          cp -a "./.jsdoc/matrix-js-sdk/$version" $RUNNER_TEMP
+
+      - name: 🧮 Checkout gh-pages
+        uses: actions/checkout@v3
+        with:
+          ref: gh-pages
+
+      - name: 🔪 Prepare
+        run: |
+          cp -a "$RUNNER_TEMP/$VERSION" .
+
+          # Add the new directory to the index if it isn't there already
+          if ! grep -q "Version $VERSION" index.html; then
+            perl -i -pe 'BEGIN {$rel=shift} $_ =~ /^<\/ul>/ && print
+              "<li><a href=\"${rel}/index.html\">Version ${rel}</a></li>\n"' "$VERSION" index.html
+          fi
+
+      - name: 🚀 Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          keep_files: true
+          publish_dir: .

.github/workflows/notify-downstream.yaml

@@ -5,6 +5,8 @@ on:
 concurrency: ${{ github.workflow }}-${{ github.ref }}
 jobs:
   notify-downstream:
+    # Only respect triggers from our develop branch, ignore that of forks
+    if: github.repository == 'matrix-org/matrix-js-sdk'
     continue-on-error: true
     strategy:
       fail-fast: false
@@ -18,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Notify matrix-react-sdk repo that a new SDK build is on develop so it can CI against it
-        uses: peter-evans/repository-dispatch@v1
+        uses: peter-evans/repository-dispatch@v2
         with:
           token: ${{ secrets.ELEMENT_BOT_TOKEN }}
           repository: ${{ matrix.repo }}

.github/workflows/pr_details.yml

@@ -1,59 +0,0 @@
-# Find details about the PR associated with this ref
-# Outputs:
-#  prnumber: the ID number of the associated PR
-#  headref: the name of the head branch of the PR
-#  baseref: the name of the base branch of the PR
-name: PR Details
-on:
-  workflow_call:
-    inputs:
-      owner:
-        type: string
-        required: true
-        description: The github username of the owner of the head branch
-      branch:
-        type: string
-        required: true
-        description: The name of the head branch
-    outputs:
-      pr_id:
-        description: The ID of the PR found
-        value: ${{ jobs.prdetails.outputs.pr_id }}
-      head_branch:
-        description: The head branch of the PR found
-        value: ${{ jobs.prdetails.outputs.head_branch }}
-      base_branch:
-        description: The base branch of the PR found
-        value: ${{ jobs.prdetails.outputs.base_branch }}
-
-jobs:
-  prdetails:
-    name: Find PR Details
-    runs-on: ubuntu-latest
-    steps:
-      - name: "🔍 Read PR details"
-        id: details
-        # We need to find the PR number that corresponds to the branch, which we do by searching the GH API
-        # The workflow_run event includes a list of pull requests, but it doesn't get populated for
-        # forked PRs: https://docs.github.com/en/rest/reference/checks#create-a-check-run
-        run: |
-          head_branch='${{ inputs.owner }}:${{ inputs.branch }}'
-          echo "Head branch: $head_branch"
-          pulls_uri="https://api.github.com/repos/${{ github.repository }}/pulls?head=$(jq -Rr '@uri' <<<$head_branch)"
-          pr_data=$(curl -s -H 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' "$pulls_uri")
-
-          pr_number=$(jq -r '.[] | .number' <<< "$pr_data")
-          echo "PR number: $pr_number"
-          echo "::set-output name=prnumber::$pr_number"
-
-          head_ref=$(jq -r '.[] | .head.ref' <<< "$pr_data")
-          echo "Head ref: $head_ref"
-          echo "::set-output name=headref::$head_ref"
-
-          base_ref=$(jq -r '.[] | .base.ref' <<< "$pr_data")
-          echo "Base ref: $base_ref"
-          echo "::set-output name=baseref::$base_ref"
-    outputs:
-      pr_id: ${{ steps.details.outputs.prnumber }}
-      head_branch: ${{ steps.details.outputs.headref }}
-      base_branch: ${{ steps.details.outputs.baseref }}

.github/workflows/pull_request.yaml

@@ -2,27 +2,90 @@ name: Pull Request
 on:
   pull_request_target:
     types: [ opened, edited, labeled, unlabeled, synchronize ]
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  workflow_call:
+    inputs:
+      labels:
+        type: string
+        default: "T-Defect,T-Deprecation,T-Enhancement,T-Task"
+        required: false
+        description: "No longer used, uses allchange logic now, will be removed at a later date"
+    secrets:
+      ELEMENT_BOT_TOKEN:
+        required: true
+concurrency: ${{ github.workflow }}-${{ github.event.pull_request.head.ref }}
 jobs:
   changelog:
     name: Preview Changelog
-    if: github.event.action != 'synchronize'
     runs-on: ubuntu-latest
     steps:
       - uses: matrix-org/allchange@main
         with:
           ghToken: ${{ secrets.GITHUB_TOKEN }}
+          requireLabel: true
 
-  enforce-label:
-    name: Enforce Labels
+  prevent-blocked:
+    name: Prevent Blocked
     runs-on: ubuntu-latest
     permissions:
       pull-requests: read
     steps:
-      - uses: yogevbd/enforce-label-action@2.1.0
+      - name: Add notice
+        uses: actions/github-script@v6
+        if: contains(github.event.pull_request.labels.*.name, 'X-Blocked')
         with:
-          REQUIRED_LABELS_ANY: "T-Defect,T-Deprecation,T-Enhancement,T-Task"
-          BANNED_LABELS: "X-Blocked"
-          BANNED_LABELS_DESCRIPTION: "Preventing merge whilst PR is marked blocked!"
+          script: |
+            core.setFailed("Preventing merge whilst PR is marked blocked!");
+
+  community-prs:
+    name: Label Community PRs
+    runs-on: ubuntu-latest
+    if: github.event.action == 'opened'
+    steps:
+      - name: Check membership
+        uses: tspascoal/get-user-teams-membership@v1
+        id: teams
+        with:
+          username: ${{ github.event.pull_request.user.login }}
+          organization: matrix-org
+          team: Core Team
+          GITHUB_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
+
+      - name: Add label
+        if: ${{ steps.teams.outputs.isTeamMember == 'false' }}
+        uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.addLabels({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: ['Z-Community-PR']
+            });
+
+  close-if-fork-develop:
+    name: Forbid develop branch fork contributions
+    runs-on: ubuntu-latest
+    if: >
+      github.event.action == 'opened' &&
+      github.event.pull_request.head.ref == 'develop' &&
+      github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - name: Close pull request
+        uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: "Thanks for opening this pull request, unfortunately we do not accept contributions from the main" +
+                " branch of your fork, please re-open once you switch to an alternative branch for everyone's sanity." +
+                " See https://github.com/matrix-org/matrix-js-sdk/blob/develop/CONTRIBUTING.md",
+            });
+
+            github.rest.pulls.update({
+              pull_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: 'closed'
+            });

.github/workflows/sonarcloud.yml

@@ -1,91 +1,24 @@
+# Must only be called from a workflow_run in the context of the upstream repo
 name: SonarCloud
 on:
   workflow_call:
-    inputs:
-      repo:
-        type: string
-        required: true
-        description: The full name of the repo in org/repo format
-      head_branch:
-        type: string
-        required: true
-        description: The name of the head branch
-      # We cannot use ${{ github.sha }} here as for pull requests it'll be a simulated merge commit instead
-      revision:
-        type: string
-        required: true
-        description: The git revision with which this sonar run should be associated
-
-      # Coverage specific parameters, assumes coverage reports live in a /coverage/ directory
-      coverage_workflow_name:
-        type: string
-        required: false
-        description: The name of the workflow which uploaded the `coverage` artifact, if any
-      coverage_run_id:
-        type: string
-        required: false
-        description: The run_id of the workflow which upload the coverage relevant to this run
-
-      # PR specific parameters
-      pr_id:
-        type: string
-        required: false
-        description: The ID number of the PR if this workflow is being triggered due to one
-      base_branch:
-        type: string
-        required: false
-        description: The base branch of the PR if this workflow is being triggered due to one
-
-      # Org specific parameters
-      main_branch:
-        type: string
-        required: false
-        description: The default branch of the repository
-        default: "develop"
     secrets:
       SONAR_TOKEN:
         required: true
 jobs:
-  analysis:
-    name: Analysis
+  sonarqube:
     runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion == 'success'
     steps:
-      - name: "🧮 Checkout code"
-        uses: actions/checkout@v3
-        with:
-          repository: ${{ inputs.repo }}
-          ref: ${{ inputs.head_branch }} # checkout commit that triggered this workflow
-          fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
-
-      # Fetch develop so that Sonar can identify new issues in PR builds
-      - name: "📕 Fetch ${{ inputs.main_branch }}"
-        if: inputs.head_branch != inputs.main_branch
-        run: git rev-parse HEAD && git fetch origin ${{ inputs.main_branch }}:${{ inputs.main_branch }} && git status && git rev-parse HEAD
-
-      # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
-      # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
-      - name: "📥 Download Coverage Report"
-        uses: dawidd6/action-download-artifact@v2
-        if: inputs.coverage_workflow_name
-        with:
-          workflow: ${{ inputs.coverage_workflow_name }}
-          run_id: ${{ inputs.coverage_run_id }}
-          name: coverage
-          path: coverage
-
-      - name: "🔍 Read package.json version"
-        id: version
-        uses: martinbeentjes/npm-get-version-action@main
-
       - name: "🩻 SonarCloud Scan"
-        uses: SonarSource/sonarcloud-github-action@master
+        uses: matrix-org/sonarcloud-workflow-action@v2.2
         with:
-          args: >
-            -Dsonar.projectVersion=${{ steps.version.outputs.current-version }}
-            -Dsonar.scm.revision=${{ inputs.revision }}
-            -Dsonar.pullrequest.key=${{ inputs.pr_id }}
-            -Dsonar.pullrequest.branch=${{ inputs.pr_id && inputs.head_branch }}
-            -Dsonar.pullrequest.base=${{ inputs.pr_id && inputs.base_branch }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          repository: ${{ github.event.workflow_run.head_repository.full_name }}
+          is_pr: ${{ github.event.workflow_run.event == 'pull_request' }}
+          version_cmd: 'cat package.json | jq -r .version'
+          branch: ${{ github.event.workflow_run.head_branch }}
+          revision: ${{ github.event.workflow_run.head_sha }}
+          token: ${{ secrets.SONAR_TOKEN }}
+          coverage_run_id: ${{ github.event.workflow_run.id }}
+          coverage_workflow_name: tests.yml
+          coverage_extract_path: coverage

.github/workflows/sonarqube.yml

@@ -5,33 +5,11 @@ on:
     types:
       - completed
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch }}
   cancel-in-progress: true
 jobs:
-  prdetails:
-    name: ℹ️ PR Details
-    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
-    uses: matrix-org/matrix-js-sdk/.github/workflows/pr_details.yml@develop
-    with:
-      owner: ${{ github.event.workflow_run.head_repository.owner.login }}
-      branch: ${{ github.event.workflow_run.head_branch }}
-
   sonarqube:
     name: 🩻 SonarQube
-    needs: prdetails
-    # Only wait for prdetails if it isn't skipped
-    if: |
-      always() &&
-      (needs.prdetails.result == 'success' || needs.prdetails.result == 'skipped') &&
-      github.event.workflow_run.conclusion == 'success'
     uses: matrix-org/matrix-js-sdk/.github/workflows/sonarcloud.yml@develop
-    with:
-      repo: ${{ github.event.workflow_run.head_repository.full_name }}
-      pr_id: ${{ needs.prdetails.outputs.pr_id }}
-      head_branch: ${{ needs.prdetails.outputs.head_branch || github.event.workflow_run.head_branch }}
-      base_branch: ${{ needs.prdetails.outputs.base_branch }}
-      revision: ${{ github.event.workflow_run.head_sha }}
-      coverage_workflow_name: tests.yml
-      coverage_run_id: ${{ github.event.workflow_run.id }}
     secrets:
       SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.github/workflows/static_analysis.yml

@@ -11,7 +11,7 @@ jobs:
     name: "Typescript Syntax Check"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - uses: actions/setup-node@v3
         with:
@@ -27,7 +27,7 @@ jobs:
     name: "ESLint"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - uses: actions/setup-node@v3
         with:
@@ -43,7 +43,7 @@ jobs:
     name: "JSDoc Checker"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - uses: actions/setup-node@v3
         with:

.github/workflows/tests.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Yarn cache
         uses: actions/setup-node@v3
@@ -26,10 +26,10 @@ jobs:
         run: "yarn build"
 
       - name: Run tests with coverage
-        run: "yarn coverage --ci"
+        run: "yarn coverage --ci --reporters github-actions"
 
       - name: Upload Artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: coverage
           path: |

.istanbul.yml

@@ -1,2 +0,0 @@
-instrumentation:
-  compact: false

CHANGELOG.md

@@ -1,3 +1,77 @@
+Changes in [19.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v19.2.0) (2022-08-02)
+==================================================================================================
+
+## 🦖 Deprecations
+ * Remove unstable support for `m.room_key.withheld` ([\#2512](https://github.com/matrix-org/matrix-js-sdk/pull/2512)). Fixes #2233.
+
+## ✨ Features
+ * Sliding sync: add missing filters from latest MSC ([\#2555](https://github.com/matrix-org/matrix-js-sdk/pull/2555)).
+ * Use stable prefixes for MSC3827 ([\#2537](https://github.com/matrix-org/matrix-js-sdk/pull/2537)).
+ * Add support for MSC3575: Sliding Sync ([\#2242](https://github.com/matrix-org/matrix-js-sdk/pull/2242)).
+
+## 🐛 Bug Fixes
+ * Correct the units in TURN servers expiry documentation ([\#2520](https://github.com/matrix-org/matrix-js-sdk/pull/2520)).
+ * Re-insert room IDs when decrypting bundled redaction events returned by `/sync` ([\#2531](https://github.com/matrix-org/matrix-js-sdk/pull/2531)). Contributed by @duxovni.
+
+Changes in [19.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v19.1.0) (2022-07-26)
+==================================================================================================
+
+## 🦖 Deprecations
+ * Remove MSC3244 support ([\#2504](https://github.com/matrix-org/matrix-js-sdk/pull/2504)).
+
+## ✨ Features
+ * `room` now exports `KNOWN_SAFE_ROOM_VERSION` ([\#2474](https://github.com/matrix-org/matrix-js-sdk/pull/2474)).
+
+## 🐛 Bug Fixes
+ * Don't crash with undefined room in `processBeaconEvents()` ([\#2500](https://github.com/matrix-org/matrix-js-sdk/pull/2500)). Fixes #2494.
+ * Properly re-insert room ID in bundled thread relation messages from sync ([\#2505](https://github.com/matrix-org/matrix-js-sdk/pull/2505)). Fixes vector-im/element-web#22094. Contributed by @duxovni.
+ * Actually store the identity server in the client when given as an option ([\#2503](https://github.com/matrix-org/matrix-js-sdk/pull/2503)). Fixes vector-im/element-web#22757.
+ * Fix call.collectCallStats() ([\#2480](https://github.com/matrix-org/matrix-js-sdk/pull/2480)).
+
+Changes in [19.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v19.0.0) (2022-07-05)
+==================================================================================================
+
+## 🚨 BREAKING CHANGES
+ * Remove unused sessionStore ([\#2455](https://github.com/matrix-org/matrix-js-sdk/pull/2455)).
+
+## ✨ Features
+ * Implement MSC3827: Filtering of `/publicRooms` by room type ([\#2469](https://github.com/matrix-org/matrix-js-sdk/pull/2469)).
+ * expose latestLocationEvent on beacon model ([\#2467](https://github.com/matrix-org/matrix-js-sdk/pull/2467)). Contributed by @kerryarchibald.
+ * Live location share - add start time leniency ([\#2465](https://github.com/matrix-org/matrix-js-sdk/pull/2465)). Contributed by @kerryarchibald.
+ * Log real errors and not just their messages, traces are useful ([\#2464](https://github.com/matrix-org/matrix-js-sdk/pull/2464)).
+ * Various changes to `src/crypto` files for correctness ([\#2137](https://github.com/matrix-org/matrix-js-sdk/pull/2137)). Contributed by @ShadowJonathan.
+ * Update MSC3786 implementation: Check the `state_key` ([\#2429](https://github.com/matrix-org/matrix-js-sdk/pull/2429)).
+ * Timeline needs to refresh when we see a MSC2716 marker event  ([\#2299](https://github.com/matrix-org/matrix-js-sdk/pull/2299)). Contributed by @MadLittleMods.
+ * Try to load keys from key backup when a message fails to decrypt ([\#2373](https://github.com/matrix-org/matrix-js-sdk/pull/2373)). Fixes vector-im/element-web#21026. Contributed by @duxovni.
+
+## 🐛 Bug Fixes
+ * Send call version `1` as a string ([\#2471](https://github.com/matrix-org/matrix-js-sdk/pull/2471)). Fixes vector-im/element-web#22629.
+ * Fix issue with `getEventTimeline` returning undefined for thread roots in main timeline ([\#2454](https://github.com/matrix-org/matrix-js-sdk/pull/2454)). Fixes vector-im/element-web#22539.
+ * Add missing `type` property on `IAuthData` ([\#2463](https://github.com/matrix-org/matrix-js-sdk/pull/2463)).
+ * Clearly indicate that `lastReply` on a Thread can return falsy ([\#2462](https://github.com/matrix-org/matrix-js-sdk/pull/2462)).
+ * Fix issues with getEventTimeline and thread roots ([\#2444](https://github.com/matrix-org/matrix-js-sdk/pull/2444)). Fixes vector-im/element-web#21613.
+ * Live location sharing - monitor liveness of beacons yet to start ([\#2437](https://github.com/matrix-org/matrix-js-sdk/pull/2437)). Contributed by @kerryarchibald.
+ * Refactor Relations to not be per-EventTimelineSet ([\#2412](https://github.com/matrix-org/matrix-js-sdk/pull/2412)). Fixes #2399 and vector-im/element-web#22298.
+ * Add tests for sendEvent threadId handling ([\#2435](https://github.com/matrix-org/matrix-js-sdk/pull/2435)). Fixes vector-im/element-web#22433.
+ * Make sure `encryptAndSendKeysToDevices` assumes devices are unique per-user. ([\#2136](https://github.com/matrix-org/matrix-js-sdk/pull/2136)). Fixes #2135. Contributed by @ShadowJonathan.
+ * Don't bug the user while re-checking key backups after decryption failures ([\#2430](https://github.com/matrix-org/matrix-js-sdk/pull/2430)). Fixes vector-im/element-web#22416. Contributed by @duxovni.
+
+Changes in [18.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v18.1.0) (2022-06-07)
+==================================================================================================
+
+## ✨ Features
+ * Convert `getLocalAliases` to a stable API call ([\#2402](https://github.com/matrix-org/matrix-js-sdk/pull/2402)).
+
+## 🐛 Bug Fixes
+ * Fix request, crypto, and bs58 imports ([\#2414](https://github.com/matrix-org/matrix-js-sdk/pull/2414)). Fixes #2415.
+ * Update relations after every decryption attempt ([\#2387](https://github.com/matrix-org/matrix-js-sdk/pull/2387)). Fixes vector-im/element-web#22258. Contributed by @weeman1337.
+ * Fix degraded mode for the IDBStore and test it ([\#2400](https://github.com/matrix-org/matrix-js-sdk/pull/2400)). Fixes matrix-org/element-web-rageshakes#13170.
+ * Don't cancel SAS verifications if `ready` is received after `start` ([\#2250](https://github.com/matrix-org/matrix-js-sdk/pull/2250)).
+ * Prevent overlapping sync accumulator persists ([\#2392](https://github.com/matrix-org/matrix-js-sdk/pull/2392)). Fixes vector-im/element-web#21541.
+ * Fix behaviour of isRelation with relation m.replace for state events ([\#2389](https://github.com/matrix-org/matrix-js-sdk/pull/2389)). Fixes vector-im/element-web#22280.
+ * Fixes #2384 ([\#2385](https://github.com/matrix-org/matrix-js-sdk/pull/2385)). Fixes undefined/matrix-js-sdk#2384. Contributed by @schmop.
+ * Ensure rooms are recalculated on re-invites ([\#2374](https://github.com/matrix-org/matrix-js-sdk/pull/2374)). Fixes vector-im/element-web#22106.
+
 Changes in [18.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v18.0.0) (2022-05-24)
 ==================================================================================================
 

CONTRIBUTING.md

@@ -23,7 +23,18 @@ Things that should go into your PR description:
  * A changelog entry in the `Notes` section (see below)
  * References to any bugs fixed by the change (in GitHub's `Fixes` notation)
  * Describe the why and what is changing in the PR description so it's easy for
-   onlookers and reviewers to onboard and context switch.
+   onlookers and reviewers to onboard and context switch. This information is
+   also helpful when we come back to look at this in 6 months and ask "why did
+   we do it like that?" we have a chance of finding out.
+      * Why didn't it work before? Why does it work now? What use cases does it
+        unlock?
+      * If you find yourself adding information on how the code works or why you
+        chose to do it the way you did, make sure this information is instead
+        written as comments in the code itself.
+      * Sometimes a PR can change considerably as it is developed. In this case,
+        the description should be updated to reflect the most recent state of
+        the PR. (It can be helpful to retain the old content under a suitable
+        heading, for additional context.)
  * Include both **before** and **after** screenshots to easily compare and discuss
    what's changing.
  * Include a step-by-step testing strategy so that a reviewer can check out the
@@ -31,11 +42,6 @@ Things that should go into your PR description:
  * Add comments to the diff for the reviewer that might help them to understand
    why the change is necessary or how they might better understand and review it.
 
-Things that should *not* go into your PR description:
- * Any information on how the code works or why you chose to do it the way
-   you did. If this isn't obvious from your code, you haven't written enough
-   comments.
-
 We rely on information in pull request to populate the information that goes
 into the changelogs our users see, both for the JS SDK itself and also for some
 projects based on it. This is picked up from both labels on the pull request and
@@ -129,6 +135,16 @@ When writing unit tests, please aim for a high level of test coverage
 for new code - 80% or greater. If you cannot achieve that, please document
 why it's not possible in your PR.
 
+Some sections of code are not sensible to add coverage for, such as those
+which explicitly inhibit noisy logging for tests. Which can be hidden using
+an istanbul magic comment as [documented here][1]. See example:
+```javascript
+/* istanbul ignore if */
+if (process.env.NODE_ENV !== "test") {
+    logger.error("Log line that is noisy enough in tests to want to skip");
+}
+```
+
 Tests validate that your change works as intended and also document
 concisely what is being changed. Ideally, your new tests fail
 prior to your change, and succeed once it has been applied. You may
@@ -244,14 +260,25 @@ on Git 2.17+ you can mass signoff using rebase:
 git rebase --signoff origin/develop
 ```
 
+Review expectations
+===================
+
+See https://github.com/vector-im/element-meta/wiki/Review-process
+
+
 Merge Strategy
 ==============
 
 The preferred method for merging pull requests is squash merging to keep the
 commit history trim, but it is up to the discretion of the team member merging
-the change. When stacking pull requests, you may wish to do the following:
+the change. We do not support rebase merges due to `allchange` being unable to
+handle them. When merging make sure to leave the default commit title, or 
+at least leave the PR number at the end in brackets like by default.
+When stacking pull requests, you may wish to do the following:
 
 1. Branch from develop to your branch (branch1), push commits onto it and open a pull request
 2. Branch from your base branch (branch1) to your work branch (branch2), push commits and open a pull request configuring the base to be branch1, saying in the description that it is based on your other PR.
 3. Merge the first PR using a merge commit otherwise your stacked PR will need a rebase. Github will automatically adjust the base branch of your other PR to be develop.
 
+
+[1]: https://github.com/gotwarlost/istanbul/blob/master/ignoring-code-for-coverage.md

package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-js-sdk",
-  "version": "18.0.0",
+  "version": "19.2.0",
   "description": "Matrix Client-Server SDK for Javascript",
   "engines": {
     "node": ">=12.9.0"
@@ -56,11 +56,11 @@
     "@babel/runtime": "^7.12.5",
     "another-json": "^0.2.0",
     "browser-request": "^0.3.3",
-    "bs58": "^4.0.1",
+    "bs58": "^5.0.0",
     "content-type": "^1.0.4",
     "loglevel": "^1.7.1",
     "matrix-events-sdk": "^0.0.1-beta.7",
-    "p-retry": "^4.5.0",
+    "p-retry": "4",
     "qs": "^6.9.6",
     "request": "^2.88.2",
     "unhomoglyph": "^1.0.6"
@@ -78,31 +78,31 @@
     "@babel/preset-env": "^7.12.11",
     "@babel/preset-typescript": "^7.12.7",
     "@babel/register": "^7.12.10",
-    "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz",
+    "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.12.tgz",
     "@types/bs58": "^4.0.1",
     "@types/content-type": "^1.1.5",
-    "@types/jest": "^26.0.20",
-    "@types/node": "12",
+    "@types/jest": "^28.0.0",
+    "@types/node": "16",
     "@types/request": "^2.48.5",
     "@typescript-eslint/eslint-plugin": "^5.6.0",
     "@typescript-eslint/parser": "^5.6.0",
     "allchange": "^1.0.6",
-    "babel-jest": "^26.6.3",
+    "babel-jest": "^28.0.0",
     "babelify": "^10.0.0",
     "better-docs": "^2.4.0-beta.9",
     "browserify": "^17.0.0",
     "docdash": "^1.2.0",
-    "eslint": "8.9.0",
+    "eslint": "8.19.0",
     "eslint-config-google": "^0.14.0",
     "eslint-plugin-import": "^2.25.4",
-    "eslint-plugin-matrix-org": "^0.4.0",
-    "exorcist": "^1.0.1",
-    "fake-indexeddb": "^3.1.2",
-    "jest": "^26.6.3",
+    "eslint-plugin-matrix-org": "^0.5.0",
+    "exorcist": "^2.0.0",
+    "fake-indexeddb": "^4.0.0",
+    "jest": "^28.0.0",
     "jest-localstorage-mock": "^2.4.6",
     "jest-sonar-reporter": "^2.0.0",
     "jsdoc": "^3.6.6",
-    "matrix-mock-request": "^1.2.3",
+    "matrix-mock-request": "^2.1.0",
     "rimraf": "^3.0.2",
     "terser": "^5.5.1",
     "tsify": "^5.0.2",

release.sh

@@ -29,7 +29,7 @@ fi
 npm --version > /dev/null || (echo "npm is required: please install it"; kill $$)
 yarn --version > /dev/null || (echo "yarn is required: please install it"; kill $$)
 
-USAGE="$0 [-xz] [-c changelog_file] vX.Y.Z"
+USAGE="$0 [-x] [-c changelog_file] vX.Y.Z"
 
 help() {
     cat <<EOF
@@ -37,7 +37,6 @@ $USAGE
 
     -c changelog_file:  specify name of file containing changelog
     -x:                 skip updating the changelog
-    -z:                 skip generating the jsdoc
     -n:                 skip publish to NPM
 EOF
 }
@@ -60,7 +59,6 @@ if ! git diff-files --quiet; then
 fi
 
 skip_changelog=
-skip_jsdoc=
 skip_npm=
 changelog_file="CHANGELOG.md"
 expected_npm_user="matrixdotorg"
@@ -76,9 +74,6 @@ while getopts hc:u:xzn f; do
         x)
             skip_changelog=1
             ;;
-        z)
-            skip_jsdoc=1
-            ;;
         n)
             skip_npm=1
             ;;
@@ -128,12 +123,12 @@ if [ $prerelease -eq 1 ]; then
     echo Making a PRE-RELEASE
 fi
 
-# we might already be on the release branch, in which case, yay
-# If we're on any branch starting with 'release', we don't create
-# a separate release branch (this allows us to use the same
+# We might already be on the release branch, in which case, yay
+# If we're on any branch starting with 'release', or the staging branch
+# we don't create a separate release branch (this allows us to use the same
 # release branch for releases and release candidates).
 curbranch=$(git symbolic-ref --short HEAD)
-if [[ "$curbranch" != release* ]]; then
+if [[ "$curbranch" != release* && "$curbranch" != "staging" ]]; then
     echo "Creating release branch"
     git checkout -b "$rel_branch"
 else
@@ -326,22 +321,6 @@ if [ -z "$skip_npm" ]; then
     fi
 fi
 
-if [ -z "$skip_jsdoc" ]; then
-    echo "generating jsdocs"
-    yarn gendoc
-
-    echo "copying jsdocs to gh-pages branch"
-    git checkout gh-pages
-    git pull
-    cp -a ".jsdoc/matrix-js-sdk/$release" .
-    perl -i -pe 'BEGIN {$rel=shift} $_ =~ /^<\/ul>/ && print
-        "<li><a href=\"${rel}/index.html\">Version ${rel}</a></li>\n"' \
-        $release index.html
-    git add "$release"
-    git commit --no-verify -m "Add jsdoc for $release" index.html "$release"
-    git push origin gh-pages
-fi
-
 # if it is a pre-release, leave it on the release branch for now.
 if [ $prerelease -eq 1 ]; then
     git checkout "$rel_branch"

renovate.json

@@ -0,0 +1,16 @@
+{
+    "extends": [
+        "config:base",
+        ":dependencyDashboardApproval"
+    ],
+    "labels": ["T-Task", "Dependencies"],
+    "lockFileMaintenance": { "enabled": true },
+    "groupName": "all",
+    "packageRules": [{
+        "matchFiles": ["package.json"],
+        "rangeStrategy": "update-lockfile"
+    }],
+    "platformAutomerge": true,
+    "automerge": true,
+    "automergeType": "pr"
+}

spec/TestClient.js

@@ -1,238 +0,0 @@
-/*
-Copyright 2016 OpenMarket Ltd
-Copyright 2017 Vector Creations Ltd
-Copyright 2018-2019 New Vector Ltd
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// load olm before the sdk if possible
-import './olm-loader';
-
-import MockHttpBackend from 'matrix-mock-request';
-
-import { LocalStorageCryptoStore } from '../src/crypto/store/localStorage-crypto-store';
-import { logger } from '../src/logger';
-import { WebStorageSessionStore } from "../src/store/session/webstorage";
-import { syncPromise } from "./test-utils/test-utils";
-import { createClient } from "../src/matrix";
-import { MockStorageApi } from "./MockStorageApi";
-
-/**
- * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
- *
- * @constructor
- * @param {string} userId
- * @param {string} deviceId
- * @param {string} accessToken
- *
- * @param {WebStorage=} sessionStoreBackend a web storage object to use for the
- *     session store. If undefined, we will create a MockStorageApi.
- * @param {object} options additional options to pass to the client
- */
-export function TestClient(
-    userId, deviceId, accessToken, sessionStoreBackend, options,
-) {
-    this.userId = userId;
-    this.deviceId = deviceId;
-
-    if (sessionStoreBackend === undefined) {
-        sessionStoreBackend = new MockStorageApi();
-    }
-    const sessionStore = new WebStorageSessionStore(sessionStoreBackend);
-
-    this.httpBackend = new MockHttpBackend();
-
-    options = Object.assign({
-        baseUrl: "http://" + userId + ".test.server",
-        userId: userId,
-        accessToken: accessToken,
-        deviceId: deviceId,
-        sessionStore: sessionStore,
-        request: this.httpBackend.requestFn,
-    }, options);
-    if (!options.cryptoStore) {
-        // expose this so the tests can get to it
-        this.cryptoStore = new LocalStorageCryptoStore(sessionStoreBackend);
-        options.cryptoStore = this.cryptoStore;
-    }
-    this.client = createClient(options);
-
-    this.deviceKeys = null;
-    this.oneTimeKeys = {};
-    this.callEventHandler = {
-        calls: new Map(),
-    };
-}
-
-TestClient.prototype.toString = function() {
-    return 'TestClient[' + this.userId + ']';
-};
-
-/**
- * start the client, and wait for it to initialise.
- *
- * @return {Promise}
- */
-TestClient.prototype.start = function() {
-    logger.log(this + ': starting');
-    this.httpBackend.when("GET", "/versions").respond(200, {});
-    this.httpBackend.when("GET", "/pushrules").respond(200, {});
-    this.httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
-    this.expectDeviceKeyUpload();
-
-    // we let the client do a very basic initial sync, which it needs before
-    // it will upload one-time keys.
-    this.httpBackend.when("GET", "/sync").respond(200, { next_batch: 1 });
-
-    this.client.startClient({
-        // set this so that we can get hold of failed events
-        pendingEventOrdering: 'detached',
-    });
-
-    return Promise.all([
-        this.httpBackend.flushAllExpected(),
-        syncPromise(this.client),
-    ]).then(() => {
-        logger.log(this + ': started');
-    });
-};
-
-/**
- * stop the client
- * @return {Promise} Resolves once the mock http backend has finished all pending flushes
- */
-TestClient.prototype.stop = function() {
-    this.client.stopClient();
-    return this.httpBackend.stop();
-};
-
-/**
- * Set up expectations that the client will upload device keys.
- */
-TestClient.prototype.expectDeviceKeyUpload = function() {
-    const self = this;
-    this.httpBackend.when("POST", "/keys/upload").respond(200, function(path, content) {
-        expect(content.one_time_keys).toBe(undefined);
-        expect(content.device_keys).toBeTruthy();
-
-        logger.log(self + ': received device keys');
-        // we expect this to happen before any one-time keys are uploaded.
-        expect(Object.keys(self.oneTimeKeys).length).toEqual(0);
-
-        self.deviceKeys = content.device_keys;
-        return { one_time_key_counts: { signed_curve25519: 0 } };
-    });
-};
-
-/**
- * If one-time keys have already been uploaded, return them. Otherwise,
- * set up an expectation that the keys will be uploaded, and wait for
- * that to happen.
- *
- * @returns {Promise} for the one-time keys
- */
-TestClient.prototype.awaitOneTimeKeyUpload = function() {
-    if (Object.keys(this.oneTimeKeys).length != 0) {
-        // already got one-time keys
-        return Promise.resolve(this.oneTimeKeys);
-    }
-
-    this.httpBackend.when("POST", "/keys/upload")
-        .respond(200, (path, content) => {
-            expect(content.device_keys).toBe(undefined);
-            expect(content.one_time_keys).toBe(undefined);
-            return { one_time_key_counts: {
-                signed_curve25519: Object.keys(this.oneTimeKeys).length,
-            } };
-        });
-
-    this.httpBackend.when("POST", "/keys/upload")
-          .respond(200, (path, content) => {
-              expect(content.device_keys).toBe(undefined);
-              expect(content.one_time_keys).toBeTruthy();
-              expect(content.one_time_keys).not.toEqual({});
-              logger.log('%s: received %i one-time keys', this,
-                          Object.keys(content.one_time_keys).length);
-              this.oneTimeKeys = content.one_time_keys;
-              return { one_time_key_counts: {
-                  signed_curve25519: Object.keys(this.oneTimeKeys).length,
-              } };
-          });
-
-    // this can take ages
-    return this.httpBackend.flush('/keys/upload', 2, 1000).then((flushed) => {
-        expect(flushed).toEqual(2);
-        return this.oneTimeKeys;
-    });
-};
-
-/**
- * Set up expectations that the client will query device keys.
- *
- * We check that the query contains each of the users in `response`.
- *
- * @param {Object} response   response to the query.
- */
-TestClient.prototype.expectKeyQuery = function(response) {
-    this.httpBackend.when('POST', '/keys/query').respond(
-        200, (path, content) => {
-            Object.keys(response.device_keys).forEach((userId) => {
-                expect(content.device_keys[userId]).toEqual(
-                    [],
-                    "Expected key query for " + userId + ", got " +
-                    Object.keys(content.device_keys),
-                );
-            });
-            return response;
-        });
-};
-
-/**
- * get the uploaded curve25519 device key
- *
- * @return {string} base64 device key
- */
-TestClient.prototype.getDeviceKey = function() {
-    const keyId = 'curve25519:' + this.deviceId;
-    return this.deviceKeys.keys[keyId];
-};
-
-/**
- * get the uploaded ed25519 device key
- *
- * @return {string} base64 device key
- */
-TestClient.prototype.getSigningKey = function() {
-    const keyId = 'ed25519:' + this.deviceId;
-    return this.deviceKeys.keys[keyId];
-};
-
-/**
- * flush a single /sync request, and wait for the syncing event
- *
- * @returns {Promise} promise which completes once the sync has been flushed
- */
-TestClient.prototype.flushSync = function() {
-    logger.log(`${this}: flushSync`);
-    return Promise.all([
-        this.httpBackend.flush('/sync', 1),
-        syncPromise(this.client),
-    ]).then(() => {
-        logger.log(`${this}: flushSync completed`);
-    });
-};
-
-TestClient.prototype.isFallbackICEServerAllowed = function() {
-    return true;
-};

spec/TestClient.ts

@@ -0,0 +1,243 @@
+/*
+Copyright 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+Copyright 2018-2019 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// load olm before the sdk if possible
+import './olm-loader';
+
+import MockHttpBackend from 'matrix-mock-request';
+
+import { LocalStorageCryptoStore } from '../src/crypto/store/localStorage-crypto-store';
+import { logger } from '../src/logger';
+import { syncPromise } from "./test-utils/test-utils";
+import { createClient } from "../src/matrix";
+import { ICreateClientOpts, IDownloadKeyResult, MatrixClient, PendingEventOrdering } from "../src/client";
+import { MockStorageApi } from "./MockStorageApi";
+import { encodeUri } from "../src/utils";
+import { IDeviceKeys, IOneTimeKey } from "../src/crypto/dehydration";
+import { IKeyBackupSession } from "../src/crypto/keybackup";
+import { IHttpOpts } from "../src/http-api";
+import { IKeysUploadResponse, IUploadKeysRequest } from '../src/client';
+
+/**
+ * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
+ */
+export class TestClient {
+    public readonly httpBackend: MockHttpBackend;
+    public readonly client: MatrixClient;
+    public deviceKeys: IDeviceKeys;
+    public oneTimeKeys: Record<string, IOneTimeKey>;
+
+    constructor(
+        public readonly userId?: string,
+        public readonly deviceId?: string,
+        accessToken?: string,
+        sessionStoreBackend?: Storage,
+        options?: Partial<ICreateClientOpts>,
+    ) {
+        if (sessionStoreBackend === undefined) {
+            sessionStoreBackend = new MockStorageApi();
+        }
+
+        this.httpBackend = new MockHttpBackend();
+
+        const fullOptions: ICreateClientOpts = {
+            baseUrl: "http://" + userId + ".test.server",
+            userId: userId,
+            accessToken: accessToken,
+            deviceId: deviceId,
+            request: this.httpBackend.requestFn as IHttpOpts["request"],
+            ...options,
+        };
+        if (!fullOptions.cryptoStore) {
+            // expose this so the tests can get to it
+            fullOptions.cryptoStore = new LocalStorageCryptoStore(sessionStoreBackend);
+        }
+        this.client = createClient(fullOptions);
+
+        this.deviceKeys = null;
+        this.oneTimeKeys = {};
+    }
+
+    public toString(): string {
+        return 'TestClient[' + this.userId + ']';
+    }
+
+    /**
+     * start the client, and wait for it to initialise.
+     */
+    public start(): Promise<void> {
+        logger.log(this + ': starting');
+        this.httpBackend.when("GET", "/versions").respond(200, {});
+        this.httpBackend.when("GET", "/pushrules").respond(200, {});
+        this.httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+        this.expectDeviceKeyUpload();
+
+        // we let the client do a very basic initial sync, which it needs before
+        // it will upload one-time keys.
+        this.httpBackend.when("GET", "/sync").respond(200, { next_batch: 1 });
+
+        this.client.startClient({
+            // set this so that we can get hold of failed events
+            pendingEventOrdering: PendingEventOrdering.Detached,
+        });
+
+        return Promise.all([
+            this.httpBackend.flushAllExpected(),
+            syncPromise(this.client),
+        ]).then(() => {
+            logger.log(this + ': started');
+        });
+    }
+
+    /**
+     * stop the client
+     * @return {Promise} Resolves once the mock http backend has finished all pending flushes
+     */
+    public async stop(): Promise<void> {
+        this.client.stopClient();
+        await this.httpBackend.stop();
+    }
+
+    /**
+     * Set up expectations that the client will upload device keys.
+     */
+    public expectDeviceKeyUpload() {
+        this.httpBackend.when("POST", "/keys/upload")
+            .respond<IKeysUploadResponse, IUploadKeysRequest>(200, (_path, content) => {
+                expect(content.one_time_keys).toBe(undefined);
+                expect(content.device_keys).toBeTruthy();
+
+                logger.log(this + ': received device keys');
+                // we expect this to happen before any one-time keys are uploaded.
+                expect(Object.keys(this.oneTimeKeys).length).toEqual(0);
+
+                this.deviceKeys = content.device_keys;
+                return { one_time_key_counts: { signed_curve25519: 0 } };
+            });
+    }
+
+    /**
+     * If one-time keys have already been uploaded, return them. Otherwise,
+     * set up an expectation that the keys will be uploaded, and wait for
+     * that to happen.
+     *
+     * @returns {Promise} for the one-time keys
+     */
+    public awaitOneTimeKeyUpload(): Promise<Record<string, IOneTimeKey>> {
+        if (Object.keys(this.oneTimeKeys).length != 0) {
+            // already got one-time keys
+            return Promise.resolve(this.oneTimeKeys);
+        }
+
+        this.httpBackend.when("POST", "/keys/upload")
+            .respond<IKeysUploadResponse, IUploadKeysRequest>(200, (_path, content: IUploadKeysRequest) => {
+                expect(content.device_keys).toBe(undefined);
+                expect(content.one_time_keys).toBe(undefined);
+                return { one_time_key_counts: {
+                    signed_curve25519: Object.keys(this.oneTimeKeys).length,
+                } };
+            });
+
+        this.httpBackend.when("POST", "/keys/upload")
+            .respond<IKeysUploadResponse, IUploadKeysRequest>(200, (_path, content: IUploadKeysRequest) => {
+                expect(content.device_keys).toBe(undefined);
+                expect(content.one_time_keys).toBeTruthy();
+                expect(content.one_time_keys).not.toEqual({});
+                logger.log('%s: received %i one-time keys', this,
+                    Object.keys(content.one_time_keys).length);
+                this.oneTimeKeys = content.one_time_keys;
+                return { one_time_key_counts: {
+                    signed_curve25519: Object.keys(this.oneTimeKeys).length,
+                } };
+            });
+
+        // this can take ages
+        return this.httpBackend.flush('/keys/upload', 2, 1000).then((flushed) => {
+            expect(flushed).toEqual(2);
+            return this.oneTimeKeys;
+        });
+    }
+
+    /**
+     * Set up expectations that the client will query device keys.
+     *
+     * We check that the query contains each of the users in `response`.
+     *
+     * @param {Object} response   response to the query.
+     */
+    public expectKeyQuery(response: IDownloadKeyResult) {
+        this.httpBackend.when('POST', '/keys/query').respond<IDownloadKeyResult>(
+            200, (_path, content) => {
+                Object.keys(response.device_keys).forEach((userId) => {
+                    expect(content.device_keys[userId]).toEqual([]);
+                });
+                return response;
+            });
+    }
+
+    /**
+     * Set up expectations that the client will query key backups for a particular session
+     */
+    public expectKeyBackupQuery(roomId: string, sessionId: string, status: number, response: IKeyBackupSession) {
+        this.httpBackend.when('GET', encodeUri("/room_keys/keys/$roomId/$sessionId", {
+            $roomId: roomId,
+            $sessionId: sessionId,
+        })).respond(status, response);
+    }
+
+    /**
+     * get the uploaded curve25519 device key
+     *
+     * @return {string} base64 device key
+     */
+    public getDeviceKey(): string {
+        const keyId = 'curve25519:' + this.deviceId;
+        return this.deviceKeys.keys[keyId];
+    }
+
+    /**
+     * get the uploaded ed25519 device key
+     *
+     * @return {string} base64 device key
+     */
+    public getSigningKey(): string {
+        const keyId = 'ed25519:' + this.deviceId;
+        return this.deviceKeys.keys[keyId];
+    }
+
+    /**
+     * flush a single /sync request, and wait for the syncing event
+     */
+    public flushSync(): Promise<void> {
+        logger.log(`${this}: flushSync`);
+        return Promise.all([
+            this.httpBackend.flush('/sync', 1),
+            syncPromise(this.client),
+        ]).then(() => {
+            logger.log(`${this}: flushSync completed`);
+        });
+    }
+
+    public isFallbackICEServerAllowed(): boolean {
+        return true;
+    }
+
+    public getUserId(): string {
+        return this.userId;
+    }
+}

spec/integ/devicelist-integ.spec.js

@@ -136,138 +136,137 @@ describe("DeviceList management:", function() {
         });
     });
 
-    it("We should not get confused by out-of-order device query responses",
-       () => {
-           // https://github.com/vector-im/element-web/issues/3126
-           aliceTestClient.expectKeyQuery({ device_keys: { '@alice:localhost': {} } });
-           return aliceTestClient.start().then(() => {
-               aliceTestClient.httpBackend.when('GET', '/sync').respond(
-                   200, getSyncResponse(['@bob:xyz', '@chris:abc']));
-               return aliceTestClient.flushSync();
-           }).then(() => {
-               // to make sure the initial device queries are flushed out, we
-               // attempt to send a message.
+    it.skip("We should not get confused by out-of-order device query responses", () => {
+        // https://github.com/vector-im/element-web/issues/3126
+        aliceTestClient.expectKeyQuery({ device_keys: { '@alice:localhost': {} } });
+        return aliceTestClient.start().then(() => {
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                200, getSyncResponse(['@bob:xyz', '@chris:abc']));
+            return aliceTestClient.flushSync();
+        }).then(() => {
+            // to make sure the initial device queries are flushed out, we
+            // attempt to send a message.
 
-               aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
-                   200, {
-                       device_keys: {
-                           '@bob:xyz': {},
-                           '@chris:abc': {},
-                       },
-                   },
-               );
+            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
+                200, {
+                    device_keys: {
+                        '@bob:xyz': {},
+                        '@chris:abc': {},
+                    },
+                },
+            );
 
-               aliceTestClient.httpBackend.when('PUT', '/send/').respond(
-                   200, { event_id: '$event1' });
+            aliceTestClient.httpBackend.when('PUT', '/send/').respond(
+                200, { event_id: '$event1' });
 
-               return Promise.all([
-                   aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
-                   aliceTestClient.httpBackend.flush('/keys/query', 1).then(
-                       () => aliceTestClient.httpBackend.flush('/send/', 1),
-                   ),
-                   aliceTestClient.client.crypto.deviceList.saveIfDirty(),
-               ]);
-           }).then(() => {
-               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
-                   expect(data.syncToken).toEqual(1);
-               });
+            return Promise.all([
+                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+                aliceTestClient.httpBackend.flush('/keys/query', 1).then(
+                    () => aliceTestClient.httpBackend.flush('/send/', 1),
+                ),
+                aliceTestClient.client.crypto.deviceList.saveIfDirty(),
+            ]);
+        }).then(() => {
+            aliceTestClient.client.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                expect(data.syncToken).toEqual(1);
+            });
 
-               // invalidate bob's and chris's device lists in separate syncs
-               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
-                   next_batch: '2',
-                   device_lists: {
-                       changed: ['@bob:xyz'],
-                   },
-               });
-               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
-                   next_batch: '3',
-                   device_lists: {
-                       changed: ['@chris:abc'],
-                   },
-               });
-               // flush both syncs
-               return aliceTestClient.flushSync().then(() => {
-                   return aliceTestClient.flushSync();
-               });
-           }).then(() => {
-               // check that we don't yet have a request for chris's devices.
-               aliceTestClient.httpBackend.when('POST', '/keys/query', {
-                   device_keys: {
-                       '@chris:abc': {},
-                   },
-                   token: '3',
-               }).respond(200, {
-                   device_keys: { '@chris:abc': {} },
-               });
-               return aliceTestClient.httpBackend.flush('/keys/query', 1);
-           }).then((flushed) => {
-               expect(flushed).toEqual(0);
-               return aliceTestClient.client.crypto.deviceList.saveIfDirty();
-           }).then(() => {
-               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
-                   const bobStat = data.trackingStatus['@bob:xyz'];
-                   if (bobStat != 1 && bobStat != 2) {
-                       throw new Error('Unexpected status for bob: wanted 1 or 2, got ' +
-                                       bobStat);
-                   }
-                   const chrisStat = data.trackingStatus['@chris:abc'];
-                   if (chrisStat != 1 && chrisStat != 2) {
-                       throw new Error(
-                           'Unexpected status for chris: wanted 1 or 2, got ' + chrisStat,
-                       );
-                   }
-               });
+            // invalidate bob's and chris's device lists in separate syncs
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                next_batch: '2',
+                device_lists: {
+                    changed: ['@bob:xyz'],
+                },
+            });
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                next_batch: '3',
+                device_lists: {
+                    changed: ['@chris:abc'],
+                },
+            });
+            // flush both syncs
+            return aliceTestClient.flushSync().then(() => {
+                return aliceTestClient.flushSync();
+            });
+        }).then(() => {
+            // check that we don't yet have a request for chris's devices.
+            aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                device_keys: {
+                    '@chris:abc': {},
+                },
+                token: '3',
+            }).respond(200, {
+                device_keys: { '@chris:abc': {} },
+            });
+            return aliceTestClient.httpBackend.flush('/keys/query', 1);
+        }).then((flushed) => {
+            expect(flushed).toEqual(0);
+            return aliceTestClient.client.crypto.deviceList.saveIfDirty();
+        }).then(() => {
+            aliceTestClient.client.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+                if (bobStat != 1 && bobStat != 2) {
+                    throw new Error('Unexpected status for bob: wanted 1 or 2, got ' +
+                        bobStat);
+                }
+                const chrisStat = data.trackingStatus['@chris:abc'];
+                if (chrisStat != 1 && chrisStat != 2) {
+                    throw new Error(
+                        'Unexpected status for chris: wanted 1 or 2, got ' + chrisStat,
+                    );
+                }
+            });
 
-               // now add an expectation for a query for bob's devices, and let
-               // it complete.
-               aliceTestClient.httpBackend.when('POST', '/keys/query', {
-                   device_keys: {
-                       '@bob:xyz': {},
-                   },
-                   token: '2',
-               }).respond(200, {
-                   device_keys: { '@bob:xyz': {} },
-               });
-               return aliceTestClient.httpBackend.flush('/keys/query', 1);
-           }).then((flushed) => {
-               expect(flushed).toEqual(1);
+            // now add an expectation for a query for bob's devices, and let
+            // it complete.
+            aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                device_keys: {
+                    '@bob:xyz': {},
+                },
+                token: '2',
+            }).respond(200, {
+                device_keys: { '@bob:xyz': {} },
+            });
+            return aliceTestClient.httpBackend.flush('/keys/query', 1);
+        }).then((flushed) => {
+            expect(flushed).toEqual(1);
 
-               // wait for the client to stop processing the response
-               return aliceTestClient.client.downloadKeys(['@bob:xyz']);
-           }).then(() => {
-               return aliceTestClient.client.crypto.deviceList.saveIfDirty();
-           }).then(() => {
-               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
-                   const bobStat = data.trackingStatus['@bob:xyz'];
-                   expect(bobStat).toEqual(3);
-                   const chrisStat = data.trackingStatus['@chris:abc'];
-                   if (chrisStat != 1 && chrisStat != 2) {
-                       throw new Error(
-                           'Unexpected status for chris: wanted 1 or 2, got ' + bobStat,
-                       );
-                   }
-               });
+            // wait for the client to stop processing the response
+            return aliceTestClient.client.downloadKeys(['@bob:xyz']);
+        }).then(() => {
+            return aliceTestClient.client.crypto.deviceList.saveIfDirty();
+        }).then(() => {
+            aliceTestClient.client.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+                expect(bobStat).toEqual(3);
+                const chrisStat = data.trackingStatus['@chris:abc'];
+                if (chrisStat != 1 && chrisStat != 2) {
+                    throw new Error(
+                        'Unexpected status for chris: wanted 1 or 2, got ' + bobStat,
+                    );
+                }
+            });
 
-               // now let the query for chris's devices complete.
-               return aliceTestClient.httpBackend.flush('/keys/query', 1);
-           }).then((flushed) => {
-               expect(flushed).toEqual(1);
+            // now let the query for chris's devices complete.
+            return aliceTestClient.httpBackend.flush('/keys/query', 1);
+        }).then((flushed) => {
+            expect(flushed).toEqual(1);
 
-               // wait for the client to stop processing the response
-               return aliceTestClient.client.downloadKeys(['@chris:abc']);
-           }).then(() => {
-               return aliceTestClient.client.crypto.deviceList.saveIfDirty();
-           }).then(() => {
-               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
-                   const bobStat = data.trackingStatus['@bob:xyz'];
-                   const chrisStat = data.trackingStatus['@bob:xyz'];
+            // wait for the client to stop processing the response
+            return aliceTestClient.client.downloadKeys(['@chris:abc']);
+        }).then(() => {
+            return aliceTestClient.client.crypto.deviceList.saveIfDirty();
+        }).then(() => {
+            aliceTestClient.client.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+                const chrisStat = data.trackingStatus['@bob:xyz'];
 
-                   expect(bobStat).toEqual(3);
-                   expect(chrisStat).toEqual(3);
-                   expect(data.syncToken).toEqual(3);
-               });
-           });
-       }).timeout(3000);
+                expect(bobStat).toEqual(3);
+                expect(chrisStat).toEqual(3);
+                expect(data.syncToken).toEqual(3);
+            });
+        });
+    });
 
     // https://github.com/vector-im/element-web/issues/4983
     describe("Alice should know she has stale device lists", () => {
@@ -288,7 +287,7 @@ describe("DeviceList management:", function() {
             await aliceTestClient.httpBackend.flush('/keys/query', 1);
             await aliceTestClient.client.crypto.deviceList.saveIfDirty();
 
-            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+            aliceTestClient.client.cryptoStore.getEndToEndDeviceData(null, (data) => {
                 const bobStat = data.trackingStatus['@bob:xyz'];
 
                 expect(bobStat).toBeGreaterThan(
@@ -324,7 +323,7 @@ describe("DeviceList management:", function() {
             await aliceTestClient.flushSync();
             await aliceTestClient.client.crypto.deviceList.saveIfDirty();
 
-            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+            aliceTestClient.client.cryptoStore.getEndToEndDeviceData(null, (data) => {
                 const bobStat = data.trackingStatus['@bob:xyz'];
 
                 expect(bobStat).toEqual(
@@ -360,7 +359,7 @@ describe("DeviceList management:", function() {
             await aliceTestClient.flushSync();
             await aliceTestClient.client.crypto.deviceList.saveIfDirty();
 
-            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+            aliceTestClient.client.cryptoStore.getEndToEndDeviceData(null, (data) => {
                 const bobStat = data.trackingStatus['@bob:xyz'];
 
                 expect(bobStat).toEqual(
@@ -381,7 +380,7 @@ describe("DeviceList management:", function() {
                 await anotherTestClient.flushSync();
                 await anotherTestClient.client.crypto.deviceList.saveIfDirty();
 
-                anotherTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                anotherTestClient.client.cryptoStore.getEndToEndDeviceData(null, (data) => {
                     const bobStat = data.trackingStatus['@bob:xyz'];
 
                     expect(bobStat).toEqual(

spec/integ/matrix-client-crypto.spec.js

@@ -1,758 +0,0 @@
-/*
-Copyright 2016 OpenMarket Ltd
-Copyright 2017 Vector Creations Ltd
-Copyright 2018 New Vector Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/* This file consists of a set of integration tests which try to simulate
- * communication via an Olm-encrypted room between two users, Alice and Bob.
- *
- * Note that megolm (group) conversation is not tested here.
- *
- * See also `megolm.spec.js`.
- */
-
-// load olm before the sdk if possible
-import '../olm-loader';
-
-import { logger } from '../../src/logger';
-import * as testUtils from "../test-utils/test-utils";
-import { TestClient } from "../TestClient";
-import { CRYPTO_ENABLED } from "../../src/client";
-
-let aliTestClient;
-const roomId = "!room:localhost";
-const aliUserId = "@ali:localhost";
-const aliDeviceId = "zxcvb";
-const aliAccessToken = "aseukfgwef";
-let bobTestClient;
-const bobUserId = "@bob:localhost";
-const bobDeviceId = "bvcxz";
-const bobAccessToken = "fewgfkuesa";
-let aliMessages;
-let bobMessages;
-
-function bobUploadsDeviceKeys() {
-    bobTestClient.expectDeviceKeyUpload();
-    return Promise.all([
-        bobTestClient.client.uploadKeys(),
-        bobTestClient.httpBackend.flush(),
-    ]).then(() => {
-        expect(Object.keys(bobTestClient.deviceKeys).length).not.toEqual(0);
-    });
-}
-
-/**
- * Set an expectation that ali will query bobs keys; then flush the http request.
- *
- * @return {promise} resolves once the http request has completed.
- */
-function expectAliQueryKeys() {
-    // can't query keys before bob has uploaded them
-    expect(bobTestClient.deviceKeys).toBeTruthy();
-
-    const bobKeys = {};
-    bobKeys[bobDeviceId] = bobTestClient.deviceKeys;
-    aliTestClient.httpBackend.when("POST", "/keys/query")
-            .respond(200, function(path, content) {
-        expect(content.device_keys[bobUserId]).toEqual(
-            [],
-            "Expected Alice to key query for " + bobUserId + ", got " +
-            Object.keys(content.device_keys),
-        );
-        const result = {};
-        result[bobUserId] = bobKeys;
-        return { device_keys: result };
-    });
-    return aliTestClient.httpBackend.flush("/keys/query", 1);
-}
-
-/**
- * Set an expectation that bob will query alis keys; then flush the http request.
- *
- * @return {promise} which resolves once the http request has completed.
- */
-function expectBobQueryKeys() {
-    // can't query keys before ali has uploaded them
-    expect(aliTestClient.deviceKeys).toBeTruthy();
-
-    const aliKeys = {};
-    aliKeys[aliDeviceId] = aliTestClient.deviceKeys;
-    logger.log("query result will be", aliKeys);
-
-    bobTestClient.httpBackend.when(
-        "POST", "/keys/query",
-    ).respond(200, function(path, content) {
-        expect(content.device_keys[aliUserId]).toEqual(
-            [],
-            "Expected Bob to key query for " + aliUserId + ", got " +
-            Object.keys(content.device_keys),
-        );
-        const result = {};
-        result[aliUserId] = aliKeys;
-        return { device_keys: result };
-    });
-    return bobTestClient.httpBackend.flush("/keys/query", 1);
-}
-
-/**
- * Set an expectation that ali will claim one of bob's keys; then flush the http request.
- *
- * @return {promise} resolves once the http request has completed.
- */
-function expectAliClaimKeys() {
-    return bobTestClient.awaitOneTimeKeyUpload().then((keys) => {
-        aliTestClient.httpBackend.when(
-            "POST", "/keys/claim",
-        ).respond(200, function(path, content) {
-            const claimType = content.one_time_keys[bobUserId][bobDeviceId];
-            expect(claimType).toEqual("signed_curve25519");
-            let keyId = null;
-            for (keyId in keys) {
-                if (bobTestClient.oneTimeKeys.hasOwnProperty(keyId)) {
-                    if (keyId.indexOf(claimType + ":") === 0) {
-                        break;
-                    }
-                }
-            }
-            const result = {};
-            result[bobUserId] = {};
-            result[bobUserId][bobDeviceId] = {};
-            result[bobUserId][bobDeviceId][keyId] = keys[keyId];
-            return { one_time_keys: result };
-        });
-    }).then(() => {
-        // it can take a while to process the key query, so give it some extra
-        // time, and make sure the claim actually happens rather than ploughing on
-        // confusingly.
-        return aliTestClient.httpBackend.flush("/keys/claim", 1, 500).then((r) => {
-            expect(r).toEqual(1, "Ali did not claim Bob's keys");
-        });
-    });
-}
-
-function aliDownloadsKeys() {
-    // can't query keys before bob has uploaded them
-    expect(bobTestClient.getSigningKey()).toBeTruthy();
-
-    const p1 = aliTestClient.client.downloadKeys([bobUserId]).then(function() {
-        return aliTestClient.client.getStoredDevicesForUser(bobUserId);
-    }).then((devices) => {
-        expect(devices.length).toEqual(1);
-        expect(devices[0].deviceId).toEqual("bvcxz");
-    });
-    const p2 = expectAliQueryKeys();
-
-    // check that the localStorage is updated as we expect (not sure this is
-    // an integration test, but meh)
-    return Promise.all([p1, p2]).then(() => {
-        return aliTestClient.client.crypto.deviceList.saveIfDirty();
-    }).then(() => {
-        aliTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
-            const devices = data.devices[bobUserId];
-            expect(devices[bobDeviceId].keys).toEqual(bobTestClient.deviceKeys.keys);
-            expect(devices[bobDeviceId].verified).
-                toBe(0); // DeviceVerification.UNVERIFIED
-        });
-    });
-}
-
-function aliEnablesEncryption() {
-    return aliTestClient.client.setRoomEncryption(roomId, {
-        algorithm: "m.olm.v1.curve25519-aes-sha2",
-    }).then(function() {
-        expect(aliTestClient.client.isRoomEncrypted(roomId)).toBeTruthy();
-    });
-}
-
-function bobEnablesEncryption() {
-    return bobTestClient.client.setRoomEncryption(roomId, {
-        algorithm: "m.olm.v1.curve25519-aes-sha2",
-    }).then(function() {
-       expect(bobTestClient.client.isRoomEncrypted(roomId)).toBeTruthy();
-    });
-}
-
-/**
- * Ali sends a message, first claiming e2e keys. Set the expectations and
- * check the results.
- *
- * @return {promise} which resolves to the ciphertext for Bob's device.
- */
-function aliSendsFirstMessage() {
-    return Promise.all([
-        sendMessage(aliTestClient.client),
-        expectAliQueryKeys()
-            .then(expectAliClaimKeys)
-            .then(expectAliSendMessageRequest),
-    ]).then(function([_, ciphertext]) {
-        return ciphertext;
-    });
-}
-
-/**
- * Ali sends a message without first claiming e2e keys. Set the expectations
- * and check the results.
- *
- * @return {promise} which resolves to the ciphertext for Bob's device.
- */
-function aliSendsMessage() {
-    return Promise.all([
-        sendMessage(aliTestClient.client),
-        expectAliSendMessageRequest(),
-    ]).then(function([_, ciphertext]) {
-        return ciphertext;
-    });
-}
-
-/**
- * Bob sends a message, first querying (but not claiming) e2e keys. Set the
- * expectations and check the results.
- *
- * @return {promise} which resolves to the ciphertext for Ali's device.
- */
-function bobSendsReplyMessage() {
-    return Promise.all([
-        sendMessage(bobTestClient.client),
-        expectBobQueryKeys()
-            .then(expectBobSendMessageRequest),
-    ]).then(function([_, ciphertext]) {
-        return ciphertext;
-    });
-}
-
-/**
- * Set an expectation that Ali will send a message, and flush the request
- *
- * @return {promise} which resolves to the ciphertext for Bob's device.
- */
-function expectAliSendMessageRequest() {
-    return expectSendMessageRequest(aliTestClient.httpBackend).then(function(content) {
-        aliMessages.push(content);
-        expect(Object.keys(content.ciphertext)).toEqual([bobTestClient.getDeviceKey()]);
-        const ciphertext = content.ciphertext[bobTestClient.getDeviceKey()];
-        expect(ciphertext).toBeTruthy();
-        return ciphertext;
-    });
-}
-
-/**
- * Set an expectation that Bob will send a message, and flush the request
- *
- * @return {promise} which resolves to the ciphertext for Bob's device.
- */
-function expectBobSendMessageRequest() {
-    return expectSendMessageRequest(bobTestClient.httpBackend).then(function(content) {
-        bobMessages.push(content);
-        const aliKeyId = "curve25519:" + aliDeviceId;
-        const aliDeviceCurve25519Key = aliTestClient.deviceKeys.keys[aliKeyId];
-        expect(Object.keys(content.ciphertext)).toEqual([aliDeviceCurve25519Key]);
-        const ciphertext = content.ciphertext[aliDeviceCurve25519Key];
-        expect(ciphertext).toBeTruthy();
-        return ciphertext;
-    });
-}
-
-function sendMessage(client) {
-    return client.sendMessage(
-        roomId, { msgtype: "m.text", body: "Hello, World" },
-    );
-}
-
-function expectSendMessageRequest(httpBackend) {
-    const path = "/send/m.room.encrypted/";
-    const prom = new Promise((resolve) => {
-        httpBackend.when("PUT", path).respond(200, function(path, content) {
-            resolve(content);
-            return {
-                event_id: "asdfgh",
-            };
-        });
-    });
-
-    // it can take a while to process the key query
-    return httpBackend.flush(path, 1).then(() => prom);
-}
-
-function aliRecvMessage() {
-    const message = bobMessages.shift();
-    return recvMessage(
-        aliTestClient.httpBackend, aliTestClient.client, bobUserId, message,
-    );
-}
-
-function bobRecvMessage() {
-    const message = aliMessages.shift();
-    return recvMessage(
-        bobTestClient.httpBackend, bobTestClient.client, aliUserId, message,
-    );
-}
-
-function recvMessage(httpBackend, client, sender, message) {
-    const syncData = {
-        next_batch: "x",
-        rooms: {
-            join: {
-
-            },
-        },
-    };
-    syncData.rooms.join[roomId] = {
-        timeline: {
-            events: [
-                testUtils.mkEvent({
-                    type: "m.room.encrypted",
-                    room: roomId,
-                    content: message,
-                    sender: sender,
-                }),
-            ],
-        },
-    };
-    httpBackend.when("GET", "/sync").respond(200, syncData);
-
-    const eventPromise = new Promise((resolve, reject) => {
-        const onEvent = function(event) {
-            // ignore the m.room.member events
-            if (event.getType() == "m.room.member") {
-                return;
-            }
-            logger.log(client.credentials.userId + " received event",
-                        event);
-
-            client.removeListener("event", onEvent);
-            resolve(event);
-        };
-        client.on("event", onEvent);
-    });
-
-    httpBackend.flush();
-
-    return eventPromise.then((event) => {
-        expect(event.isEncrypted()).toBeTruthy();
-
-        // it may still be being decrypted
-        return testUtils.awaitDecryption(event);
-    }).then((event) => {
-        expect(event.getType()).toEqual("m.room.message");
-        expect(event.getContent()).toMatchObject({
-            msgtype: "m.text",
-            body: "Hello, World",
-        });
-        expect(event.isEncrypted()).toBeTruthy();
-    });
-}
-
-/**
- * Send an initial sync response to the client (which just includes the member
- * list for our test room).
- *
- * @param {TestClient} testClient
- * @returns {Promise} which resolves when the sync has been flushed.
- */
-function firstSync(testClient) {
-    // send a sync response including our test room.
-    const syncData = {
-        next_batch: "x",
-        rooms: {
-            join: { },
-        },
-    };
-    syncData.rooms.join[roomId] = {
-        state: {
-            events: [
-                testUtils.mkMembership({
-                    mship: "join",
-                    user: aliUserId,
-                }),
-                testUtils.mkMembership({
-                    mship: "join",
-                    user: bobUserId,
-                }),
-            ],
-        },
-        timeline: {
-            events: [],
-        },
-    };
-
-    testClient.httpBackend.when("GET", "/sync").respond(200, syncData);
-    return testClient.flushSync();
-}
-
-describe("MatrixClient crypto", function() {
-    if (!CRYPTO_ENABLED) {
-        return;
-    }
-
-    beforeEach(async function() {
-        aliTestClient = new TestClient(aliUserId, aliDeviceId, aliAccessToken);
-        await aliTestClient.client.initCrypto();
-
-        bobTestClient = new TestClient(bobUserId, bobDeviceId, bobAccessToken);
-        await bobTestClient.client.initCrypto();
-
-        aliMessages = [];
-        bobMessages = [];
-    });
-
-    afterEach(function() {
-        aliTestClient.httpBackend.verifyNoOutstandingExpectation();
-        bobTestClient.httpBackend.verifyNoOutstandingExpectation();
-
-        return Promise.all([aliTestClient.stop(), bobTestClient.stop()]);
-    });
-
-    it("Bob uploads device keys", function() {
-        return Promise.resolve()
-            .then(bobUploadsDeviceKeys);
-    });
-
-    it("Ali downloads Bobs device keys", function() {
-        return Promise.resolve()
-            .then(bobUploadsDeviceKeys)
-            .then(aliDownloadsKeys);
-    });
-
-    it("Ali gets keys with an invalid signature", function() {
-        return Promise.resolve()
-            .then(bobUploadsDeviceKeys)
-            .then(function() {
-                // tamper bob's keys
-                const bobDeviceKeys = bobTestClient.deviceKeys;
-                expect(bobDeviceKeys.keys["curve25519:" + bobDeviceId]).toBeTruthy();
-                bobDeviceKeys.keys["curve25519:" + bobDeviceId] += "abc";
-
-                return Promise.all([
-                    aliTestClient.client.downloadKeys([bobUserId]),
-                    expectAliQueryKeys(),
-                ]);
-            }).then(function() {
-                return aliTestClient.client.getStoredDevicesForUser(bobUserId);
-            }).then((devices) => {
-                // should get an empty list
-                expect(devices).toEqual([]);
-            });
-    });
-
-    it("Ali gets keys with an incorrect userId", function() {
-        const eveUserId = "@eve:localhost";
-
-        const bobDeviceKeys = {
-            algorithms: ['m.olm.v1.curve25519-aes-sha2', 'm.megolm.v1.aes-sha2'],
-            device_id: 'bvcxz',
-            keys: {
-                'ed25519:bvcxz': 'pYuWKMCVuaDLRTM/eWuB8OlXEb61gZhfLVJ+Y54tl0Q',
-                'curve25519:bvcxz': '7Gni0loo/nzF0nFp9847RbhElGewzwUXHPrljjBGPTQ',
-            },
-            user_id: '@eve:localhost',
-            signatures: {
-                '@eve:localhost': {
-                    'ed25519:bvcxz': 'CliUPZ7dyVPBxvhSA1d+X+LYa5b2AYdjcTwG' +
-                        '0stXcIxjaJNemQqtdgwKDtBFl3pN2I13SEijRDCf1A8bYiQMDg',
-                },
-            },
-        };
-
-        const bobKeys = {};
-        bobKeys[bobDeviceId] = bobDeviceKeys;
-        aliTestClient.httpBackend.when(
-            "POST", "/keys/query",
-        ).respond(200, function(path, content) {
-            const result = {};
-            result[bobUserId] = bobKeys;
-            return { device_keys: result };
-        });
-
-        return Promise.all([
-            aliTestClient.client.downloadKeys([bobUserId, eveUserId]),
-            aliTestClient.httpBackend.flush("/keys/query", 1),
-        ]).then(function() {
-            return Promise.all([
-                aliTestClient.client.getStoredDevicesForUser(bobUserId),
-                aliTestClient.client.getStoredDevicesForUser(eveUserId),
-            ]);
-        }).then(([bobDevices, eveDevices]) => {
-            // should get an empty list
-            expect(bobDevices).toEqual([]);
-            expect(eveDevices).toEqual([]);
-        });
-    });
-
-    it("Ali gets keys with an incorrect deviceId", function() {
-        const bobDeviceKeys = {
-            algorithms: ['m.olm.v1.curve25519-aes-sha2', 'm.megolm.v1.aes-sha2'],
-            device_id: 'bad_device',
-            keys: {
-                'ed25519:bad_device': 'e8XlY5V8x2yJcwa5xpSzeC/QVOrU+D5qBgyTK0ko+f0',
-                'curve25519:bad_device': 'YxuuLG/4L5xGeP8XPl5h0d7DzyYVcof7J7do+OXz0xc',
-            },
-            user_id: '@bob:localhost',
-            signatures: {
-                '@bob:localhost': {
-                    'ed25519:bad_device': 'fEFTq67RaSoIEVBJ8DtmRovbwUBKJ0A' +
-                        'me9m9PDzM9azPUwZ38Xvf6vv1A7W1PSafH4z3Y2ORIyEnZgHaNby3CQ',
-                },
-            },
-        };
-
-        const bobKeys = {};
-        bobKeys[bobDeviceId] = bobDeviceKeys;
-        aliTestClient.httpBackend.when(
-            "POST", "/keys/query",
-        ).respond(200, function(path, content) {
-            const result = {};
-            result[bobUserId] = bobKeys;
-            return { device_keys: result };
-        });
-
-        return Promise.all([
-            aliTestClient.client.downloadKeys([bobUserId]),
-            aliTestClient.httpBackend.flush("/keys/query", 1),
-        ]).then(function() {
-            return aliTestClient.client.getStoredDevicesForUser(bobUserId);
-        }).then((devices) => {
-            // should get an empty list
-            expect(devices).toEqual([]);
-        });
-    });
-
-    it("Bob starts his client and uploads device keys and one-time keys", function() {
-        return Promise.resolve()
-            .then(() => bobTestClient.start())
-            .then(() => bobTestClient.awaitOneTimeKeyUpload())
-            .then((keys) => {
-                expect(Object.keys(keys).length).toEqual(5);
-                expect(Object.keys(bobTestClient.deviceKeys).length).not.toEqual(0);
-            });
-    });
-
-    it("Ali sends a message", function() {
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => bobTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(aliEnablesEncryption)
-            .then(aliSendsFirstMessage);
-    });
-
-    it("Bob receives a message", function() {
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => bobTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(aliEnablesEncryption)
-            .then(aliSendsFirstMessage)
-            .then(bobRecvMessage);
-    });
-
-    it("Bob receives a message with a bogus sender", function() {
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => bobTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(aliEnablesEncryption)
-            .then(aliSendsFirstMessage)
-            .then(function() {
-                const message = aliMessages.shift();
-                const syncData = {
-                    next_batch: "x",
-                    rooms: {
-                        join: {
-
-                        },
-                    },
-                };
-                syncData.rooms.join[roomId] = {
-                    timeline: {
-                        events: [
-                            testUtils.mkEvent({
-                                type: "m.room.encrypted",
-                                room: roomId,
-                                content: message,
-                                sender: "@bogus:sender",
-                            }),
-                        ],
-                    },
-                };
-                bobTestClient.httpBackend.when("GET", "/sync").respond(200, syncData);
-
-                const eventPromise = new Promise((resolve, reject) => {
-                    const onEvent = function(event) {
-                        logger.log(bobUserId + " received event",
-                                    event);
-                        resolve(event);
-                    };
-                    bobTestClient.client.once("event", onEvent);
-                });
-
-                bobTestClient.httpBackend.flush();
-                return eventPromise;
-            }).then((event) => {
-                expect(event.isEncrypted()).toBeTruthy();
-
-                // it may still be being decrypted
-                return testUtils.awaitDecryption(event);
-            }).then((event) => {
-                expect(event.getType()).toEqual("m.room.message");
-                expect(event.getContent().msgtype).toEqual("m.bad.encrypted");
-            });
-    });
-
-    it("Ali blocks Bob's device", function() {
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => bobTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(aliEnablesEncryption)
-            .then(aliDownloadsKeys)
-            .then(function() {
-                aliTestClient.client.setDeviceBlocked(bobUserId, bobDeviceId, true);
-                const p1 = sendMessage(aliTestClient.client);
-                const p2 = expectSendMessageRequest(aliTestClient.httpBackend)
-                      .then(function(sentContent) {
-                          // no unblocked devices, so the ciphertext should be empty
-                          expect(sentContent.ciphertext).toEqual({});
-                      });
-                return Promise.all([p1, p2]);
-            });
-    });
-
-    it("Bob receives two pre-key messages", function() {
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => bobTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(aliEnablesEncryption)
-            .then(aliSendsFirstMessage)
-            .then(bobRecvMessage)
-            .then(aliSendsMessage)
-            .then(bobRecvMessage);
-    });
-
-    it("Bob replies to the message", function() {
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        bobTestClient.expectKeyQuery({ device_keys: { [bobUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => bobTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(() => firstSync(bobTestClient))
-            .then(aliEnablesEncryption)
-            .then(aliSendsFirstMessage)
-            .then(bobRecvMessage)
-            .then(bobEnablesEncryption)
-            .then(bobSendsReplyMessage).then(function(ciphertext) {
-                expect(ciphertext.type).toEqual(1, "Unexpected cipghertext type.");
-            }).then(aliRecvMessage);
-    });
-
-    it("Ali does a key query when encryption is enabled", function() {
-        // enabling encryption in the room should make alice download devices
-        // for both members.
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(() => {
-                const syncData = {
-                    next_batch: '2',
-                    rooms: {
-                        join: {},
-                    },
-                };
-                syncData.rooms.join[roomId] = {
-                    state: {
-                        events: [
-                            testUtils.mkEvent({
-                                type: 'm.room.encryption',
-                                skey: '',
-                                content: {
-                                    algorithm: 'm.olm.v1.curve25519-aes-sha2',
-                                },
-                            }),
-                        ],
-                    },
-                };
-
-                aliTestClient.httpBackend.when('GET', '/sync').respond(
-                    200, syncData);
-                return aliTestClient.httpBackend.flush('/sync', 1);
-            }).then(() => {
-                aliTestClient.expectKeyQuery({
-                    device_keys: {
-                        [bobUserId]: {},
-                    },
-                });
-                return aliTestClient.httpBackend.flushAllExpected();
-            });
-    });
-
-    it("Upload new oneTimeKeys based on a /sync request - no count-asking", function() {
-        // Send a response which causes a key upload
-        const httpBackend = aliTestClient.httpBackend;
-        const syncDataEmpty = {
-            next_batch: "a",
-            device_one_time_keys_count: {
-                signed_curve25519: 0,
-            },
-        };
-
-        // enqueue expectations:
-        // * Sync with empty one_time_keys => upload keys
-
-        return Promise.resolve()
-            .then(() => {
-                logger.log(aliTestClient + ': starting');
-                httpBackend.when("GET", "/versions").respond(200, {});
-                httpBackend.when("GET", "/pushrules").respond(200, {});
-                httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
-                aliTestClient.expectDeviceKeyUpload();
-
-                // we let the client do a very basic initial sync, which it needs before
-                // it will upload one-time keys.
-                httpBackend.when("GET", "/sync").respond(200, syncDataEmpty);
-
-                aliTestClient.client.startClient({});
-
-                return httpBackend.flushAllExpected().then(() => {
-                    logger.log(aliTestClient + ': started');
-                });
-            })
-            .then(() => httpBackend.when("POST", "/keys/upload")
-                .respond(200, (path, content) => {
-                    expect(content.one_time_keys).toBeTruthy();
-                    expect(content.one_time_keys).not.toEqual({});
-                    expect(Object.keys(content.one_time_keys).length)
-                        .toBeGreaterThanOrEqual(1);
-                    logger.log('received %i one-time keys',
-                                Object.keys(content.one_time_keys).length);
-                    // cancel futher calls by telling the client
-                    // we have more than we need
-                    return {
-                       one_time_key_counts: {
-                           signed_curve25519: 70,
-                       },
-                    };
-                }))
-            .then(() => httpBackend.flushAllExpected());
-    });
-});

spec/integ/matrix-client-crypto.spec.ts

@@ -0,0 +1,676 @@
+/*
+Copyright 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/* This file consists of a set of integration tests which try to simulate
+ * communication via an Olm-encrypted room between two users, Alice and Bob.
+ *
+ * Note that megolm (group) conversation is not tested here.
+ *
+ * See also `megolm.spec.js`.
+ */
+
+// load olm before the sdk if possible
+import '../olm-loader';
+
+import { logger } from '../../src/logger';
+import * as testUtils from "../test-utils/test-utils";
+import { TestClient } from "../TestClient";
+import { CRYPTO_ENABLED } from "../../src/client";
+import { ClientEvent, IContent, ISendEventResponse, MatrixClient, MatrixEvent } from "../../src/matrix";
+
+let aliTestClient: TestClient;
+const roomId = "!room:localhost";
+const aliUserId = "@ali:localhost";
+const aliDeviceId = "zxcvb";
+const aliAccessToken = "aseukfgwef";
+let bobTestClient: TestClient;
+const bobUserId = "@bob:localhost";
+const bobDeviceId = "bvcxz";
+const bobAccessToken = "fewgfkuesa";
+let aliMessages: IContent[];
+let bobMessages: IContent[];
+
+// IMessage isn't exported by src/crypto/algorithms/olm.ts
+interface OlmPayload {
+    type: number;
+    body: string;
+}
+
+async function bobUploadsDeviceKeys(): Promise<void> {
+    bobTestClient.expectDeviceKeyUpload();
+    await Promise.all([
+        bobTestClient.client.uploadKeys(),
+        bobTestClient.httpBackend.flushAllExpected(),
+    ]);
+    expect(Object.keys(bobTestClient.deviceKeys).length).not.toEqual(0);
+}
+
+/**
+ * Set an expectation that querier will query uploader's keys; then flush the http request.
+ *
+ * @return {promise} resolves once the http request has completed.
+ */
+function expectQueryKeys(querier: TestClient, uploader: TestClient): Promise<number> {
+    // can't query keys before bob has uploaded them
+    expect(uploader.deviceKeys).toBeTruthy();
+
+    const uploaderKeys = {};
+    uploaderKeys[uploader.deviceId] = uploader.deviceKeys;
+    querier.httpBackend.when("POST", "/keys/query")
+        .respond(200, function(_path, content) {
+            expect(content.device_keys[uploader.userId]).toEqual([]);
+            const result = {};
+            result[uploader.userId] = uploaderKeys;
+            return { device_keys: result };
+        });
+    return querier.httpBackend.flush("/keys/query", 1);
+}
+const expectAliQueryKeys = () => expectQueryKeys(aliTestClient, bobTestClient);
+const expectBobQueryKeys = () => expectQueryKeys(bobTestClient, aliTestClient);
+
+/**
+ * Set an expectation that ali will claim one of bob's keys; then flush the http request.
+ *
+ * @return {promise} resolves once the http request has completed.
+ */
+async function expectAliClaimKeys(): Promise<void> {
+    const keys = await bobTestClient.awaitOneTimeKeyUpload();
+    aliTestClient.httpBackend.when(
+        "POST", "/keys/claim",
+    ).respond(200, function(_path, content) {
+        const claimType = content.one_time_keys[bobUserId][bobDeviceId];
+        expect(claimType).toEqual("signed_curve25519");
+        let keyId = null;
+        for (keyId in keys) {
+            if (bobTestClient.oneTimeKeys.hasOwnProperty(keyId)) {
+                if (keyId.indexOf(claimType + ":") === 0) {
+                    break;
+                }
+            }
+        }
+        const result = {};
+        result[bobUserId] = {};
+        result[bobUserId][bobDeviceId] = {};
+        result[bobUserId][bobDeviceId][keyId] = keys[keyId];
+        return { one_time_keys: result };
+    });
+    // it can take a while to process the key query, so give it some extra
+    // time, and make sure the claim actually happens rather than ploughing on
+    // confusingly.
+    const r = await aliTestClient.httpBackend.flush("/keys/claim", 1, 500);
+    expect(r).toEqual(1);
+}
+
+async function aliDownloadsKeys(): Promise<void> {
+    // can't query keys before bob has uploaded them
+    expect(bobTestClient.getSigningKey()).toBeTruthy();
+
+    const p1 = async () => {
+        await aliTestClient.client.downloadKeys([bobUserId]);
+        const devices = aliTestClient.client.getStoredDevicesForUser(bobUserId);
+        expect(devices.length).toEqual(1);
+        expect(devices[0].deviceId).toEqual("bvcxz");
+    };
+    const p2 = expectAliQueryKeys;
+
+    // check that the localStorage is updated as we expect (not sure this is
+    // an integration test, but meh)
+    await Promise.all([p1(), p2()]);
+    await aliTestClient.client.crypto.deviceList.saveIfDirty();
+    // @ts-ignore - protected
+    aliTestClient.client.cryptoStore.getEndToEndDeviceData(null, (data) => {
+        const devices = data.devices[bobUserId];
+        expect(devices[bobDeviceId].keys).toEqual(bobTestClient.deviceKeys.keys);
+        expect(devices[bobDeviceId].verified).
+            toBe(0); // DeviceVerification.UNVERIFIED
+    });
+}
+
+async function clientEnablesEncryption(client: MatrixClient): Promise<void> {
+    await client.setRoomEncryption(roomId, {
+        algorithm: "m.olm.v1.curve25519-aes-sha2",
+    });
+    expect(client.isRoomEncrypted(roomId)).toBeTruthy();
+}
+const aliEnablesEncryption = () => clientEnablesEncryption(aliTestClient.client);
+const bobEnablesEncryption = () => clientEnablesEncryption(bobTestClient.client);
+
+/**
+ * Ali sends a message, first claiming e2e keys. Set the expectations and
+ * check the results.
+ *
+ * @return {promise} which resolves to the ciphertext for Bob's device.
+ */
+async function aliSendsFirstMessage(): Promise<OlmPayload> {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const [_, ciphertext] = await Promise.all([
+        sendMessage(aliTestClient.client),
+        expectAliQueryKeys()
+            .then(expectAliClaimKeys)
+            .then(expectAliSendMessageRequest),
+    ]);
+    return ciphertext;
+}
+
+/**
+ * Ali sends a message without first claiming e2e keys. Set the expectations
+ * and check the results.
+ *
+ * @return {promise} which resolves to the ciphertext for Bob's device.
+ */
+async function aliSendsMessage(): Promise<OlmPayload> {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const [_, ciphertext] = await Promise.all([
+        sendMessage(aliTestClient.client),
+        expectAliSendMessageRequest(),
+    ]);
+    return ciphertext;
+}
+
+/**
+ * Bob sends a message, first querying (but not claiming) e2e keys. Set the
+ * expectations and check the results.
+ *
+ * @return {promise} which resolves to the ciphertext for Ali's device.
+ */
+async function bobSendsReplyMessage(): Promise<OlmPayload> {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const [_, ciphertext] = await Promise.all([
+        sendMessage(bobTestClient.client),
+        expectBobQueryKeys()
+            .then(expectBobSendMessageRequest),
+    ]);
+    return ciphertext;
+}
+
+/**
+ * Set an expectation that Ali will send a message, and flush the request
+ *
+ * @return {promise} which resolves to the ciphertext for Bob's device.
+ */
+async function expectAliSendMessageRequest(): Promise<OlmPayload> {
+    const content = await expectSendMessageRequest(aliTestClient.httpBackend);
+    aliMessages.push(content);
+    expect(Object.keys(content.ciphertext)).toEqual([bobTestClient.getDeviceKey()]);
+    const ciphertext = content.ciphertext[bobTestClient.getDeviceKey()];
+    expect(ciphertext).toBeTruthy();
+    return ciphertext;
+}
+
+/**
+ * Set an expectation that Bob will send a message, and flush the request
+ *
+ * @return {promise} which resolves to the ciphertext for Bob's device.
+ */
+async function expectBobSendMessageRequest(): Promise<OlmPayload> {
+    const content = await expectSendMessageRequest(bobTestClient.httpBackend);
+    bobMessages.push(content);
+    const aliKeyId = "curve25519:" + aliDeviceId;
+    const aliDeviceCurve25519Key = aliTestClient.deviceKeys.keys[aliKeyId];
+    expect(Object.keys(content.ciphertext)).toEqual([aliDeviceCurve25519Key]);
+    const ciphertext = content.ciphertext[aliDeviceCurve25519Key];
+    expect(ciphertext).toBeTruthy();
+    return ciphertext;
+}
+
+function sendMessage(client: MatrixClient): Promise<ISendEventResponse> {
+    return client.sendMessage(
+        roomId, { msgtype: "m.text", body: "Hello, World" },
+    );
+}
+
+async function expectSendMessageRequest(httpBackend: TestClient["httpBackend"]): Promise<IContent> {
+    const path = "/send/m.room.encrypted/";
+    const prom = new Promise((resolve) => {
+        httpBackend.when("PUT", path).respond(200, function(_path, content) {
+            resolve(content);
+            return {
+                event_id: "asdfgh",
+            };
+        });
+    });
+
+    // it can take a while to process the key query
+    await httpBackend.flush(path, 1);
+    return prom;
+}
+
+function aliRecvMessage(): Promise<void> {
+    const message = bobMessages.shift();
+    return recvMessage(
+        aliTestClient.httpBackend, aliTestClient.client, bobUserId, message,
+    );
+}
+
+function bobRecvMessage(): Promise<void> {
+    const message = aliMessages.shift();
+    return recvMessage(
+        bobTestClient.httpBackend, bobTestClient.client, aliUserId, message,
+    );
+}
+
+async function recvMessage(
+    httpBackend: TestClient["httpBackend"],
+    client: MatrixClient,
+    sender: string,
+    message: IContent,
+): Promise<void> {
+    const syncData = {
+        next_batch: "x",
+        rooms: {
+            join: {
+
+            },
+        },
+    };
+    syncData.rooms.join[roomId] = {
+        timeline: {
+            events: [
+                testUtils.mkEvent({
+                    type: "m.room.encrypted",
+                    room: roomId,
+                    content: message,
+                    sender: sender,
+                }),
+            ],
+        },
+    };
+    httpBackend.when("GET", "/sync").respond(200, syncData);
+
+    const eventPromise = new Promise<MatrixEvent>((resolve) => {
+        const onEvent = function(event: MatrixEvent) {
+            // ignore the m.room.member events
+            if (event.getType() == "m.room.member") {
+                return;
+            }
+            logger.log(client.credentials.userId + " received event",
+                event);
+
+            client.removeListener(ClientEvent.Event, onEvent);
+            resolve(event);
+        };
+        client.on(ClientEvent.Event, onEvent);
+    });
+
+    await httpBackend.flushAllExpected();
+
+    const preDecryptionEvent = await eventPromise;
+    expect(preDecryptionEvent.isEncrypted()).toBeTruthy();
+    // it may still be being decrypted
+    const event = await testUtils.awaitDecryption(preDecryptionEvent);
+    expect(event.getType()).toEqual("m.room.message");
+    expect(event.getContent()).toMatchObject({
+        msgtype: "m.text",
+        body: "Hello, World",
+    });
+    expect(event.isEncrypted()).toBeTruthy();
+}
+
+/**
+ * Send an initial sync response to the client (which just includes the member
+ * list for our test room).
+ *
+ * @param {TestClient} testClient
+ * @returns {Promise} which resolves when the sync has been flushed.
+ */
+function firstSync(testClient: TestClient): Promise<void> {
+    // send a sync response including our test room.
+    const syncData = {
+        next_batch: "x",
+        rooms: {
+            join: { },
+        },
+    };
+    syncData.rooms.join[roomId] = {
+        state: {
+            events: [
+                testUtils.mkMembership({
+                    mship: "join",
+                    user: aliUserId,
+                }),
+                testUtils.mkMembership({
+                    mship: "join",
+                    user: bobUserId,
+                }),
+            ],
+        },
+        timeline: {
+            events: [],
+        },
+    };
+
+    testClient.httpBackend.when("GET", "/sync").respond(200, syncData);
+    return testClient.flushSync();
+}
+
+describe("MatrixClient crypto", () => {
+    if (!CRYPTO_ENABLED) {
+        return;
+    }
+
+    beforeEach(async () => {
+        aliTestClient = new TestClient(aliUserId, aliDeviceId, aliAccessToken);
+        await aliTestClient.client.initCrypto();
+
+        bobTestClient = new TestClient(bobUserId, bobDeviceId, bobAccessToken);
+        await bobTestClient.client.initCrypto();
+
+        aliMessages = [];
+        bobMessages = [];
+    });
+
+    afterEach(() => {
+        aliTestClient.httpBackend.verifyNoOutstandingExpectation();
+        bobTestClient.httpBackend.verifyNoOutstandingExpectation();
+
+        return Promise.all([aliTestClient.stop(), bobTestClient.stop()]);
+    });
+
+    it("Bob uploads device keys", bobUploadsDeviceKeys);
+
+    it("Ali downloads Bobs device keys", async () => {
+        await bobUploadsDeviceKeys();
+        await aliDownloadsKeys();
+    });
+
+    it("Ali gets keys with an invalid signature", async () => {
+        await bobUploadsDeviceKeys();
+        // tamper bob's keys
+        const bobDeviceKeys = bobTestClient.deviceKeys;
+        expect(bobDeviceKeys.keys["curve25519:" + bobDeviceId]).toBeTruthy();
+        bobDeviceKeys.keys["curve25519:" + bobDeviceId] += "abc";
+        await Promise.all([
+            aliTestClient.client.downloadKeys([bobUserId]),
+            expectAliQueryKeys(),
+        ]);
+        const devices = aliTestClient.client.getStoredDevicesForUser(bobUserId);
+        // should get an empty list
+        expect(devices).toEqual([]);
+    });
+
+    it("Ali gets keys with an incorrect userId", async () => {
+        const eveUserId = "@eve:localhost";
+
+        const bobDeviceKeys = {
+            algorithms: ['m.olm.v1.curve25519-aes-sha2', 'm.megolm.v1.aes-sha2'],
+            device_id: 'bvcxz',
+            keys: {
+                'ed25519:bvcxz': 'pYuWKMCVuaDLRTM/eWuB8OlXEb61gZhfLVJ+Y54tl0Q',
+                'curve25519:bvcxz': '7Gni0loo/nzF0nFp9847RbhElGewzwUXHPrljjBGPTQ',
+            },
+            user_id: '@eve:localhost',
+            signatures: {
+                '@eve:localhost': {
+                    'ed25519:bvcxz': 'CliUPZ7dyVPBxvhSA1d+X+LYa5b2AYdjcTwG' +
+                        '0stXcIxjaJNemQqtdgwKDtBFl3pN2I13SEijRDCf1A8bYiQMDg',
+                },
+            },
+        };
+
+        const bobKeys = {};
+        bobKeys[bobDeviceId] = bobDeviceKeys;
+        aliTestClient.httpBackend.when(
+            "POST", "/keys/query",
+        ).respond(200, { device_keys: { [bobUserId]: bobKeys } });
+
+        await Promise.all([
+            aliTestClient.client.downloadKeys([bobUserId, eveUserId]),
+            aliTestClient.httpBackend.flush("/keys/query", 1),
+        ]);
+        const [bobDevices, eveDevices] = await Promise.all([
+            aliTestClient.client.getStoredDevicesForUser(bobUserId),
+            aliTestClient.client.getStoredDevicesForUser(eveUserId),
+        ]);
+        // should get an empty list
+        expect(bobDevices).toEqual([]);
+        expect(eveDevices).toEqual([]);
+    });
+
+    it("Ali gets keys with an incorrect deviceId", async () => {
+        const bobDeviceKeys = {
+            algorithms: ['m.olm.v1.curve25519-aes-sha2', 'm.megolm.v1.aes-sha2'],
+            device_id: 'bad_device',
+            keys: {
+                'ed25519:bad_device': 'e8XlY5V8x2yJcwa5xpSzeC/QVOrU+D5qBgyTK0ko+f0',
+                'curve25519:bad_device': 'YxuuLG/4L5xGeP8XPl5h0d7DzyYVcof7J7do+OXz0xc',
+            },
+            user_id: '@bob:localhost',
+            signatures: {
+                '@bob:localhost': {
+                    'ed25519:bad_device': 'fEFTq67RaSoIEVBJ8DtmRovbwUBKJ0A' +
+                        'me9m9PDzM9azPUwZ38Xvf6vv1A7W1PSafH4z3Y2ORIyEnZgHaNby3CQ',
+                },
+            },
+        };
+
+        const bobKeys = {};
+        bobKeys[bobDeviceId] = bobDeviceKeys;
+        aliTestClient.httpBackend.when(
+            "POST", "/keys/query",
+        ).respond(200, { device_keys: { [bobUserId]: bobKeys } });
+
+        await Promise.all([
+            aliTestClient.client.downloadKeys([bobUserId]),
+            aliTestClient.httpBackend.flush("/keys/query", 1),
+        ]);
+        const devices = aliTestClient.client.getStoredDevicesForUser(bobUserId);
+        // should get an empty list
+        expect(devices).toEqual([]);
+    });
+
+    it("Bob starts his client and uploads device keys and one-time keys", async () => {
+        await bobTestClient.start();
+        const keys = await bobTestClient.awaitOneTimeKeyUpload();
+        expect(Object.keys(keys).length).toEqual(5);
+        expect(Object.keys(bobTestClient.deviceKeys).length).not.toEqual(0);
+    });
+
+    it("Ali sends a message", async () => {
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await bobTestClient.start();
+        await firstSync(aliTestClient);
+        await aliEnablesEncryption();
+        await aliSendsFirstMessage();
+    });
+
+    it("Bob receives a message", async () => {
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await bobTestClient.start();
+        await firstSync(aliTestClient);
+        await aliEnablesEncryption();
+        await aliSendsFirstMessage();
+        await bobRecvMessage();
+    });
+
+    it("Bob receives a message with a bogus sender", async () => {
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await bobTestClient.start();
+        await firstSync(aliTestClient);
+        await aliEnablesEncryption();
+        await aliSendsFirstMessage();
+        const message = aliMessages.shift();
+        const syncData = {
+            next_batch: "x",
+            rooms: {
+                join: {
+
+                },
+            },
+        };
+        syncData.rooms.join[roomId] = {
+            timeline: {
+                events: [
+                    testUtils.mkEvent({
+                        type: "m.room.encrypted",
+                        room: roomId,
+                        content: message,
+                        sender: "@bogus:sender",
+                    }),
+                ],
+            },
+        };
+        bobTestClient.httpBackend.when("GET", "/sync").respond(200, syncData);
+
+        const eventPromise = new Promise<MatrixEvent>((resolve) => {
+            const onEvent = function(event: MatrixEvent) {
+                logger.log(bobUserId + " received event", event);
+                resolve(event);
+            };
+            bobTestClient.client.once(ClientEvent.Event, onEvent);
+        });
+        await bobTestClient.httpBackend.flushAllExpected();
+        const preDecryptionEvent = await eventPromise;
+        expect(preDecryptionEvent.isEncrypted()).toBeTruthy();
+        // it may still be being decrypted
+        const event = await testUtils.awaitDecryption(preDecryptionEvent);
+        expect(event.getType()).toEqual("m.room.message");
+        expect(event.getContent().msgtype).toEqual("m.bad.encrypted");
+    });
+
+    it("Ali blocks Bob's device", async () => {
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await bobTestClient.start();
+        await firstSync(aliTestClient);
+        await aliEnablesEncryption();
+        await aliDownloadsKeys();
+        aliTestClient.client.setDeviceBlocked(bobUserId, bobDeviceId, true);
+        const p1 = sendMessage(aliTestClient.client);
+        const p2 = expectSendMessageRequest(aliTestClient.httpBackend)
+            .then(function(sentContent) {
+                // no unblocked devices, so the ciphertext should be empty
+                expect(sentContent.ciphertext).toEqual({});
+            });
+        await Promise.all([p1, p2]);
+    });
+
+    it("Bob receives two pre-key messages", async () => {
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await bobTestClient.start();
+        await firstSync(aliTestClient);
+        await aliEnablesEncryption();
+        await aliSendsFirstMessage();
+        await bobRecvMessage();
+        await aliSendsMessage();
+        await bobRecvMessage();
+    });
+
+    it("Bob replies to the message", async () => {
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        bobTestClient.expectKeyQuery({ device_keys: { [bobUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await bobTestClient.start();
+        await firstSync(aliTestClient);
+        await firstSync(bobTestClient);
+        await aliEnablesEncryption();
+        await aliSendsFirstMessage();
+        await bobRecvMessage();
+        await bobEnablesEncryption();
+        const ciphertext = await bobSendsReplyMessage();
+        expect(ciphertext.type).toEqual(1);
+        await aliRecvMessage();
+    });
+
+    it("Ali does a key query when encryption is enabled", async () => {
+        // enabling encryption in the room should make alice download devices
+        // for both members.
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await firstSync(aliTestClient);
+        const syncData = {
+            next_batch: '2',
+            rooms: {
+                join: {},
+            },
+        };
+        syncData.rooms.join[roomId] = {
+            state: {
+                events: [
+                    testUtils.mkEvent({
+                        type: 'm.room.encryption',
+                        skey: '',
+                        content: {
+                            algorithm: 'm.olm.v1.curve25519-aes-sha2',
+                        },
+                    }),
+                ],
+            },
+        };
+
+        aliTestClient.httpBackend.when('GET', '/sync').respond(
+            200, syncData);
+        await aliTestClient.httpBackend.flush('/sync', 1);
+        aliTestClient.expectKeyQuery({
+            device_keys: {
+                [bobUserId]: {},
+            },
+            failures: {},
+        });
+        await aliTestClient.httpBackend.flushAllExpected();
+    });
+
+    it("Upload new oneTimeKeys based on a /sync request - no count-asking", async () => {
+        // Send a response which causes a key upload
+        const httpBackend = aliTestClient.httpBackend;
+        const syncDataEmpty = {
+            next_batch: "a",
+            device_one_time_keys_count: {
+                signed_curve25519: 0,
+            },
+        };
+
+        // enqueue expectations:
+        // * Sync with empty one_time_keys => upload keys
+
+        logger.log(aliTestClient + ': starting');
+        httpBackend.when("GET", "/versions").respond(200, {});
+        httpBackend.when("GET", "/pushrules").respond(200, {});
+        httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+        aliTestClient.expectDeviceKeyUpload();
+
+        // we let the client do a very basic initial sync, which it needs before
+        // it will upload one-time keys.
+        httpBackend.when("GET", "/sync").respond(200, syncDataEmpty);
+
+        await Promise.all([
+            aliTestClient.client.startClient({}),
+            httpBackend.flushAllExpected(),
+        ]);
+        logger.log(aliTestClient + ': started');
+        httpBackend.when("POST", "/keys/upload")
+            .respond(200, (_path, content) => {
+                expect(content.one_time_keys).toBeTruthy();
+                expect(content.one_time_keys).not.toEqual({});
+                expect(Object.keys(content.one_time_keys).length).toBeGreaterThanOrEqual(1);
+                logger.log('received %i one-time keys', Object.keys(content.one_time_keys).length);
+                // cancel futher calls by telling the client
+                // we have more than we need
+                return {
+                    one_time_key_counts: {
+                        signed_curve25519: 70,
+                    },
+                };
+            });
+        await httpBackend.flushAllExpected();
+    });
+});

spec/integ/matrix-client-event-timeline.spec.ts

@@ -1,5 +1,21 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 import * as utils from "../test-utils/test-utils";
-import { EventTimeline } from "../../src/matrix";
+import { ClientEvent, EventTimeline, Filter, IEvent, MatrixClient, MatrixEvent, Room } from "../../src/matrix";
 import { logger } from "../../src/logger";
 import { TestClient } from "../TestClient";
 import { Thread, THREAD_RELATION_TYPE } from "../../src/models/thread";
@@ -10,8 +26,14 @@ const accessToken = "aseukfgwef";
 const roomId = "!foo:bar";
 const otherUserId = "@bob:localhost";
 
+const withoutRoomId = (e: Partial<IEvent>): Partial<IEvent> => {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const { room_id: _, ...copy } = e;
+    return copy;
+};
+
 const USER_MEMBERSHIP_EVENT = utils.mkMembership({
-    room: roomId, mship: "join", user: userId, name: userName,
+    room: roomId, mship: "join", user: userId, name: userName, event: false,
 });
 
 const ROOM_NAME_EVENT = utils.mkEvent({
@@ -19,34 +41,37 @@ const ROOM_NAME_EVENT = utils.mkEvent({
     content: {
         name: "Old room name",
     },
+    event: false,
 });
 
 const INITIAL_SYNC_DATA = {
     next_batch: "s_5_3",
     rooms: {
         join: {
-            "!foo:bar": {  // roomId
+            [roomId]: {
                 timeline: {
                     events: [
                         utils.mkMessage({
-                            room: roomId, user: otherUserId, msg: "hello",
+                            user: otherUserId, msg: "hello", event: false,
                         }),
                     ],
                     prev_batch: "f_1_1",
                 },
                 state: {
                     events: [
-                        ROOM_NAME_EVENT,
+                        withoutRoomId(ROOM_NAME_EVENT),
                         utils.mkMembership({
-                            room: roomId, mship: "join",
+                            mship: "join",
                             user: otherUserId, name: "Bob",
+                            event: false,
                         }),
-                        USER_MEMBERSHIP_EVENT,
+                        withoutRoomId(USER_MEMBERSHIP_EVENT),
                         utils.mkEvent({
-                            type: "m.room.create", room: roomId, user: userId,
+                            type: "m.room.create", user: userId,
                             content: {
                                 creator: userId,
                             },
+                            event: false,
                         }),
                     ],
                 },
@@ -57,23 +82,37 @@ const INITIAL_SYNC_DATA = {
 
 const EVENTS = [
     utils.mkMessage({
-        room: roomId, user: userId, msg: "we",
+        room: roomId, user: userId, msg: "we", event: false,
     }),
     utils.mkMessage({
-        room: roomId, user: userId, msg: "could",
+        room: roomId, user: userId, msg: "could", event: false,
     }),
     utils.mkMessage({
-        room: roomId, user: userId, msg: "be",
+        room: roomId, user: userId, msg: "be", event: false,
     }),
     utils.mkMessage({
-        room: roomId, user: userId, msg: "heroes",
+        room: roomId, user: userId, msg: "heroes", event: false,
     }),
 ];
 
-const THREAD_ROOT = utils.mkMessage({
+const THREAD_ROOT = utils.mkEvent({
     room: roomId,
     user: userId,
-    msg: "thread root",
+    type: "m.room.message",
+    content: {
+        "body": "thread root",
+        "msgtype": "m.text",
+    },
+    unsigned: {
+        "m.relations": {
+            "io.element.thread": {
+                //"latest_event": undefined,
+                "count": 1,
+                "current_user_participated": true,
+            },
+        },
+    },
+    event: false,
 });
 
 const THREAD_REPLY = utils.mkEvent({
@@ -89,10 +128,25 @@ const THREAD_REPLY = utils.mkEvent({
             event_id: THREAD_ROOT.event_id,
         },
     },
+    event: false,
 });
 
+THREAD_ROOT.unsigned["m.relations"]["io.element.thread"].latest_event = THREAD_REPLY;
+
+const SYNC_THREAD_ROOT = withoutRoomId(THREAD_ROOT);
+const SYNC_THREAD_REPLY = withoutRoomId(THREAD_REPLY);
+SYNC_THREAD_ROOT.unsigned = {
+    "m.relations": {
+        "io.element.thread": {
+            "latest_event": SYNC_THREAD_REPLY,
+            "count": 1,
+            "current_user_participated": true,
+        },
+    },
+};
+
 // start the client, and wait for it to initialise
-function startClient(httpBackend, client) {
+function startClient(httpBackend: TestClient["httpBackend"], client: MatrixClient) {
     httpBackend.when("GET", "/versions").respond(200, {});
     httpBackend.when("GET", "/pushrules").respond(200, {});
     httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
@@ -101,8 +155,8 @@ function startClient(httpBackend, client) {
     client.startClient();
 
     // set up a promise which will resolve once the client is initialised
-    const prom = new Promise((resolve) => {
-        client.on("sync", function(state) {
+    const prom = new Promise<void>((resolve) => {
+        client.on(ClientEvent.Sync, function(state) {
             logger.log("sync", state);
             if (state != "SYNCING") {
                 return;
@@ -118,8 +172,8 @@ function startClient(httpBackend, client) {
 }
 
 describe("getEventTimeline support", function() {
-    let httpBackend;
-    let client;
+    let httpBackend: TestClient["httpBackend"];
+    let client: MatrixClient;
 
     beforeEach(function() {
         const testClient = new TestClient(userId, "DEVICE", accessToken);
@@ -162,7 +216,7 @@ describe("getEventTimeline support", function() {
 
     it("scrollback should be able to scroll back to before a gappy /sync", function() {
         // need a client with timelineSupport disabled to make this work
-        let room;
+        let room: Room;
 
         return startClient(httpBackend, client).then(function() {
             room = client.getRoom(roomId);
@@ -174,7 +228,7 @@ describe("getEventTimeline support", function() {
                         "!foo:bar": {
                             timeline: {
                                 events: [
-                                    EVENTS[0],
+                                    withoutRoomId(EVENTS[0]),
                                 ],
                                 prev_batch: "f_1_1",
                             },
@@ -190,7 +244,7 @@ describe("getEventTimeline support", function() {
                         "!foo:bar": {
                             timeline: {
                                 events: [
-                                    EVENTS[1],
+                                    withoutRoomId(EVENTS[1]),
                                 ],
                                 limited: true,
                                 prev_batch: "f_1_2",
@@ -225,8 +279,8 @@ describe("getEventTimeline support", function() {
 });
 
 describe("MatrixClient event timelines", function() {
-    let client = null;
-    let httpBackend = null;
+    let client: MatrixClient;
+    let httpBackend: TestClient["httpBackend"];
 
     beforeEach(function() {
         const testClient = new TestClient(
@@ -245,7 +299,7 @@ describe("MatrixClient event timelines", function() {
     afterEach(function() {
         httpBackend.verifyNoOutstandingExpectation();
         client.stopClient();
-        Thread.setServerSideSupport(false);
+        Thread.setServerSideSupport(false, false);
     });
 
     describe("getEventTimeline", function() {
@@ -293,7 +347,7 @@ describe("MatrixClient event timelines", function() {
                         "!foo:bar": {
                             timeline: {
                                 events: [
-                                    EVENTS[0],
+                                    withoutRoomId(EVENTS[0]),
                                 ],
                                 prev_batch: "f_1_2",
                             },
@@ -328,7 +382,7 @@ describe("MatrixClient event timelines", function() {
                         "!foo:bar": {
                             timeline: {
                                 events: [
-                                    EVENTS[3],
+                                    withoutRoomId(EVENTS[3]),
                                 ],
                                 prev_batch: "f_1_2",
                             },
@@ -351,7 +405,7 @@ describe("MatrixClient event timelines", function() {
                 });
 
             const prom = new Promise((resolve, reject) => {
-                client.on("sync", function() {
+                client.on(ClientEvent.Sync, function() {
                     client.getEventTimeline(timelineSet, EVENTS[2].event_id,
                     ).then(function(tl) {
                         expect(tl.getEvents().length).toEqual(4);
@@ -496,11 +550,13 @@ describe("MatrixClient event timelines", function() {
         });
 
         it("should handle thread replies with server support by fetching a contiguous thread timeline", async () => {
+            // @ts-ignore
             client.clientOpts.experimentalThreadSupport = true;
-            Thread.setServerSideSupport(true);
+            Thread.setServerSideSupport(true, false);
             client.stopClient(); // we don't need the client to be syncing at this time
             const room = client.getRoom(roomId);
-            const timelineSet = room.getTimelineSets()[0];
+            const thread = room.createThread(THREAD_ROOT.event_id, undefined, [], false);
+            const timelineSet = thread.timelineSet;
 
             httpBackend.when("GET", "/rooms/!foo%3Abar/context/" + encodeURIComponent(THREAD_REPLY.event_id))
                 .respond(200, function() {
@@ -538,6 +594,191 @@ describe("MatrixClient event timelines", function() {
             expect(timeline.getEvents().find(e => e.getId() === THREAD_ROOT.event_id)).toBeTruthy();
             expect(timeline.getEvents().find(e => e.getId() === THREAD_REPLY.event_id)).toBeTruthy();
         });
+
+        it("should return relevant timeline from non-thread timelineSet when asking for the thread root", async () => {
+            // @ts-ignore
+            client.clientOpts.experimentalThreadSupport = true;
+            Thread.setServerSideSupport(true, false);
+            client.stopClient(); // we don't need the client to be syncing at this time
+            const room = client.getRoom(roomId);
+            const threadRoot = new MatrixEvent(THREAD_ROOT);
+            const thread = room.createThread(THREAD_ROOT.event_id, threadRoot, [threadRoot], false);
+            const timelineSet = room.getTimelineSets()[0];
+
+            httpBackend.when("GET", "/rooms/!foo%3Abar/context/" + encodeURIComponent(THREAD_ROOT.event_id))
+                .respond(200, function() {
+                    return {
+                        start: "start_token0",
+                        events_before: [],
+                        event: THREAD_ROOT,
+                        events_after: [],
+                        end: "end_token0",
+                        state: [],
+                    };
+                });
+
+            const [timeline] = await Promise.all([
+                client.getEventTimeline(timelineSet, THREAD_ROOT.event_id),
+                httpBackend.flushAllExpected(),
+            ]);
+
+            expect(timeline).not.toBe(thread.liveTimeline);
+            expect(timelineSet.getTimelines()).toContain(timeline);
+            expect(timeline.getEvents().find(e => e.getId() === THREAD_ROOT.event_id)).toBeTruthy();
+        });
+
+        it("should return undefined when event is not in the thread that the given timelineSet is representing", () => {
+            // @ts-ignore
+            client.clientOpts.experimentalThreadSupport = true;
+            Thread.setServerSideSupport(true, false);
+            client.stopClient(); // we don't need the client to be syncing at this time
+            const room = client.getRoom(roomId);
+            const threadRoot = new MatrixEvent(THREAD_ROOT);
+            const thread = room.createThread(THREAD_ROOT.event_id, threadRoot, [threadRoot], false);
+            const timelineSet = thread.timelineSet;
+
+            httpBackend.when("GET", "/rooms/!foo%3Abar/context/" + encodeURIComponent(EVENTS[0].event_id))
+                .respond(200, function() {
+                    return {
+                        start: "start_token0",
+                        events_before: [],
+                        event: EVENTS[0],
+                        events_after: [],
+                        end: "end_token0",
+                        state: [],
+                    };
+                });
+
+            return Promise.all([
+                expect(client.getEventTimeline(timelineSet, EVENTS[0].event_id)).resolves.toBeUndefined(),
+                httpBackend.flushAllExpected(),
+            ]);
+        });
+
+        it("should return undefined when event is within a thread but timelineSet is not", () => {
+            // @ts-ignore
+            client.clientOpts.experimentalThreadSupport = true;
+            Thread.setServerSideSupport(true, false);
+            client.stopClient(); // we don't need the client to be syncing at this time
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
+
+            httpBackend.when("GET", "/rooms/!foo%3Abar/context/" + encodeURIComponent(THREAD_REPLY.event_id))
+                .respond(200, function() {
+                    return {
+                        start: "start_token0",
+                        events_before: [],
+                        event: THREAD_REPLY,
+                        events_after: [],
+                        end: "end_token0",
+                        state: [],
+                    };
+                });
+
+            return Promise.all([
+                expect(client.getEventTimeline(timelineSet, THREAD_REPLY.event_id)).resolves.toBeUndefined(),
+                httpBackend.flushAllExpected(),
+            ]);
+        });
+
+        it("should should add lazy loading filter when requested", async () => {
+            // @ts-ignore
+            client.clientOpts.lazyLoadMembers = true;
+            client.stopClient(); // we don't need the client to be syncing at this time
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
+
+            const req = httpBackend.when("GET", "/rooms/!foo%3Abar/context/" + encodeURIComponent(EVENTS[0].event_id));
+            req.respond(200, function() {
+                return {
+                    start: "start_token0",
+                    events_before: [],
+                    event: EVENTS[0],
+                    events_after: [],
+                    end: "end_token0",
+                    state: [],
+                };
+            });
+            req.check((request) => {
+                expect(request.queryParams.filter).toEqual(JSON.stringify(Filter.LAZY_LOADING_MESSAGES_FILTER));
+            });
+
+            await Promise.all([
+                client.getEventTimeline(timelineSet, EVENTS[0].event_id),
+                httpBackend.flushAllExpected(),
+            ]);
+        });
+    });
+
+    describe("getLatestTimeline", function() {
+        it("should create a new timeline for new events", function() {
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
+
+            const latestMessageId = 'event1:bar';
+
+            httpBackend.when("GET", "/rooms/!foo%3Abar/messages")
+                .respond(200, function() {
+                    return {
+                        chunk: [{
+                            event_id: latestMessageId,
+                        }],
+                    };
+                });
+
+            httpBackend.when("GET", `/rooms/!foo%3Abar/context/${encodeURIComponent(latestMessageId)}`)
+                .respond(200, function() {
+                    return {
+                        start: "start_token",
+                        events_before: [EVENTS[1], EVENTS[0]],
+                        event: EVENTS[2],
+                        events_after: [EVENTS[3]],
+                        state: [
+                            ROOM_NAME_EVENT,
+                            USER_MEMBERSHIP_EVENT,
+                        ],
+                        end: "end_token",
+                    };
+                });
+
+            return Promise.all([
+                client.getLatestTimeline(timelineSet).then(function(tl) {
+                    // Instead of this assertion logic, we could just add a spy
+                    // for `getEventTimeline` and make sure it's called with the
+                    // correct parameters. This doesn't feel too bad to make sure
+                    // `getLatestTimeline` is doing the right thing though.
+                    expect(tl.getEvents().length).toEqual(4);
+                    for (let i = 0; i < 4; i++) {
+                        expect(tl.getEvents()[i].event).toEqual(EVENTS[i]);
+                        expect(tl.getEvents()[i].sender.name).toEqual(userName);
+                    }
+                    expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toEqual("start_token");
+                    expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                        .toEqual("end_token");
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
+        });
+
+        it("should throw error when /messages does not return a message", () => {
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
+
+            httpBackend.when("GET", "/rooms/!foo%3Abar/messages")
+                .respond(200, () => {
+                    return {
+                        chunk: [
+                            // No messages to return
+                        ],
+                    };
+                });
+
+            return Promise.all([
+                expect(client.getLatestTimeline(timelineSet)).rejects.toThrow(),
+                httpBackend.flushAllExpected(),
+            ]);
+        });
     });
 
     describe("paginateEventTimeline", function() {
@@ -666,7 +907,7 @@ describe("MatrixClient event timelines", function() {
                         "!foo:bar": {
                             timeline: {
                                 events: [
-                                    event,
+                                    withoutRoomId(event),
                                 ],
                                 prev_batch: "f_1_1",
                             },
@@ -744,11 +985,10 @@ describe("MatrixClient event timelines", function() {
 
         // a state event, followed by a redaction thereof
         const event = utils.mkMembership({
-            room: roomId, mship: "join", user: otherUserId,
+            mship: "join", user: otherUserId,
         });
         const redaction = utils.mkEvent({
             type: "m.room.redaction",
-            room_id: roomId,
             sender: otherUserId,
             content: {},
         });
@@ -790,7 +1030,7 @@ describe("MatrixClient event timelines", function() {
                 timeline: {
                     events: [
                         utils.mkMessage({
-                            room: roomId, user: otherUserId, msg: "world",
+                            user: otherUserId, msg: "world",
                         }),
                     ],
                     limited: true,
@@ -809,4 +1049,75 @@ describe("MatrixClient event timelines", function() {
             expect(tl.getEvents().length).toEqual(1);
         });
     });
+
+    it("should re-insert room IDs for bundled thread relation events", async () => {
+        // @ts-ignore
+        client.clientOpts.experimentalThreadSupport = true;
+        Thread.setServerSideSupport(true, false);
+
+        httpBackend.when("GET", "/sync").respond(200, {
+            next_batch: "s_5_4",
+            rooms: {
+                join: {
+                    [roomId]: {
+                        timeline: {
+                            events: [
+                                SYNC_THREAD_ROOT,
+                            ],
+                            prev_batch: "f_1_1",
+                        },
+                    },
+                },
+            },
+        });
+        await Promise.all([httpBackend.flushAllExpected(), utils.syncPromise(client)]);
+
+        const room = client.getRoom(roomId);
+        const thread = room.getThread(THREAD_ROOT.event_id);
+        const timelineSet = thread.timelineSet;
+
+        httpBackend.when("GET", "/rooms/!foo%3Abar/context/" + encodeURIComponent(THREAD_ROOT.event_id))
+            .respond(200, {
+                start: "start_token",
+                events_before: [],
+                event: THREAD_ROOT,
+                events_after: [],
+                state: [],
+                end: "end_token",
+            });
+        httpBackend.when("GET", "/rooms/!foo%3Abar/relations/" +
+            encodeURIComponent(THREAD_ROOT.event_id) + "/" +
+            encodeURIComponent(THREAD_RELATION_TYPE.name) + "?limit=20")
+            .respond(200, function() {
+                return {
+                    original_event: THREAD_ROOT,
+                    chunk: [THREAD_REPLY],
+                    // no next batch as this is the oldest end of the timeline
+                };
+            });
+        await Promise.all([
+            client.getEventTimeline(timelineSet, THREAD_ROOT.event_id),
+            httpBackend.flushAllExpected(),
+        ]);
+
+        httpBackend.when("GET", "/sync").respond(200, {
+            next_batch: "s_5_5",
+            rooms: {
+                join: {
+                    [roomId]: {
+                        timeline: {
+                            events: [
+                                SYNC_THREAD_REPLY,
+                            ],
+                            prev_batch: "f_1_2",
+                        },
+                    },
+                },
+            },
+        });
+
+        await Promise.all([httpBackend.flushAllExpected(), utils.syncPromise(client)]);
+
+        expect(thread.liveTimeline.getEvents()[1].event).toEqual(THREAD_REPLY);
+    });
 });

spec/integ/matrix-client-methods.spec.js

@@ -27,11 +27,19 @@ describe("MatrixClient", function() {
     let store = null;
     const userId = "@alice:localhost";
     const accessToken = "aseukfgwef";
+    const idServerDomain = "identity.localhost"; // not a real server
+    const identityAccessToken = "woop-i-am-a-secret";
 
     beforeEach(function() {
         store = new MemoryStore();
 
-        const testClient = new TestClient(userId, "aliceDevice", accessToken, undefined, { store });
+        const testClient = new TestClient(userId, "aliceDevice", accessToken, undefined, {
+            store,
+            identityServer: {
+                getAccessToken: () => Promise.resolve(identityAccessToken),
+            },
+            idBaseUrl: `https://${idServerDomain}`,
+        });
         httpBackend = testClient.httpBackend;
         client = testClient.client;
     });
@@ -42,7 +50,7 @@ describe("MatrixClient", function() {
     });
 
     describe("uploadContent", function() {
-        const buf = new Buffer('hello world');
+        const buf = Buffer.from('hello world');
         it("should upload the file", function() {
             httpBackend.when(
                 "POST", "/_matrix/media/r0/upload",
@@ -474,6 +482,10 @@ describe("MatrixClient", function() {
             return client.initCrypto();
         });
 
+        afterEach(() => {
+            client.stopClient();
+        });
+
         it("should do an HTTP request and then store the keys", function() {
             const ed25519key = "7wG2lzAqbjcyEkOP7O4gU7ItYcn+chKzh5sT/5r2l78";
             // ed25519key = client.getDeviceEd25519Key();
@@ -989,7 +1001,7 @@ describe("MatrixClient", function() {
             };
 
             httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
-                versions: ["r0.5.0"],
+                versions: ["r0.6.0"],
             });
 
             const prom = client.requestRegisterEmailToken("bob@email", "secret", 1);
@@ -1004,6 +1016,64 @@ describe("MatrixClient", function() {
             expect(await prom).toStrictEqual(response);
         });
     });
+
+    describe("inviteByThreePid", () => {
+        it("should supply an id_access_token", async () => {
+            const targetEmail = "gerald@example.org";
+
+            httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
+                versions: ["r0.6.0"],
+            });
+
+            httpBackend.when("POST", "/invite").check(req => {
+                expect(req.data).toStrictEqual({
+                    id_server: idServerDomain,
+                    id_access_token: identityAccessToken,
+                    medium: "email",
+                    address: targetEmail,
+                });
+            }).respond(200, {});
+
+            const prom = client.inviteByThreePid("!room:example.org", "email", targetEmail);
+            await httpBackend.flush();
+            await prom; // returns empty object, so no validation needed
+        });
+    });
+
+    describe("createRoom", () => {
+        it("should populate id_access_token on 3pid invites", async () => {
+            const targetEmail = "gerald@example.org";
+            const response = {
+                room_id: "!room:localhost",
+            };
+            const input = {
+                invite_3pid: [{
+                    // we intentionally exclude the access token here, so it can be populated for us
+                    id_server: idServerDomain,
+                    medium: "email",
+                    address: targetEmail,
+                }],
+            };
+
+            httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
+                versions: ["r0.6.0"],
+            });
+
+            httpBackend.when("POST", "/createRoom").check(req => {
+                expect(req.data).toMatchObject({
+                    invite_3pid: expect.arrayContaining([{
+                        ...input.invite_3pid[0],
+                        id_access_token: identityAccessToken,
+                    }]),
+                });
+                expect(req.data.invite_3pid.length).toBe(1);
+            }).respond(200, response);
+
+            const prom = client.createRoom(input);
+            await httpBackend.flush();
+            expect(await prom).toStrictEqual(response);
+        });
+    });
 });
 
 function withThreadId(event, newThreadId) {

spec/integ/matrix-client-opts.spec.js

@@ -8,7 +8,6 @@ import { MatrixError } from "../../src/http-api";
 
 describe("MatrixClient opts", function() {
     const baseUrl = "http://localhost.or.something";
-    let client = null;
     let httpBackend = null;
     const userId = "@alice:localhost";
     const userB = "@bob:localhost";
@@ -65,6 +64,7 @@ describe("MatrixClient opts", function() {
     });
 
     describe("without opts.store", function() {
+        let client;
         beforeEach(function() {
             client = new MatrixClient({
                 request: httpBackend.requestFn,
@@ -124,6 +124,7 @@ describe("MatrixClient opts", function() {
     });
 
     describe("without opts.scheduler", function() {
+        let client;
         beforeEach(function() {
             client = new MatrixClient({
                 request: httpBackend.requestFn,
@@ -135,6 +136,10 @@ describe("MatrixClient opts", function() {
             });
         });
 
+        afterEach(function() {
+            client.stopClient();
+        });
+
         it("shouldn't retry sending events", function(done) {
             httpBackend.when("PUT", "/txn1").fail(500, new MatrixError({
                 errcode: "M_SOMETHING",

spec/integ/matrix-client-retrying.spec.ts

@@ -1,10 +1,26 @@
-import { EventStatus, RoomEvent } from "../../src/matrix";
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventStatus, RoomEvent, MatrixClient } from "../../src/matrix";
 import { MatrixScheduler } from "../../src/scheduler";
 import { Room } from "../../src/models/room";
 import { TestClient } from "../TestClient";
 
 describe("MatrixClient retrying", function() {
-    let client: TestClient = null;
+    let client: MatrixClient = null;
     let httpBackend: TestClient["httpBackend"] = null;
     let scheduler;
     const userId = "@alice:localhost";

spec/integ/matrix-client-room-timeline.spec.js

@@ -1,5 +1,6 @@
 import * as utils from "../test-utils/test-utils";
 import { EventStatus } from "../../src/models/event";
+import { RoomEvent } from "../../src";
 import { TestClient } from "../TestClient";
 
 describe("MatrixClient room timelines", function() {
@@ -579,7 +580,7 @@ describe("MatrixClient room timelines", function() {
             });
         });
 
-        it("should emit a 'Room.timelineReset' event", function() {
+        it("should emit a `RoomEvent.TimelineReset` event when the sync response is `limited`", function() {
             const eventData = [
                 utils.mkMessage({ user: userId, room: roomId }),
             ];
@@ -608,4 +609,271 @@ describe("MatrixClient room timelines", function() {
             });
         });
     });
+
+    describe('Refresh live timeline', () => {
+        const initialSyncEventData = [
+            utils.mkMessage({ user: userId, room: roomId }),
+            utils.mkMessage({ user: userId, room: roomId }),
+            utils.mkMessage({ user: userId, room: roomId }),
+        ];
+
+        const contextUrl = `/rooms/${encodeURIComponent(roomId)}/context/` +
+        `${encodeURIComponent(initialSyncEventData[2].event_id)}`;
+        const contextResponse = {
+            start: "start_token",
+            events_before: [initialSyncEventData[1], initialSyncEventData[0]],
+            event: initialSyncEventData[2],
+            events_after: [],
+            state: [
+                USER_MEMBERSHIP_EVENT,
+            ],
+            end: "end_token",
+        };
+
+        let room;
+        beforeEach(async () => {
+            setNextSyncData(initialSyncEventData);
+
+            // Create a room from the sync
+            await Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 1),
+            ]);
+
+            // Get the room after the first sync so the room is created
+            room = client.getRoom(roomId);
+            expect(room).toBeTruthy();
+        });
+
+        it('should clear and refresh messages in timeline', async () => {
+            // `/context` request for `refreshLiveTimeline()` -> `getEventTimeline()`
+            // to construct a new timeline from.
+            httpBackend.when("GET", contextUrl)
+                .respond(200, function() {
+                    // The timeline should be cleared at this point in the refresh
+                    expect(room.timeline.length).toEqual(0);
+
+                    return contextResponse;
+                });
+
+            // Refresh the timeline.
+            await Promise.all([
+                room.refreshLiveTimeline(),
+                httpBackend.flushAllExpected(),
+            ]);
+
+            // Make sure the message are visible
+            const resultantEventsInTimeline = room.getUnfilteredTimelineSet().getLiveTimeline().getEvents();
+            const resultantEventIdsInTimeline = resultantEventsInTimeline.map((event) => event.getId());
+            expect(resultantEventIdsInTimeline).toEqual([
+                initialSyncEventData[0].event_id,
+                initialSyncEventData[1].event_id,
+                initialSyncEventData[2].event_id,
+            ]);
+        });
+
+        it('Perfectly merges timelines if a sync finishes while refreshing the timeline', async () => {
+            // `/context` request for `refreshLiveTimeline()` ->
+            // `getEventTimeline()` to construct a new timeline from.
+            //
+            // We only resolve this request after we detect that the timeline
+            // was reset(when it goes blank) and force a sync to happen in the
+            // middle of all of this refresh timeline logic. We want to make
+            // sure the sync pagination still works as expected after messing
+            // the refresh timline logic messes with the pagination tokens.
+            httpBackend.when("GET", contextUrl)
+                .respond(200, () => {
+                    // Now finally return and make the `/context` request respond
+                    return contextResponse;
+                });
+
+            // Wait for the timeline to reset(when it goes blank) which means
+            // it's in the middle of the refrsh logic right before the
+            // `getEventTimeline()` -> `/context`. Then simulate a racey `/sync`
+            // to happen in the middle of all of this refresh timeline logic. We
+            // want to make sure the sync pagination still works as expected
+            // after messing the refresh timline logic messes with the
+            // pagination tokens.
+            //
+            // We define this here so the event listener is in place before we
+            // call `room.refreshLiveTimeline()`.
+            const racingSyncEventData = [
+                utils.mkMessage({ user: userId, room: roomId }),
+            ];
+            const waitForRaceySyncAfterResetPromise = new Promise((resolve, reject) => {
+                let eventFired = false;
+                // Throw a more descriptive error if this part of the test times out.
+                const failTimeout = setTimeout(() => {
+                    if (eventFired) {
+                        reject(new Error(
+                            'TestError: `RoomEvent.TimelineReset` fired but we timed out trying to make' +
+                            'a `/sync` happen in time.',
+                        ));
+                    } else {
+                        reject(new Error(
+                            'TestError: Timed out while waiting for `RoomEvent.TimelineReset` to fire.',
+                        ));
+                    }
+                }, 4000 /* FIXME: Is there a way to reference the current timeout of this test in Jest? */);
+
+                room.on(RoomEvent.TimelineReset, async () => {
+                    try {
+                        eventFired = true;
+
+                        // The timeline should be cleared at this point in the refresh
+                        expect(room.getUnfilteredTimelineSet().getLiveTimeline().getEvents().length).toEqual(0);
+
+                        // Then make a `/sync` happen by sending a message and seeing that it
+                        // shows up (simulate a /sync naturally racing with us).
+                        setNextSyncData(racingSyncEventData);
+                        httpBackend.when("GET", "/sync").respond(200, function() {
+                            return NEXT_SYNC_DATA;
+                        });
+                        await Promise.all([
+                            httpBackend.flush("/sync", 1),
+                            utils.syncPromise(client, 1),
+                        ]);
+                        // Make sure the timeline has the racey sync data
+                        const afterRaceySyncTimelineEvents = room
+                            .getUnfilteredTimelineSet()
+                            .getLiveTimeline()
+                            .getEvents();
+                        const afterRaceySyncTimelineEventIds = afterRaceySyncTimelineEvents
+                            .map((event) => event.getId());
+                        expect(afterRaceySyncTimelineEventIds).toEqual([
+                            racingSyncEventData[0].event_id,
+                        ]);
+
+                        clearTimeout(failTimeout);
+                        resolve();
+                    } catch (err) {
+                        reject(err);
+                    }
+                });
+            });
+
+            // Refresh the timeline. Just start the function, we will wait for
+            // it to finish after the racey sync.
+            const refreshLiveTimelinePromise = room.refreshLiveTimeline();
+
+            await waitForRaceySyncAfterResetPromise;
+
+            await Promise.all([
+                refreshLiveTimelinePromise,
+                // Then flush the remaining `/context` to left the refresh logic complete
+                httpBackend.flushAllExpected(),
+            ]);
+
+            // Make sure sync pagination still works by seeing a new message show up
+            // after refreshing the timeline.
+            const afterRefreshEventData = [
+                utils.mkMessage({ user: userId, room: roomId }),
+            ];
+            setNextSyncData(afterRefreshEventData);
+            httpBackend.when("GET", "/sync").respond(200, function() {
+                return NEXT_SYNC_DATA;
+            });
+            await Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 1),
+            ]);
+
+            // Make sure the timeline includes the the events from the `/sync`
+            // that raced and beat us in the middle of everything and the
+            // `/sync` after the refresh. Since the `/sync` beat us to create
+            // the timeline, `initialSyncEventData` won't be visible unless we
+            // paginate backwards with `/messages`.
+            const resultantEventsInTimeline = room.getUnfilteredTimelineSet().getLiveTimeline().getEvents();
+            const resultantEventIdsInTimeline = resultantEventsInTimeline.map((event) => event.getId());
+            expect(resultantEventIdsInTimeline).toEqual([
+                racingSyncEventData[0].event_id,
+                afterRefreshEventData[0].event_id,
+            ]);
+        });
+
+        it('Timeline recovers after `/context` request to generate new timeline fails', async () => {
+            // `/context` request for `refreshLiveTimeline()` -> `getEventTimeline()`
+            // to construct a new timeline from.
+            httpBackend.when("GET", contextUrl)
+                .respond(500, function() {
+                    // The timeline should be cleared at this point in the refresh
+                    expect(room.timeline.length).toEqual(0);
+
+                    return {
+                        errcode: 'TEST_FAKE_ERROR',
+                        error: 'We purposely intercepted this /context request to make it fail ' +
+                                 'in order to test whether the refresh timeline code is resilient',
+                    };
+                });
+
+            // Refresh the timeline and expect it to fail
+            const settledFailedRefreshPromises = await Promise.allSettled([
+                room.refreshLiveTimeline(),
+                httpBackend.flushAllExpected(),
+            ]);
+            // We only expect `TEST_FAKE_ERROR` here. Anything else is
+            // unexpected and should fail the test.
+            if (settledFailedRefreshPromises[0].status === 'fulfilled') {
+                throw new Error('Expected the /context request to fail with a 500');
+            } else if (settledFailedRefreshPromises[0].reason.errcode !== 'TEST_FAKE_ERROR') {
+                throw settledFailedRefreshPromises[0].reason;
+            }
+
+            // The timeline will be empty after we refresh the timeline and fail
+            // to construct a new timeline.
+            expect(room.timeline.length).toEqual(0);
+
+            // `/messages` request for `refreshLiveTimeline()` ->
+            // `getLatestTimeline()` to construct a new timeline from.
+            httpBackend.when("GET", `/rooms/${encodeURIComponent(roomId)}/messages`)
+                .respond(200, function() {
+                    return {
+                        chunk: [{
+                            // The latest message in the room
+                            event_id: initialSyncEventData[2].event_id,
+                        }],
+                    };
+                });
+            // `/context` request for `refreshLiveTimeline()` ->
+            // `getLatestTimeline()` -> `getEventTimeline()` to construct a new
+            // timeline from.
+            httpBackend.when("GET", contextUrl)
+                .respond(200, function() {
+                    // The timeline should be cleared at this point in the refresh
+                    expect(room.timeline.length).toEqual(0);
+
+                    return contextResponse;
+                });
+
+            // Refresh the timeline again but this time it should pass
+            await Promise.all([
+                room.refreshLiveTimeline(),
+                httpBackend.flushAllExpected(),
+            ]);
+
+            // Make sure sync pagination still works by seeing a new message show up
+            // after refreshing the timeline.
+            const afterRefreshEventData = [
+                utils.mkMessage({ user: userId, room: roomId }),
+            ];
+            setNextSyncData(afterRefreshEventData);
+            httpBackend.when("GET", "/sync").respond(200, function() {
+                return NEXT_SYNC_DATA;
+            });
+            await Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 1),
+            ]);
+
+            // Make sure the message are visible
+            const resultantEventsInTimeline = room.getUnfilteredTimelineSet().getLiveTimeline().getEvents();
+            const resultantEventIdsInTimeline = resultantEventsInTimeline.map((event) => event.getId());
+            expect(resultantEventIdsInTimeline).toEqual([
+                initialSyncEventData[0].event_id,
+                initialSyncEventData[1].event_id,
+                initialSyncEventData[2].event_id,
+                afterRefreshEventData[0].event_id,
+            ]);
+        });
+    });
 });

spec/integ/matrix-client-syncing.spec.js

@@ -1,5 +1,21 @@
-import { MatrixEvent } from "../../src/models/event";
-import { EventTimeline } from "../../src/models/event-timeline";
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventTimeline, MatrixEvent, RoomEvent, RoomStateEvent, RoomMemberEvent } from "../../src";
+import { UNSTABLE_MSC2716_MARKER } from "../../src/@types/event";
 import * as utils from "../test-utils/test-utils";
 import { TestClient } from "../TestClient";
 
@@ -60,6 +76,112 @@ describe("MatrixClient syncing", function() {
                 done();
             });
         });
+
+        it("should emit RoomEvent.MyMembership for invite->leave->invite cycles", async () => {
+            const roomId = "!cycles:example.org";
+
+            // First sync: an invite
+            const inviteSyncRoomSection = {
+                invite: {
+                    [roomId]: {
+                        invite_state: {
+                            events: [{
+                                type: "m.room.member",
+                                state_key: selfUserId,
+                                content: {
+                                    membership: "invite",
+                                },
+                            }],
+                        },
+                    },
+                },
+            };
+            httpBackend.when("GET", "/sync").respond(200, {
+                ...syncData,
+                rooms: inviteSyncRoomSection,
+            });
+
+            // Second sync: a leave (reject of some kind)
+            httpBackend.when("POST", "/leave").respond(200, {});
+            httpBackend.when("GET", "/sync").respond(200, {
+                ...syncData,
+                rooms: {
+                    leave: {
+                        [roomId]: {
+                            account_data: { events: [] },
+                            ephemeral: { events: [] },
+                            state: {
+                                events: [{
+                                    type: "m.room.member",
+                                    state_key: selfUserId,
+                                    content: {
+                                        membership: "leave",
+                                    },
+                                    prev_content: {
+                                        membership: "invite",
+                                    },
+                                    // XXX: And other fields required on an event
+                                }],
+                            },
+                            timeline: {
+                                limited: false,
+                                events: [{
+                                    type: "m.room.member",
+                                    state_key: selfUserId,
+                                    content: {
+                                        membership: "leave",
+                                    },
+                                    prev_content: {
+                                        membership: "invite",
+                                    },
+                                    // XXX: And other fields required on an event
+                                }],
+                            },
+                        },
+                    },
+                },
+            });
+
+            // Third sync: another invite
+            httpBackend.when("GET", "/sync").respond(200, {
+                ...syncData,
+                rooms: inviteSyncRoomSection,
+            });
+
+            // First fire: an initial invite
+            let fires = 0;
+            client.once(RoomEvent.MyMembership, (room, membership, oldMembership) => { // Room, string, string
+                fires++;
+                expect(room.roomId).toBe(roomId);
+                expect(membership).toBe("invite");
+                expect(oldMembership).toBeFalsy();
+
+                // Second fire: a leave
+                client.once(RoomEvent.MyMembership, (room, membership, oldMembership) => {
+                    fires++;
+                    expect(room.roomId).toBe(roomId);
+                    expect(membership).toBe("leave");
+                    expect(oldMembership).toBe("invite");
+
+                    // Third/final fire: a second invite
+                    client.once(RoomEvent.MyMembership, (room, membership, oldMembership) => {
+                        fires++;
+                        expect(room.roomId).toBe(roomId);
+                        expect(membership).toBe("invite");
+                        expect(oldMembership).toBe("leave");
+                    });
+                });
+
+                // For maximum safety, "leave" the room after we register the handler
+                client.leave(roomId);
+            });
+
+            // noinspection ES6MissingAwait
+            client.startClient();
+            await httpBackend.flushAllExpected();
+
+            expect(fires).toBe(3);
+        });
     });
 
     describe("resolving invites to profile info", function() {
@@ -177,7 +299,7 @@ describe("MatrixClient syncing", function() {
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
             let latestFiredName = null;
-            client.on("RoomMember.name", function(event, m) {
+            client.on(RoomMemberEvent.Name, function(event, m) {
                 if (m.userId === userC && m.roomId === roomOne) {
                     latestFiredName = m.name;
                 }
@@ -461,6 +583,477 @@ describe("MatrixClient syncing", function() {
         xit("should update the room topic", function() {
 
         });
+
+        describe("onMarkerStateEvent", () => {
+            const normalMessageEvent = utils.mkMessage({
+                room: roomOne, user: otherUserId, msg: "hello",
+            });
+
+            it('new marker event *NOT* from the room creator in a subsequent syncs ' +
+                'should *NOT* mark the timeline as needing a refresh', async () => {
+                const roomCreateEvent = utils.mkEvent({
+                    type: "m.room.create", room: roomOne, user: otherUserId,
+                    content: {
+                        creator: otherUserId,
+                        room_version: '9',
+                    },
+                });
+                const normalFirstSync = {
+                    next_batch: "batch_token",
+                    rooms: {
+                        join: {},
+                    },
+                };
+                normalFirstSync.rooms.join[roomOne] = {
+                    timeline: {
+                        events: [normalMessageEvent],
+                        prev_batch: "pagTok",
+                    },
+                    state: {
+                        events: [roomCreateEvent],
+                    },
+                };
+
+                const nextSyncData = {
+                    next_batch: "batch_token",
+                    rooms: {
+                        join: {},
+                    },
+                };
+                nextSyncData.rooms.join[roomOne] = {
+                    timeline: {
+                        events: [
+                            // In subsequent syncs, a marker event in timeline
+                            // range should normally trigger
+                            // `timelineNeedsRefresh=true` but this marker isn't
+                            // being sent by the room creator so it has no
+                            // special meaning in existing room versions.
+                            utils.mkEvent({
+                                type: UNSTABLE_MSC2716_MARKER.name,
+                                room: roomOne,
+                                // The important part we're testing is here!
+                                // `userC` is not the room creator.
+                                user: userC,
+                                skey: "",
+                                content: {
+                                    "m.insertion_id": "$abc",
+                                },
+                            }),
+                        ],
+                        prev_batch: "pagTok",
+                    },
+                };
+
+                // Ensure the marker is being sent by someone who is not the room creator
+                // because this is the main thing we're testing in this spec.
+                const markerEvent = nextSyncData.rooms.join[roomOne].timeline.events[0];
+                expect(markerEvent.sender).toBeDefined();
+                expect(markerEvent.sender).not.toEqual(roomCreateEvent.sender);
+
+                httpBackend.when("GET", "/sync").respond(200, normalFirstSync);
+                httpBackend.when("GET", "/sync").respond(200, nextSyncData);
+
+                client.startClient();
+                await Promise.all([
+                    httpBackend.flushAllExpected(),
+                    awaitSyncEvent(2),
+                ]);
+
+                const room = client.getRoom(roomOne);
+                expect(room.getTimelineNeedsRefresh()).toEqual(false);
+            });
+
+            [{
+                label: 'In existing room versions (when the room creator sends the MSC2716 events)',
+                roomVersion: '9',
+            }, {
+                label: 'In a MSC2716 supported room version',
+                roomVersion: 'org.matrix.msc2716v3',
+            }].forEach((testMeta) => {
+                describe(testMeta.label, () => {
+                    const roomCreateEvent = utils.mkEvent({
+                        type: "m.room.create", room: roomOne, user: otherUserId,
+                        content: {
+                            creator: otherUserId,
+                            room_version: testMeta.roomVersion,
+                        },
+                    });
+
+                    const markerEventFromRoomCreator = utils.mkEvent({
+                        type: UNSTABLE_MSC2716_MARKER.name, room: roomOne, user: otherUserId,
+                        skey: "",
+                        content: {
+                            "m.insertion_id": "$abc",
+                        },
+                    });
+
+                    const normalFirstSync = {
+                        next_batch: "batch_token",
+                        rooms: {
+                            join: {},
+                        },
+                    };
+                    normalFirstSync.rooms.join[roomOne] = {
+                        timeline: {
+                            events: [normalMessageEvent],
+                            prev_batch: "pagTok",
+                        },
+                        state: {
+                            events: [roomCreateEvent],
+                        },
+                    };
+
+                    it('no marker event in sync response '+
+                        'should *NOT* mark the timeline as needing a refresh (check for a sane default)', async () => {
+                        const syncData = {
+                            next_batch: "batch_token",
+                            rooms: {
+                                join: {},
+                            },
+                        };
+                        syncData.rooms.join[roomOne] = {
+                            timeline: {
+                                events: [normalMessageEvent],
+                                prev_batch: "pagTok",
+                            },
+                            state: {
+                                events: [roomCreateEvent],
+                            },
+                        };
+
+                        httpBackend.when("GET", "/sync").respond(200, syncData);
+
+                        client.startClient();
+                        await Promise.all([
+                            httpBackend.flushAllExpected(),
+                            awaitSyncEvent(),
+                        ]);
+
+                        const room = client.getRoom(roomOne);
+                        expect(room.getTimelineNeedsRefresh()).toEqual(false);
+                    });
+
+                    it('marker event already sent within timeline range when you join ' +
+                        'should *NOT* mark the timeline as needing a refresh (timelineWasEmpty)', async () => {
+                        const syncData = {
+                            next_batch: "batch_token",
+                            rooms: {
+                                join: {},
+                            },
+                        };
+                        syncData.rooms.join[roomOne] = {
+                            timeline: {
+                                events: [markerEventFromRoomCreator],
+                                prev_batch: "pagTok",
+                            },
+                            state: {
+                                events: [roomCreateEvent],
+                            },
+                        };
+
+                        httpBackend.when("GET", "/sync").respond(200, syncData);
+
+                        client.startClient();
+                        await Promise.all([
+                            httpBackend.flushAllExpected(),
+                            awaitSyncEvent(),
+                        ]);
+
+                        const room = client.getRoom(roomOne);
+                        expect(room.getTimelineNeedsRefresh()).toEqual(false);
+                    });
+
+                    it('marker event already sent before joining (in state) ' +
+                        'should *NOT* mark the timeline as needing a refresh (timelineWasEmpty)', async () => {
+                        const syncData = {
+                            next_batch: "batch_token",
+                            rooms: {
+                                join: {},
+                            },
+                        };
+                        syncData.rooms.join[roomOne] = {
+                            timeline: {
+                                events: [normalMessageEvent],
+                                prev_batch: "pagTok",
+                            },
+                            state: {
+                                events: [
+                                    roomCreateEvent,
+                                    markerEventFromRoomCreator,
+                                ],
+                            },
+                        };
+
+                        httpBackend.when("GET", "/sync").respond(200, syncData);
+
+                        client.startClient();
+                        await Promise.all([
+                            httpBackend.flushAllExpected(),
+                            awaitSyncEvent(),
+                        ]);
+
+                        const room = client.getRoom(roomOne);
+                        expect(room.getTimelineNeedsRefresh()).toEqual(false);
+                    });
+
+                    it('new marker event in a subsequent syncs timeline range ' +
+                        'should mark the timeline as needing a refresh', async () => {
+                        const nextSyncData = {
+                            next_batch: "batch_token",
+                            rooms: {
+                                join: {},
+                            },
+                        };
+                        nextSyncData.rooms.join[roomOne] = {
+                            timeline: {
+                                events: [
+                                    // In subsequent syncs, a marker event in timeline
+                                    // range should trigger `timelineNeedsRefresh=true`
+                                    markerEventFromRoomCreator,
+                                ],
+                                prev_batch: "pagTok",
+                            },
+                        };
+
+                        const markerEventId = nextSyncData.rooms.join[roomOne].timeline.events[0].event_id;
+
+                        // Only do the first sync
+                        httpBackend.when("GET", "/sync").respond(200, normalFirstSync);
+                        client.startClient();
+                        await Promise.all([
+                            httpBackend.flushAllExpected(),
+                            awaitSyncEvent(),
+                        ]);
+
+                        // Get the room after the first sync so the room is created
+                        const room = client.getRoom(roomOne);
+
+                        let emitCount = 0;
+                        room.on(RoomEvent.HistoryImportedWithinTimeline, function(markerEvent, room) {
+                            expect(markerEvent.getId()).toEqual(markerEventId);
+                            expect(room.roomId).toEqual(roomOne);
+                            emitCount += 1;
+                        });
+
+                        // Now do a subsequent sync with the marker event
+                        httpBackend.when("GET", "/sync").respond(200, nextSyncData);
+                        await Promise.all([
+                            httpBackend.flushAllExpected(),
+                            awaitSyncEvent(),
+                        ]);
+
+                        expect(room.getTimelineNeedsRefresh()).toEqual(true);
+                        // Make sure `RoomEvent.HistoryImportedWithinTimeline` was emitted
+                        expect(emitCount).toEqual(1);
+                    });
+
+                    // Mimic a marker event being sent far back in the scroll back but since our last sync
+                    it('new marker event in sync state should mark the timeline as needing a refresh', async () => {
+                        const nextSyncData = {
+                            next_batch: "batch_token",
+                            rooms: {
+                                join: {},
+                            },
+                        };
+                        nextSyncData.rooms.join[roomOne] = {
+                            timeline: {
+                                events: [
+                                    utils.mkMessage({
+                                        room: roomOne, user: otherUserId, msg: "hello again",
+                                    }),
+                                ],
+                                prev_batch: "pagTok",
+                            },
+                            state: {
+                                events: [
+                                    // In subsequent syncs, a marker event in state
+                                    // should trigger `timelineNeedsRefresh=true`
+                                    markerEventFromRoomCreator,
+                                ],
+                            },
+                        };
+
+                        httpBackend.when("GET", "/sync").respond(200, normalFirstSync);
+                        httpBackend.when("GET", "/sync").respond(200, nextSyncData);
+
+                        client.startClient();
+                        await Promise.all([
+                            httpBackend.flushAllExpected(),
+                            awaitSyncEvent(2),
+                        ]);
+
+                        const room = client.getRoom(roomOne);
+                        expect(room.getTimelineNeedsRefresh()).toEqual(true);
+                    });
+                });
+            });
+        });
+
+        // Make sure the state listeners work and events are re-emitted properly from
+        // the client regardless if we reset and refresh the timeline.
+        describe('state listeners and re-registered when RoomEvent.CurrentStateUpdated is fired', () => {
+            const EVENTS = [
+                utils.mkMessage({
+                    room: roomOne, user: userA, msg: "we",
+                }),
+                utils.mkMessage({
+                    room: roomOne, user: userA, msg: "could",
+                }),
+                utils.mkMessage({
+                    room: roomOne, user: userA, msg: "be",
+                }),
+                utils.mkMessage({
+                    room: roomOne, user: userA, msg: "heroes",
+                }),
+            ];
+
+            const SOME_STATE_EVENT = utils.mkEvent({
+                event: true,
+                type: 'org.matrix.test_state',
+                room: roomOne,
+                user: userA,
+                skey: "",
+                content: {
+                    "foo": "bar",
+                },
+            });
+
+            const USER_MEMBERSHIP_EVENT = utils.mkMembership({
+                room: roomOne, mship: "join", user: userA,
+            });
+
+            // This appears to work even if we comment out
+            // `RoomEvent.CurrentStateUpdated` part which triggers everything to
+            // re-listen after the `room.currentState` reference changes. I'm
+            // not sure how it's getting re-emitted.
+            it("should be able to listen to state events even after " +
+               "the timeline is reset during `limited` sync response", async () => {
+                // Create a room from the sync
+                httpBackend.when("GET", "/sync").respond(200, syncData);
+                client.startClient();
+                await Promise.all([
+                    httpBackend.flushAllExpected(),
+                    awaitSyncEvent(),
+                ]);
+
+                // Get the room after the first sync so the room is created
+                const room = client.getRoom(roomOne);
+                expect(room).toBeTruthy();
+
+                let stateEventEmitCount = 0;
+                client.on(RoomStateEvent.Update, () => {
+                    stateEventEmitCount += 1;
+                });
+
+                // Cause `RoomStateEvent.Update` to be fired
+                room.currentState.setStateEvents([SOME_STATE_EVENT]);
+                // Make sure we can listen to the room state events before the reset
+                expect(stateEventEmitCount).toEqual(1);
+
+                // Make a `limited` sync which will cause a `room.resetLiveTimeline`
+                const limitedSyncData = {
+                    next_batch: "batch_token",
+                    rooms: {
+                        join: {},
+                    },
+                };
+                limitedSyncData.rooms.join[roomOne] = {
+                    timeline: {
+                        events: [
+                            utils.mkMessage({
+                                room: roomOne, user: otherUserId, msg: "world",
+                            }),
+                        ],
+                        // The important part, make the sync `limited`
+                        limited: true,
+                        prev_batch: "newerTok",
+                    },
+                };
+                httpBackend.when("GET", "/sync").respond(200, limitedSyncData);
+
+                await Promise.all([
+                    httpBackend.flushAllExpected(),
+                    awaitSyncEvent(),
+                ]);
+
+                // This got incremented again from processing the sync response
+                expect(stateEventEmitCount).toEqual(2);
+
+                // Cause `RoomStateEvent.Update` to be fired
+                room.currentState.setStateEvents([SOME_STATE_EVENT]);
+                // Make sure we can still listen to the room state events after the reset
+                expect(stateEventEmitCount).toEqual(3);
+            });
+
+            // Make sure it re-registers the state listeners after the
+            // `room.currentState` reference changes
+            it("should be able to listen to state events even after " +
+               "refreshing the timeline", async () => {
+                const testClientWithTimelineSupport = new TestClient(
+                    selfUserId,
+                    "DEVICE",
+                    selfAccessToken,
+                    undefined,
+                    { timelineSupport: true },
+                );
+                httpBackend = testClientWithTimelineSupport.httpBackend;
+                httpBackend.when("GET", "/versions").respond(200, {});
+                httpBackend.when("GET", "/pushrules").respond(200, {});
+                httpBackend.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
+                client = testClientWithTimelineSupport.client;
+
+                // Create a room from the sync
+                httpBackend.when("GET", "/sync").respond(200, syncData);
+                client.startClient();
+                await Promise.all([
+                    httpBackend.flushAllExpected(),
+                    awaitSyncEvent(),
+                ]);
+
+                // Get the room after the first sync so the room is created
+                const room = client.getRoom(roomOne);
+                expect(room).toBeTruthy();
+
+                let stateEventEmitCount = 0;
+                client.on(RoomStateEvent.Update, () => {
+                    stateEventEmitCount += 1;
+                });
+
+                // Cause `RoomStateEvent.Update` to be fired
+                room.currentState.setStateEvents([SOME_STATE_EVENT]);
+                // Make sure we can listen to the room state events before the reset
+                expect(stateEventEmitCount).toEqual(1);
+
+                const eventsInRoom = syncData.rooms.join[roomOne].timeline.events;
+                const contextUrl = `/rooms/${encodeURIComponent(roomOne)}/context/` +
+                    `${encodeURIComponent(eventsInRoom[0].event_id)}`;
+                httpBackend.when("GET", contextUrl)
+                    .respond(200, function() {
+                        return {
+                            start: "start_token",
+                            events_before: [EVENTS[1], EVENTS[0]],
+                            event: EVENTS[2],
+                            events_after: [EVENTS[3]],
+                            state: [
+                                USER_MEMBERSHIP_EVENT,
+                            ],
+                            end: "end_token",
+                        };
+                    });
+
+                // Refresh the timeline. This will cause the `room.currentState`
+                // reference to change
+                await Promise.all([
+                    room.refreshLiveTimeline(),
+                    httpBackend.flushAllExpected(),
+                ]);
+
+                // Cause `RoomStateEvent.Update` to be fired
+                room.currentState.setStateEvents([SOME_STATE_EVENT]);
+                // Make sure we can still listen to the room state events after the reset
+                expect(stateEventEmitCount).toEqual(2);
+            });
+        });
     });
 
     describe("timeline", function() {
@@ -516,7 +1109,7 @@ describe("MatrixClient syncing", function() {
                 awaitSyncEvent(),
             ]).then(function() {
                 const room = client.getRoom(roomTwo);
-                expect(room).toBeDefined();
+                expect(room).toBeTruthy();
                 const tok = room.getLiveTimeline()
                     .getPaginationToken(EventTimeline.BACKWARDS);
                 expect(tok).toEqual("roomtwotok");
@@ -545,7 +1138,7 @@ describe("MatrixClient syncing", function() {
 
             let resetCallCount = 0;
             // the token should be set *before* timelineReset is emitted
-            client.on("Room.timelineReset", function(room) {
+            client.on(RoomEvent.TimelineReset, function(room) {
                 resetCallCount++;
 
                 const tl = room.getLiveTimeline();

spec/integ/megolm-backup.spec.ts

@@ -0,0 +1,165 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { Account } from "@matrix-org/olm";
+
+import { logger } from "../../src/logger";
+import { decodeRecoveryKey } from "../../src/crypto/recoverykey";
+import { IKeyBackupInfo, IKeyBackupSession } from "../../src/crypto/keybackup";
+import { TestClient } from "../TestClient";
+import { IEvent } from "../../src";
+import { MatrixEvent, MatrixEventEvent } from "../../src/models/event";
+
+const ROOM_ID = '!ROOM:ID';
+
+const SESSION_ID = 'o+21hSjP+mgEmcfdslPsQdvzWnkdt0Wyo00Kp++R8Kc';
+
+const ENCRYPTED_EVENT: Partial<IEvent> = {
+    type: 'm.room.encrypted',
+    content: {
+        algorithm: 'm.megolm.v1.aes-sha2',
+        sender_key: 'SENDER_CURVE25519',
+        session_id: SESSION_ID,
+        ciphertext: 'AwgAEjD+VwXZ7PoGPRS/H4kwpAsMp/g+WPvJVtPEKE8fmM9IcT/N'
+            + 'CiwPb8PehecDKP0cjm1XO88k6Bw3D17aGiBHr5iBoP7oSw8CXULXAMTkBl'
+            + 'mkufRQq2+d0Giy1s4/Cg5n13jSVrSb2q7VTSv1ZHAFjUCsLSfR0gxqcQs',
+    },
+    room_id: '!ROOM:ID',
+    event_id: '$event1',
+    origin_server_ts: 1507753886000,
+};
+
+const CURVE25519_KEY_BACKUP_DATA: IKeyBackupSession = {
+    first_message_index: 0,
+    forwarded_count: 0,
+    is_verified: false,
+    session_data: {
+        ciphertext: '2z2M7CZ+azAiTHN1oFzZ3smAFFt+LEOYY6h3QO3XXGdw'
+            + '6YpNn/gpHDO6I/rgj1zNd4FoTmzcQgvKdU8kN20u5BWRHxaHTZ'
+            + 'Slne5RxE6vUdREsBgZePglBNyG0AogR/PVdcrv/v18Y6rLM5O9'
+            + 'SELmwbV63uV9Kuu/misMxoqbuqEdG7uujyaEKtjlQsJ5MGPQOy'
+            + 'Syw7XrnesSwF6XWRMxcPGRV0xZr3s9PI350Wve3EncjRgJ9IGF'
+            + 'ru1bcptMqfXgPZkOyGvrphHoFfoK7nY3xMEHUiaTRfRIjq8HNV'
+            + '4o8QY1qmWGnxNBQgOlL8MZlykjg3ULmQ3DtFfQPj/YYGS3jzxv'
+            + 'C+EBjaafmsg+52CTeK3Rswu72PX450BnSZ1i3If4xWAUKvjTpe'
+            + 'Ug5aDLqttOv1pITolTJDw5W/SD+b5rjEKg1CFCHGEGE9wwV3Nf'
+            + 'QHVCQL+dfpd7Or0poy4dqKMAi3g0o3Tg7edIF8d5rREmxaALPy'
+            + 'iie8PHD8mj/5Y0GLqrac4CD6+Mop7eUTzVovprjg',
+        mac: '5lxYBHQU80M',
+        ephemeral: '/Bn0A4UMFwJaDDvh0aEk1XZj3k1IfgCxgFY9P9a0b14',
+    },
+};
+
+const CURVE25519_BACKUP_INFO: IKeyBackupInfo = {
+    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+    version: "1",
+    auth_data: {
+        public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+    },
+};
+
+const RECOVERY_KEY = "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d";
+
+/**
+ * start an Olm session with a given recipient
+ */
+function createOlmSession(olmAccount: Olm.Account, recipientTestClient: TestClient): Promise<Olm.Session> {
+    return recipientTestClient.awaitOneTimeKeyUpload().then((keys) => {
+        const otkId = Object.keys(keys)[0];
+        const otk = keys[otkId];
+
+        const session = new global.Olm.Session();
+        session.create_outbound(
+            olmAccount, recipientTestClient.getDeviceKey(), otk.key,
+        );
+        return session;
+    });
+}
+
+describe("megolm key backups", function() {
+    if (!global.Olm) {
+        logger.warn('not running megolm tests: Olm not present');
+        return;
+    }
+    const Olm = global.Olm;
+
+    let testOlmAccount: Account;
+    let aliceTestClient: TestClient;
+
+    beforeAll(function() {
+        return Olm.init();
+    });
+
+    beforeEach(async function() {
+        aliceTestClient = new TestClient(
+            "@alice:localhost", "xzcvb", "akjgkrgjs",
+        );
+        testOlmAccount = new Olm.Account();
+        testOlmAccount.create();
+        await aliceTestClient.client.initCrypto();
+        aliceTestClient.client.crypto.backupManager.backupInfo = CURVE25519_BACKUP_INFO;
+    });
+
+    afterEach(function() {
+        return aliceTestClient.stop();
+    });
+
+    it("Alice checks key backups when receiving a message she can't decrypt", function() {
+        const syncResponse = {
+            next_batch: 1,
+            rooms: {
+                join: {},
+            },
+        };
+        syncResponse.rooms.join[ROOM_ID] = {
+            timeline: {
+                events: [ENCRYPTED_EVENT],
+            },
+        };
+
+        return aliceTestClient.start().then(() => {
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then(() => {
+            const privkey = decodeRecoveryKey(RECOVERY_KEY);
+            return aliceTestClient.client.crypto.storeSessionBackupPrivateKey(privkey);
+        }).then(() => {
+            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
+            aliceTestClient.expectKeyBackupQuery(
+                ROOM_ID,
+                SESSION_ID,
+                200,
+                CURVE25519_KEY_BACKUP_DATA,
+            );
+            return aliceTestClient.httpBackend.flushAllExpected();
+        }).then(function(): Promise<MatrixEvent> {
+            const room = aliceTestClient.client.getRoom(ROOM_ID);
+            const event = room.getLiveTimeline().getEvents()[0];
+
+            if (event.getContent()) {
+                return Promise.resolve(event);
+            }
+
+            return new Promise((resolve, reject) => {
+                event.once(MatrixEventEvent.Decrypted, (ev) => {
+                    logger.log(`${Date.now()} event ${event.getId()} now decrypted`);
+                    resolve(ev);
+                });
+            });
+        }).then((event) => {
+            expect(event.getContent()).toEqual('testytest');
+        });
+    });
+});

spec/integ/sliding-sync-sdk.spec.ts

@@ -0,0 +1,732 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// eslint-disable-next-line no-restricted-imports
+import MockHttpBackend from "matrix-mock-request";
+import { fail } from "assert";
+
+import { SlidingSync, SlidingSyncEvent, MSC3575RoomData, SlidingSyncState, Extension } from "../../src/sliding-sync";
+import { TestClient } from "../TestClient";
+import { IRoomEvent, IStateEvent } from "../../src/sync-accumulator";
+import {
+    MatrixClient, MatrixEvent, NotificationCountType, JoinRule, MatrixError,
+    EventType, IPushRules, PushRuleKind, TweakName, ClientEvent,
+} from "../../src";
+import { SlidingSyncSdk } from "../../src/sliding-sync-sdk";
+import { SyncState } from "../../src/sync";
+import { IStoredClientOpts } from "../../src/client";
+
+describe("SlidingSyncSdk", () => {
+    let client: MatrixClient = null;
+    let httpBackend: MockHttpBackend = null;
+    let sdk: SlidingSyncSdk = null;
+    let mockSlidingSync: SlidingSync = null;
+    const selfUserId = "@alice:localhost";
+    const selfAccessToken = "aseukfgwef";
+
+    const mockifySlidingSync = (s: SlidingSync): SlidingSync => {
+        s.getList = jest.fn();
+        s.getListData = jest.fn();
+        s.getRoomSubscriptions = jest.fn();
+        s.listLength = jest.fn();
+        s.modifyRoomSubscriptionInfo = jest.fn();
+        s.modifyRoomSubscriptions = jest.fn();
+        s.registerExtension = jest.fn();
+        s.setList = jest.fn();
+        s.setListRanges = jest.fn();
+        s.start = jest.fn();
+        s.stop = jest.fn();
+        s.resend = jest.fn();
+        return s;
+    };
+
+    // shorthand way to make events without filling in all the fields
+    let eventIdCounter = 0;
+    const mkOwnEvent = (evType: string, content: object): IRoomEvent => {
+        eventIdCounter++;
+        return {
+            type: evType,
+            content: content,
+            sender: selfUserId,
+            origin_server_ts: Date.now(),
+            event_id: "$" + eventIdCounter,
+        };
+    };
+    const mkOwnStateEvent = (evType: string, content: object, stateKey?: string): IStateEvent => {
+        eventIdCounter++;
+        return {
+            type: evType,
+            state_key: stateKey,
+            content: content,
+            sender: selfUserId,
+            origin_server_ts: Date.now(),
+            event_id: "$" + eventIdCounter,
+        };
+    };
+    const assertTimelineEvents = (got: MatrixEvent[], want: IRoomEvent[]): void => {
+        expect(got.length).toEqual(want.length);
+        got.forEach((m, i) => {
+            expect(m.getType()).toEqual(want[i].type);
+            expect(m.getSender()).toEqual(want[i].sender);
+            expect(m.getId()).toEqual(want[i].event_id);
+            expect(m.getContent()).toEqual(want[i].content);
+            expect(m.getTs()).toEqual(want[i].origin_server_ts);
+            if (want[i].unsigned) {
+                expect(m.getUnsigned()).toEqual(want[i].unsigned);
+            }
+            const maybeStateEvent = want[i] as IStateEvent;
+            if (maybeStateEvent.state_key) {
+                expect(m.getStateKey()).toEqual(maybeStateEvent.state_key);
+            }
+        });
+    };
+
+    // assign client/httpBackend globals
+    const setupClient = async (testOpts?: Partial<IStoredClientOpts&{withCrypto: boolean}>) => {
+        testOpts = testOpts || {};
+        const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
+        httpBackend = testClient.httpBackend;
+        client = testClient.client;
+        mockSlidingSync = mockifySlidingSync(new SlidingSync("", [], {}, client, 0));
+        if (testOpts.withCrypto) {
+            httpBackend.when("GET", "/room_keys/version").respond(404, {});
+            await client.initCrypto();
+            testOpts.crypto = client.crypto;
+        }
+        httpBackend.when("GET", "/_matrix/client/r0/pushrules").respond(200, {});
+        sdk = new SlidingSyncSdk(mockSlidingSync, client, testOpts);
+    };
+
+    // tear down client/httpBackend globals
+    const teardownClient = () => {
+        client.stopClient();
+        return httpBackend.stop();
+    };
+
+    // find an extension on a SlidingSyncSdk instance
+    const findExtension = (name: string): Extension => {
+        expect(mockSlidingSync.registerExtension).toHaveBeenCalled();
+        const mockFn = mockSlidingSync.registerExtension as jest.Mock;
+        // find the extension
+        for (let i = 0; i < mockFn.mock.calls.length; i++) {
+            const calledExtension = mockFn.mock.calls[i][0] as Extension;
+            if (calledExtension && calledExtension.name() === name) {
+                return calledExtension;
+            }
+        }
+        fail("cannot find extension " + name);
+    };
+
+    describe("sync/stop", () => {
+        beforeAll(async () => {
+            await setupClient();
+        });
+        afterAll(teardownClient);
+        it("can sync()", async () => {
+            const hasSynced = sdk.sync();
+            await httpBackend.flushAllExpected();
+            await hasSynced;
+            expect(mockSlidingSync.start).toBeCalled();
+        });
+        it("can stop()", async () => {
+            sdk.stop();
+            expect(mockSlidingSync.stop).toBeCalled();
+        });
+    });
+
+    describe("rooms", () => {
+        beforeAll(async () => {
+            await setupClient();
+        });
+        afterAll(teardownClient);
+
+        describe("initial", () => {
+            beforeAll(async () => {
+                const hasSynced = sdk.sync();
+                await httpBackend.flushAllExpected();
+                await hasSynced;
+            });
+            // inject some rooms with different fields set.
+            // All rooms are new so they all have initial: true
+            const roomA = "!a_state_and_timeline:localhost";
+            const roomB = "!b_timeline_only:localhost";
+            const roomC = "!c_with_highlight_count:localhost";
+            const roomD = "!d_with_notif_count:localhost";
+            const roomE = "!e_with_invite:localhost";
+            const roomF = "!f_calc_room_name:localhost";
+            const data: Record<string, MSC3575RoomData> = {
+                [roomA]: {
+                    name: "A",
+                    required_state: [
+                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                        mkOwnStateEvent(EventType.RoomName, { name: "A" }, ""),
+                    ],
+                    timeline: [
+                        mkOwnEvent(EventType.RoomMessage, { body: "hello A" }),
+                        mkOwnEvent(EventType.RoomMessage, { body: "world A" }),
+                    ],
+                    initial: true,
+                },
+                [roomB]: {
+                    name: "B",
+                    required_state: [],
+                    timeline: [
+                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                        mkOwnEvent(EventType.RoomMessage, { body: "hello B" }),
+                        mkOwnEvent(EventType.RoomMessage, { body: "world B" }),
+
+                    ],
+                    initial: true,
+                },
+                [roomC]: {
+                    name: "C",
+                    required_state: [],
+                    timeline: [
+                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                        mkOwnEvent(EventType.RoomMessage, { body: "hello C" }),
+                        mkOwnEvent(EventType.RoomMessage, { body: "world C" }),
+                    ],
+                    highlight_count: 5,
+                    initial: true,
+                },
+                [roomD]: {
+                    name: "D",
+                    required_state: [],
+                    timeline: [
+                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                        mkOwnEvent(EventType.RoomMessage, { body: "hello D" }),
+                        mkOwnEvent(EventType.RoomMessage, { body: "world D" }),
+                    ],
+                    notification_count: 5,
+                    initial: true,
+                },
+                [roomE]: {
+                    name: "E",
+                    required_state: [],
+                    timeline: [],
+                    invite_state: [
+                        {
+                            type: EventType.RoomMember,
+                            content: { membership: "invite" },
+                            state_key: selfUserId,
+                            sender: "@bob:localhost",
+                            event_id: "$room_e_invite",
+                            origin_server_ts: 123456,
+                        },
+                        {
+                            type: "m.room.join_rules",
+                            content: { join_rule: "invite" },
+                            state_key: "",
+                            sender: "@bob:localhost",
+                            event_id: "$room_e_join_rule",
+                            origin_server_ts: 123456,
+                        },
+                    ],
+                    initial: true,
+                },
+                [roomF]: {
+                    name: "#foo:localhost",
+                    required_state: [
+                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                        mkOwnStateEvent(EventType.RoomCanonicalAlias, { alias: "#foo:localhost" }, ""),
+                        mkOwnStateEvent(EventType.RoomName, { name: "This should be ignored" }, ""),
+                    ],
+                    timeline: [
+                        mkOwnEvent(EventType.RoomMessage, { body: "hello A" }),
+                        mkOwnEvent(EventType.RoomMessage, { body: "world A" }),
+                    ],
+                    initial: true,
+                },
+            };
+
+            it("can be created with required_state and timeline", () => {
+                mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomA, data[roomA]);
+                const gotRoom = client.getRoom(roomA);
+                expect(gotRoom).toBeDefined();
+                expect(gotRoom.name).toEqual(data[roomA].name);
+                expect(gotRoom.getMyMembership()).toEqual("join");
+                assertTimelineEvents(gotRoom.getLiveTimeline().getEvents().slice(-2), data[roomA].timeline);
+            });
+
+            it("can be created with timeline only", () => {
+                mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomB, data[roomB]);
+                const gotRoom = client.getRoom(roomB);
+                expect(gotRoom).toBeDefined();
+                expect(gotRoom.name).toEqual(data[roomB].name);
+                expect(gotRoom.getMyMembership()).toEqual("join");
+                assertTimelineEvents(gotRoom.getLiveTimeline().getEvents().slice(-5), data[roomB].timeline);
+            });
+
+            it("can be created with a highlight_count", () => {
+                mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomC, data[roomC]);
+                const gotRoom = client.getRoom(roomC);
+                expect(gotRoom).toBeDefined();
+                expect(
+                    gotRoom.getUnreadNotificationCount(NotificationCountType.Highlight),
+                ).toEqual(data[roomC].highlight_count);
+            });
+
+            it("can be created with a notification_count", () => {
+                mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomD, data[roomD]);
+                const gotRoom = client.getRoom(roomD);
+                expect(gotRoom).toBeDefined();
+                expect(
+                    gotRoom.getUnreadNotificationCount(NotificationCountType.Total),
+                ).toEqual(data[roomD].notification_count);
+            });
+
+            it("can be created with invite_state", () => {
+                mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomE, data[roomE]);
+                const gotRoom = client.getRoom(roomE);
+                expect(gotRoom).toBeDefined();
+                expect(gotRoom.getMyMembership()).toEqual("invite");
+                expect(gotRoom.currentState.getJoinRule()).toEqual(JoinRule.Invite);
+            });
+
+            it("uses the 'name' field to caluclate the room name", () => {
+                mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomF, data[roomF]);
+                const gotRoom = client.getRoom(roomF);
+                expect(gotRoom).toBeDefined();
+                expect(
+                    gotRoom.name,
+                ).toEqual(data[roomF].name);
+            });
+
+            describe("updating", () => {
+                it("can update with a new timeline event", async () => {
+                    const newEvent = mkOwnEvent(EventType.RoomMessage, { body: "new event A" });
+                    mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomA, {
+                        timeline: [newEvent],
+                        required_state: [],
+                        name: data[roomA].name,
+                    });
+                    const gotRoom = client.getRoom(roomA);
+                    expect(gotRoom).toBeDefined();
+                    const newTimeline = data[roomA].timeline;
+                    newTimeline.push(newEvent);
+                    assertTimelineEvents(gotRoom.getLiveTimeline().getEvents().slice(-3), newTimeline);
+                });
+
+                it("can update with a new required_state event", async () => {
+                    let gotRoom = client.getRoom(roomB);
+                    expect(gotRoom.getJoinRule()).toEqual(JoinRule.Invite); // default
+                    mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomB, {
+                        required_state: [
+                            mkOwnStateEvent("m.room.join_rules", { join_rule: "restricted" }, ""),
+                        ],
+                        timeline: [],
+                        name: data[roomB].name,
+                    });
+                    gotRoom = client.getRoom(roomB);
+                    expect(gotRoom).toBeDefined();
+                    expect(gotRoom.getJoinRule()).toEqual(JoinRule.Restricted);
+                });
+
+                it("can update with a new highlight_count", async () => {
+                    mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomC, {
+                        name: data[roomC].name,
+                        required_state: [],
+                        timeline: [],
+                        highlight_count: 1,
+                    });
+                    const gotRoom = client.getRoom(roomC);
+                    expect(gotRoom).toBeDefined();
+                    expect(
+                        gotRoom.getUnreadNotificationCount(NotificationCountType.Highlight),
+                    ).toEqual(1);
+                });
+
+                it("can update with a new notification_count", async () => {
+                    mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomD, {
+                        name: data[roomD].name,
+                        required_state: [],
+                        timeline: [],
+                        notification_count: 1,
+                    });
+                    const gotRoom = client.getRoom(roomD);
+                    expect(gotRoom).toBeDefined();
+                    expect(
+                        gotRoom.getUnreadNotificationCount(NotificationCountType.Total),
+                    ).toEqual(1);
+                });
+            });
+        });
+    });
+
+    describe("lifecycle", () => {
+        beforeAll(async () => {
+            await setupClient();
+            const hasSynced = sdk.sync();
+            await httpBackend.flushAllExpected();
+            await hasSynced;
+        });
+        const FAILED_SYNC_ERROR_THRESHOLD = 3; // would be nice to export the const in the actual class...
+
+        it("emits SyncState.Reconnecting when < FAILED_SYNC_ERROR_THRESHOLD & SyncState.Error when over", async () => {
+            mockSlidingSync.emit(
+                SlidingSyncEvent.Lifecycle, SlidingSyncState.Complete,
+                { pos: "h", lists: [], rooms: {}, extensions: {} }, null,
+            );
+            expect(sdk.getSyncState()).toEqual(SyncState.Syncing);
+
+            mockSlidingSync.emit(
+                SlidingSyncEvent.Lifecycle, SlidingSyncState.RequestFinished, null, new Error("generic"),
+            );
+            expect(sdk.getSyncState()).toEqual(SyncState.Reconnecting);
+
+            for (let i = 0; i < FAILED_SYNC_ERROR_THRESHOLD; i++) {
+                mockSlidingSync.emit(
+                    SlidingSyncEvent.Lifecycle, SlidingSyncState.RequestFinished, null, new Error("generic"),
+                );
+            }
+            expect(sdk.getSyncState()).toEqual(SyncState.Error);
+        });
+
+        it("emits SyncState.Syncing after a previous SyncState.Error", async () => {
+            mockSlidingSync.emit(
+                SlidingSyncEvent.Lifecycle,
+                SlidingSyncState.Complete,
+                { pos: "i", lists: [], rooms: {}, extensions: {} },
+                null,
+            );
+            expect(sdk.getSyncState()).toEqual(SyncState.Syncing);
+        });
+
+        it("emits SyncState.Error immediately when receiving M_UNKNOWN_TOKEN and stops syncing", async () => {
+            expect(mockSlidingSync.stop).not.toBeCalled();
+            mockSlidingSync.emit(SlidingSyncEvent.Lifecycle, SlidingSyncState.RequestFinished, null, new MatrixError({
+                errcode: "M_UNKNOWN_TOKEN",
+                message: "Oh no your access token is no longer valid",
+            }));
+            expect(sdk.getSyncState()).toEqual(SyncState.Error);
+            expect(mockSlidingSync.stop).toBeCalled();
+        });
+    });
+
+    describe("opts", () => {
+        afterEach(teardownClient);
+        it("can resolveProfilesToInvites", async () => {
+            await setupClient({
+                resolveInvitesToProfiles: true,
+            });
+            const roomId = "!resolveProfilesToInvites:localhost";
+            const invitee = "@invitee:localhost";
+            const inviteeProfile = {
+                avatar_url: "mxc://foobar",
+                displayname: "The Invitee",
+            };
+            httpBackend.when("GET", "/profile").respond(200, inviteeProfile);
+            mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomId, {
+                initial: true,
+                name: "Room with Invite",
+                required_state: [],
+                timeline: [
+                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                    mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                    mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                    mkOwnStateEvent(EventType.RoomMember, { membership: "invite" }, invitee),
+                ],
+            });
+            await httpBackend.flush("/profile", 1, 1000);
+            const room = client.getRoom(roomId);
+            expect(room).toBeDefined();
+            const inviteeMember = room.getMember(invitee);
+            expect(inviteeMember).toBeDefined();
+            expect(inviteeMember.getMxcAvatarUrl()).toEqual(inviteeProfile.avatar_url);
+            expect(inviteeMember.name).toEqual(inviteeProfile.displayname);
+        });
+    });
+
+    describe("ExtensionE2EE", () => {
+        let ext: Extension;
+        beforeAll(async () => {
+            await setupClient({
+                withCrypto: true,
+            });
+            const hasSynced = sdk.sync();
+            await httpBackend.flushAllExpected();
+            await hasSynced;
+            ext = findExtension("e2ee");
+        });
+        afterAll(async () => {
+            // needed else we do some async operations in the background which can cause Jest to whine:
+            // "Cannot log after tests are done. Did you forget to wait for something async in your test?"
+            // Attempted to log "Saving device tracking data null"."
+            client.crypto.stop();
+        });
+        it("gets enabled on the initial request only", () => {
+            expect(ext.onRequest(true)).toEqual({
+                enabled: true,
+            });
+            expect(ext.onRequest(false)).toEqual(undefined);
+        });
+        it("can update device lists", () => {
+            ext.onResponse({
+                device_lists: {
+                    changed: ["@alice:localhost"],
+                    left: ["@bob:localhost"],
+                },
+            });
+            // TODO: more assertions?
+        });
+        it("can update OTK counts", () => {
+            client.crypto.updateOneTimeKeyCount = jest.fn();
+            ext.onResponse({
+                device_one_time_keys_count: {
+                    signed_curve25519: 42,
+                },
+            });
+            expect(client.crypto.updateOneTimeKeyCount).toHaveBeenCalledWith(42);
+            ext.onResponse({
+                device_one_time_keys_count: {
+                    not_signed_curve25519: 42,
+                    // missing field -> default to 0
+                },
+            });
+            expect(client.crypto.updateOneTimeKeyCount).toHaveBeenCalledWith(0);
+        });
+        it("can update fallback keys", () => {
+            ext.onResponse({
+                device_unused_fallback_key_types: ["signed_curve25519"],
+            });
+            expect(client.crypto.getNeedsNewFallback()).toEqual(false);
+            ext.onResponse({
+                device_unused_fallback_key_types: ["not_signed_curve25519"],
+            });
+            expect(client.crypto.getNeedsNewFallback()).toEqual(true);
+        });
+    });
+    describe("ExtensionAccountData", () => {
+        let ext: Extension;
+        beforeAll(async () => {
+            await setupClient();
+            const hasSynced = sdk.sync();
+            await httpBackend.flushAllExpected();
+            await hasSynced;
+            ext = findExtension("account_data");
+        });
+        it("gets enabled on the initial request only", () => {
+            expect(ext.onRequest(true)).toEqual({
+                enabled: true,
+            });
+            expect(ext.onRequest(false)).toEqual(undefined);
+        });
+        it("processes global account data", async () => {
+            const globalType = "global_test";
+            const globalContent = {
+                info: "here",
+            };
+            let globalData = client.getAccountData(globalType);
+            expect(globalData).toBeUndefined();
+            ext.onResponse({
+                global: [
+                    {
+                        type: globalType,
+                        content: globalContent,
+                    },
+                ],
+            });
+            globalData = client.getAccountData(globalType);
+            expect(globalData).toBeDefined();
+            expect(globalData.getContent()).toEqual(globalContent);
+        });
+        it("processes rooms account data", async () => {
+            const roomId = "!room:id";
+            mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomId, {
+                name: "Room with account data",
+                required_state: [],
+                timeline: [
+                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                    mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                    mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                    mkOwnEvent(EventType.RoomMessage, { body: "hello" }),
+
+                ],
+                initial: true,
+            });
+            const roomContent = {
+                foo: "bar",
+            };
+            const roomType = "test";
+            ext.onResponse({
+                rooms: {
+                    [roomId]: [
+                        {
+                            type: roomType,
+                            content: roomContent,
+                        },
+                    ],
+                },
+            });
+            const room = client.getRoom(roomId);
+            expect(room).toBeDefined();
+            const event = room.getAccountData(roomType);
+            expect(event).toBeDefined();
+            expect(event.getContent()).toEqual(roomContent);
+        });
+        it("doesn't crash for unknown room account data", async () => {
+            const unknownRoomId = "!unknown:id";
+            const roomType = "tester";
+            ext.onResponse({
+                rooms: {
+                    [unknownRoomId]: [
+                        {
+                            type: roomType,
+                            content: {
+                                foo: "Bar",
+                            },
+                        },
+                    ],
+                },
+            });
+            const room = client.getRoom(unknownRoomId);
+            expect(room).toBeNull();
+            expect(client.getAccountData(roomType)).toBeUndefined();
+        });
+        it("can update push rules via account data", async () => {
+            const roomId = "!foo:bar";
+            const pushRulesContent: IPushRules = {
+                global: {
+                    [PushRuleKind.RoomSpecific]: [{
+                        enabled: true,
+                        default: true,
+                        pattern: "monkey",
+                        actions: [
+                            {
+                                set_tweak: TweakName.Sound,
+                                value: "default",
+                            },
+                        ],
+                        rule_id: roomId,
+                    }],
+                },
+            };
+            let pushRule = client.getRoomPushRule("global", roomId);
+            expect(pushRule).toBeUndefined();
+            ext.onResponse({
+                global: [
+                    {
+                        type: EventType.PushRules,
+                        content: pushRulesContent,
+                    },
+                ],
+            });
+            pushRule = client.getRoomPushRule("global", roomId);
+            expect(pushRule).toEqual(pushRulesContent.global[PushRuleKind.RoomSpecific][0]);
+        });
+    });
+    describe("ExtensionToDevice", () => {
+        let ext: Extension;
+        beforeAll(async () => {
+            await setupClient();
+            const hasSynced = sdk.sync();
+            await httpBackend.flushAllExpected();
+            await hasSynced;
+            ext = findExtension("to_device");
+        });
+        it("gets enabled with a limit on the initial request only", () => {
+            const reqJson: any = ext.onRequest(true);
+            expect(reqJson.enabled).toEqual(true);
+            expect(reqJson.limit).toBeGreaterThan(0);
+            expect(reqJson.since).toBeUndefined();
+        });
+        it("updates the since value", async () => {
+            ext.onResponse({
+                next_batch: "12345",
+                events: [],
+            });
+            expect(ext.onRequest(false)).toEqual({
+                since: "12345",
+            });
+        });
+        it("can handle missing fields", async () => {
+            ext.onResponse({
+                next_batch: "23456",
+                // no events array
+            });
+        });
+        it("emits to-device events on the client", async () => {
+            const toDeviceType = "custom_test";
+            const toDeviceContent = {
+                foo: "bar",
+            };
+            let called = false;
+            client.once(ClientEvent.ToDeviceEvent, (ev) => {
+                expect(ev.getContent()).toEqual(toDeviceContent);
+                expect(ev.getType()).toEqual(toDeviceType);
+                called = true;
+            });
+            ext.onResponse({
+                next_batch: "34567",
+                events: [
+                    {
+                        type: toDeviceType,
+                        content: toDeviceContent,
+                    },
+                ],
+            });
+            expect(called).toBe(true);
+        });
+        it("can cancel key verification requests", async () => {
+            const seen: Record<string, boolean> = {};
+            client.on(ClientEvent.ToDeviceEvent, (ev) => {
+                const evType = ev.getType();
+                expect(seen[evType]).toBeFalsy();
+                seen[evType] = true;
+                if (evType === "m.key.verification.start" || evType === "m.key.verification.request") {
+                    expect(ev.isCancelled()).toEqual(true);
+                } else {
+                    expect(ev.isCancelled()).toEqual(false);
+                }
+            });
+            ext.onResponse({
+                next_batch: "45678",
+                events: [
+                    // someone tries to verify keys
+                    {
+                        type: "m.key.verification.start",
+                        content: {
+                            transaction_id: "a",
+                        },
+                    },
+                    {
+                        type: "m.key.verification.request",
+                        content: {
+                            transaction_id: "a",
+                        },
+                    },
+                    // then gives up
+                    {
+                        type: "m.key.verification.cancel",
+                        content: {
+                            transaction_id: "a",
+                        },
+                    },
+                ],
+            });
+        });
+    });
+});

spec/integ/sliding-sync.spec.ts

@@ -0,0 +1,758 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// eslint-disable-next-line no-restricted-imports
+import EventEmitter from "events";
+import MockHttpBackend from "matrix-mock-request";
+
+import { SlidingSync, SlidingSyncState, ExtensionState, SlidingSyncEvent } from "../../src/sliding-sync";
+import { TestClient } from "../TestClient";
+import { logger } from "../../src/logger";
+import { MatrixClient } from "../../src";
+import { sleep } from "../../src/utils";
+
+/**
+ * Tests for sliding sync. These tests are broken down into sub-tests which are reliant upon one another.
+ * Each test suite (describe block) uses a single MatrixClient/HTTPBackend and a single SlidingSync class.
+ * Each test will call different functions on SlidingSync which may depend on state from previous tests.
+ */
+describe("SlidingSync", () => {
+    let client: MatrixClient = null;
+    let httpBackend: MockHttpBackend = null;
+    const selfUserId = "@alice:localhost";
+    const selfAccessToken = "aseukfgwef";
+    const proxyBaseUrl = "http://localhost:8008";
+    const syncUrl = proxyBaseUrl + "/_matrix/client/unstable/org.matrix.msc3575/sync";
+
+    // assign client/httpBackend globals
+    const setupClient = () => {
+        const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
+        httpBackend = testClient.httpBackend;
+        client = testClient.client;
+    };
+
+    // tear down client/httpBackend globals
+    const teardownClient = () => {
+        httpBackend.verifyNoOutstandingExpectation();
+        client.stopClient();
+        return httpBackend.stop();
+    };
+
+    describe("start/stop", () => {
+        beforeAll(setupClient);
+        afterAll(teardownClient);
+        let slidingSync: SlidingSync;
+
+        it("should start the sync loop upon calling start()", async () => {
+            slidingSync = new SlidingSync(proxyBaseUrl, [], {}, client, 1);
+            const fakeResp = {
+                pos: "a",
+                lists: [],
+                rooms: {},
+                extensions: {},
+            };
+            httpBackend.when("POST", syncUrl).respond(200, fakeResp);
+            const p = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state, resp, err) => {
+                expect(state).toEqual(SlidingSyncState.RequestFinished);
+                expect(resp).toEqual(fakeResp);
+                expect(err).toBeFalsy();
+                return true;
+            });
+            slidingSync.start();
+            await httpBackend.flushAllExpected();
+            await p;
+        });
+
+        it("should stop the sync loop upon calling stop()", () => {
+            slidingSync.stop();
+            httpBackend.verifyNoOutstandingExpectation();
+        });
+    });
+
+    describe("room subscriptions", () => {
+        beforeAll(setupClient);
+        afterAll(teardownClient);
+        const roomId = "!foo:bar";
+        const anotherRoomID = "!another:room";
+        let roomSubInfo = {
+            timeline_limit: 1,
+            required_state: [
+                ["m.room.name", ""],
+            ],
+        };
+        const wantRoomData = {
+            name: "foo bar",
+            required_state: [],
+            timeline: [],
+        };
+
+        let slidingSync: SlidingSync;
+
+        it("should be able to subscribe to a room", async () => {
+            // add the subscription
+            slidingSync = new SlidingSync(proxyBaseUrl, [], roomSubInfo, client, 1);
+            slidingSync.modifyRoomSubscriptions(new Set([roomId]));
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("room sub", body);
+                expect(body.room_subscriptions).toBeTruthy();
+                expect(body.room_subscriptions[roomId]).toEqual(roomSubInfo);
+            }).respond(200, {
+                pos: "a",
+                lists: [],
+                extensions: {},
+                rooms: {
+                    [roomId]: wantRoomData,
+                },
+            });
+
+            const p = listenUntil(slidingSync, "SlidingSync.RoomData", (gotRoomId, gotRoomData) => {
+                expect(gotRoomId).toEqual(roomId);
+                expect(gotRoomData).toEqual(wantRoomData);
+                return true;
+            });
+            slidingSync.start();
+            await httpBackend.flushAllExpected();
+            await p;
+        });
+
+        it("should be possible to adjust room subscription info whilst syncing", async () => {
+            // listen for updated request
+            const newSubInfo = {
+                timeline_limit: 100,
+                required_state: [
+                    ["m.room.member", "*"],
+                ],
+            };
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("adjusted sub", body);
+                expect(body.room_subscriptions).toBeTruthy();
+                expect(body.room_subscriptions[roomId]).toEqual(newSubInfo);
+            }).respond(200, {
+                pos: "a",
+                lists: [],
+                extensions: {},
+                rooms: {
+                    [roomId]: wantRoomData,
+                },
+            });
+
+            const p = listenUntil(slidingSync, "SlidingSync.RoomData", (gotRoomId, gotRoomData) => {
+                expect(gotRoomId).toEqual(roomId);
+                expect(gotRoomData).toEqual(wantRoomData);
+                return true;
+            });
+
+            slidingSync.modifyRoomSubscriptionInfo(newSubInfo);
+            await httpBackend.flushAllExpected();
+            await p;
+            // need to set what the new subscription info is for subsequent tests
+            roomSubInfo = newSubInfo;
+        });
+
+        it("should be possible to add room subscriptions whilst syncing", async () => {
+            // listen for updated request
+            const anotherRoomData = {
+                name: "foo bar 2",
+                room_id: anotherRoomID,
+                // we should not fall over if fields are missing.
+                // required_state: [],
+                // timeline: [],
+            };
+            const anotherRoomDataFixed = {
+                name: anotherRoomData.name,
+                room_id: anotherRoomID,
+                required_state: [],
+                timeline: [],
+            };
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("new subs", body);
+                expect(body.room_subscriptions).toBeTruthy();
+                // only the new room is sent, the other is sticky
+                expect(body.room_subscriptions[anotherRoomID]).toEqual(roomSubInfo);
+                expect(body.room_subscriptions[roomId]).toBeUndefined();
+            }).respond(200, {
+                pos: "b",
+                lists: [],
+                extensions: {},
+                rooms: {
+                    [anotherRoomID]: anotherRoomData,
+                },
+            });
+
+            const p = listenUntil(slidingSync, "SlidingSync.RoomData", (gotRoomId, gotRoomData) => {
+                expect(gotRoomId).toEqual(anotherRoomID);
+                expect(gotRoomData).toEqual(anotherRoomDataFixed);
+                return true;
+            });
+
+            const subs = slidingSync.getRoomSubscriptions();
+            subs.add(anotherRoomID);
+            slidingSync.modifyRoomSubscriptions(subs);
+            await httpBackend.flushAllExpected();
+            await p;
+        });
+
+        it("should be able to unsubscribe from a room", async () => {
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("unsub request", body);
+                expect(body.room_subscriptions).toBeFalsy();
+                expect(body.unsubscribe_rooms).toEqual([roomId]);
+            }).respond(200, {
+                pos: "b",
+                lists: [],
+            });
+
+            const p = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+
+            // remove the subscription for the first room
+            slidingSync.modifyRoomSubscriptions(new Set([anotherRoomID]));
+
+            await httpBackend.flushAllExpected();
+            await p;
+
+            slidingSync.stop();
+        });
+    });
+
+    describe("lists", () => {
+        beforeAll(setupClient);
+        afterAll(teardownClient);
+
+        const roomA = "!a:localhost";
+        const roomB = "!b:localhost";
+        const roomC = "!c:localhost";
+        const rooms = {
+            [roomA]: {
+                name: "A",
+                required_state: [],
+                timeline: [],
+            },
+            [roomB]: {
+                name: "B",
+                required_state: [],
+                timeline: [],
+            },
+            [roomC]: {
+                name: "C",
+                required_state: [],
+                timeline: [],
+            },
+        };
+        const newRanges = [[0, 2], [3, 5]];
+
+        let slidingSync: SlidingSync;
+        it("should be possible to subscribe to a list", async () => {
+            // request first 3 rooms
+            const listReq = {
+                ranges: [[0, 2]],
+                sort: ["by_name"],
+                timeline_limit: 1,
+                required_state: [
+                    ["m.room.topic", ""],
+                ],
+                filters: {
+                    is_dm: true,
+                },
+            };
+            slidingSync = new SlidingSync(proxyBaseUrl, [listReq], {}, client, 1);
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("list", body);
+                expect(body.lists).toBeTruthy();
+                expect(body.lists[0]).toEqual(listReq);
+            }).respond(200, {
+                pos: "a",
+                lists: [{
+                    count: 500,
+                    ops: [{
+                        op: "SYNC",
+                        range: [0, 2],
+                        room_ids: Object.keys(rooms),
+                    }],
+                }],
+                rooms: rooms,
+            });
+            const listenerData = {};
+            const dataListener = (roomId, roomData) => {
+                expect(listenerData[roomId]).toBeFalsy();
+                listenerData[roomId] = roomData;
+            };
+            slidingSync.on(SlidingSyncEvent.RoomData, dataListener);
+            const responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+            slidingSync.start();
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+
+            expect(listenerData[roomA]).toEqual(rooms[roomA]);
+            expect(listenerData[roomB]).toEqual(rooms[roomB]);
+            expect(listenerData[roomC]).toEqual(rooms[roomC]);
+
+            expect(slidingSync.listLength()).toEqual(1);
+            slidingSync.off(SlidingSyncEvent.RoomData, dataListener);
+        });
+
+        it("should be possible to retrieve list data", () => {
+            expect(slidingSync.getList(0)).toBeDefined();
+            expect(slidingSync.getList(5)).toBeNull();
+            expect(slidingSync.getListData(5)).toBeNull();
+            const syncData = slidingSync.getListData(0);
+            expect(syncData.joinedCount).toEqual(500); // from previous test
+            expect(syncData.roomIndexToRoomId).toEqual({
+                0: roomA,
+                1: roomB,
+                2: roomC,
+            });
+        });
+
+        it("should be possible to adjust list ranges", async () => {
+            // modify the list ranges
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("next ranges", body.lists[0].ranges);
+                expect(body.lists).toBeTruthy();
+                expect(body.lists[0]).toEqual({
+                    // only the ranges should be sent as the rest are unchanged and sticky
+                    ranges: newRanges,
+                });
+            }).respond(200, {
+                pos: "b",
+                lists: [{
+                    count: 500,
+                    ops: [{
+                        op: "SYNC",
+                        range: [0, 2],
+                        room_ids: Object.keys(rooms),
+                    }],
+                }],
+            });
+
+            const responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.RequestFinished;
+            });
+            slidingSync.setListRanges(0, newRanges);
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+        });
+
+        it("should be possible to add an extra list", async () => {
+            // add extra list
+            const extraListReq = {
+                ranges: [[0, 100]],
+                sort: ["by_name"],
+                filters: {
+                    "is_dm": true,
+                },
+            };
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("extra list", body);
+                expect(body.lists).toBeTruthy();
+                expect(body.lists[0]).toEqual({
+                    // only the ranges should be sent as the rest are unchanged and sticky
+                    ranges: newRanges,
+                });
+                expect(body.lists[1]).toEqual(extraListReq);
+            }).respond(200, {
+                pos: "c",
+                lists: [
+                    {
+                        count: 500,
+                    },
+                    {
+                        count: 50,
+                        ops: [{
+                            op: "SYNC",
+                            range: [0, 2],
+                            room_ids: Object.keys(rooms),
+                        }],
+                    },
+                ],
+            });
+            listenUntil(slidingSync, "SlidingSync.List", (listIndex, joinedCount, roomIndexToRoomId) => {
+                expect(listIndex).toEqual(1);
+                expect(joinedCount).toEqual(50);
+                expect(roomIndexToRoomId).toEqual({
+                    0: roomA,
+                    1: roomB,
+                    2: roomC,
+                });
+                return true;
+            });
+            const responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+            slidingSync.setList(1, extraListReq);
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+        });
+
+        it("should be possible to get list DELETE/INSERTs", async () => {
+            // move C (2) to A (0)
+            httpBackend.when("POST", syncUrl).respond(200, {
+                pos: "e",
+                lists: [{
+                    count: 500,
+                    ops: [{
+                        op: "DELETE",
+                        index: 2,
+                    }, {
+                        op: "INSERT",
+                        index: 0,
+                        room_id: roomC,
+                    }],
+                },
+                {
+                    count: 50,
+                }],
+            });
+            let listPromise = listenUntil(slidingSync, "SlidingSync.List",
+                (listIndex, joinedCount, roomIndexToRoomId) => {
+                    expect(listIndex).toEqual(0);
+                    expect(joinedCount).toEqual(500);
+                    expect(roomIndexToRoomId).toEqual({
+                        0: roomC,
+                        1: roomA,
+                        2: roomB,
+                    });
+                    return true;
+                });
+            let responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+            await listPromise;
+
+            // move C (0) back to A (2)
+            httpBackend.when("POST", syncUrl).respond(200, {
+                pos: "f",
+                lists: [{
+                    count: 500,
+                    ops: [{
+                        op: "DELETE",
+                        index: 0,
+                    }, {
+                        op: "INSERT",
+                        index: 2,
+                        room_id: roomC,
+                    }],
+                },
+                {
+                    count: 50,
+                }],
+            });
+            listPromise = listenUntil(slidingSync, "SlidingSync.List",
+                (listIndex, joinedCount, roomIndexToRoomId) => {
+                    expect(listIndex).toEqual(0);
+                    expect(joinedCount).toEqual(500);
+                    expect(roomIndexToRoomId).toEqual({
+                        0: roomA,
+                        1: roomB,
+                        2: roomC,
+                    });
+                    return true;
+                });
+            responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+            await listPromise;
+        });
+
+        it("should ignore invalid list indexes", async () => {
+            httpBackend.when("POST", syncUrl).respond(200, {
+                pos: "e",
+                lists: [{
+                    count: 500,
+                    ops: [{
+                        op: "DELETE",
+                        index: 2324324,
+                    }],
+                },
+                {
+                    count: 50,
+                }],
+            });
+            const listPromise = listenUntil(slidingSync, "SlidingSync.List",
+                (listIndex, joinedCount, roomIndexToRoomId) => {
+                    expect(listIndex).toEqual(0);
+                    expect(joinedCount).toEqual(500);
+                    expect(roomIndexToRoomId).toEqual({
+                        0: roomA,
+                        1: roomB,
+                        2: roomC,
+                    });
+                    return true;
+                });
+            const responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+            await listPromise;
+        });
+
+        it("should be possible to update a list", async () => {
+            httpBackend.when("POST", syncUrl).respond(200, {
+                pos: "g",
+                lists: [{
+                    count: 42,
+                    ops: [
+                        {
+                            op: "INVALIDATE",
+                            range: [0, 2],
+                        },
+                        {
+                            op: "SYNC",
+                            range: [0, 1],
+                            room_ids: [roomB, roomC],
+                        },
+                    ],
+                },
+                {
+                    count: 50,
+                }],
+            });
+            // update the list with a new filter
+            slidingSync.setList(0, {
+                filters: {
+                    is_encrypted: true,
+                },
+                ranges: [[0, 100]],
+            });
+            const listPromise = listenUntil(slidingSync, "SlidingSync.List",
+                (listIndex, joinedCount, roomIndexToRoomId) => {
+                    expect(listIndex).toEqual(0);
+                    expect(joinedCount).toEqual(42);
+                    expect(roomIndexToRoomId).toEqual({
+                        0: roomB,
+                        1: roomC,
+                    });
+                    return true;
+                });
+            const responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+            await listPromise;
+            slidingSync.stop();
+        });
+    });
+
+    describe("extensions", () => {
+        beforeAll(setupClient);
+        afterAll(teardownClient);
+        let slidingSync: SlidingSync;
+        const extReq = {
+            foo: "bar",
+        };
+        const extResp = {
+            baz: "quuz",
+        };
+
+        // Pre-extensions get called BEFORE processing the sync response
+        const preExtName = "foobar";
+        let onPreExtensionRequest;
+        let onPreExtensionResponse;
+
+        // Post-extensions get called AFTER processing the sync response
+        const postExtName = "foobar2";
+        let onPostExtensionRequest;
+        let onPostExtensionResponse;
+
+        const extPre = {
+            name: () => preExtName,
+            onRequest: (initial) => { return onPreExtensionRequest(initial); },
+            onResponse: (res) => { return onPreExtensionResponse(res); },
+            when: () => ExtensionState.PreProcess,
+        };
+        const extPost = {
+            name: () => postExtName,
+            onRequest: (initial) => { return onPostExtensionRequest(initial); },
+            onResponse: (res) => { return onPostExtensionResponse(res); },
+            when: () => ExtensionState.PostProcess,
+        };
+
+        it("should be able to register an extension", async () => {
+            slidingSync = new SlidingSync(proxyBaseUrl, [], {}, client, 1);
+            slidingSync.registerExtension(extPre);
+
+            const callbackOrder = [];
+            let extensionOnResponseCalled = false;
+            onPreExtensionRequest = () => {
+                return extReq;
+            };
+            onPreExtensionResponse = (resp) => {
+                extensionOnResponseCalled = true;
+                callbackOrder.push("onPreExtensionResponse");
+                expect(resp).toEqual(extResp);
+            };
+
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("ext req", body);
+                expect(body.extensions).toBeTruthy();
+                expect(body.extensions[preExtName]).toEqual(extReq);
+            }).respond(200, {
+                pos: "a",
+                ops: [],
+                counts: [],
+                extensions: {
+                    [preExtName]: extResp,
+                },
+            });
+
+            const p = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state, resp, err) => {
+                if (state === SlidingSyncState.Complete) {
+                    callbackOrder.push("Lifecycle");
+                    return true;
+                }
+            });
+            slidingSync.start();
+            await httpBackend.flushAllExpected();
+            await p;
+            expect(extensionOnResponseCalled).toBe(true);
+            expect(callbackOrder).toEqual(["onPreExtensionResponse", "Lifecycle"]);
+        });
+
+        it("should be able to send nothing in an extension request/response", async () => {
+            onPreExtensionRequest = () => {
+                return undefined;
+            };
+            let responseCalled = false;
+            onPreExtensionResponse = (resp) => {
+                responseCalled = true;
+            };
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("ext req nothing", body);
+                expect(body.extensions).toBeTruthy();
+                expect(body.extensions[preExtName]).toBeUndefined();
+            }).respond(200, {
+                pos: "a",
+                ops: [],
+                counts: [],
+                extensions: {},
+            });
+            // we need to resend as sliding sync will already have a buffered request with the old
+            // extension values from the previous test.
+            slidingSync.resend();
+
+            const p = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state, resp, err) => {
+                return state === SlidingSyncState.Complete;
+            });
+            await httpBackend.flushAllExpected();
+            await p;
+            expect(responseCalled).toBe(false);
+        });
+
+        it("is possible to register extensions after start() has been called", async () => {
+            slidingSync.registerExtension(extPost);
+            onPostExtensionRequest = () => {
+                return extReq;
+            };
+            let responseCalled = false;
+            const callbackOrder = [];
+            onPostExtensionResponse = (resp) => {
+                expect(resp).toEqual(extResp);
+                responseCalled = true;
+                callbackOrder.push("onPostExtensionResponse");
+            };
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("ext req after start", body);
+                expect(body.extensions).toBeTruthy();
+                expect(body.extensions[preExtName]).toBeUndefined(); // from the earlier test
+                expect(body.extensions[postExtName]).toEqual(extReq);
+            }).respond(200, {
+                pos: "c",
+                ops: [],
+                counts: [],
+                extensions: {
+                    [postExtName]: extResp,
+                },
+            });
+            // we need to resend as sliding sync will already have a buffered request with the old
+            // extension values from the previous test.
+            slidingSync.resend();
+
+            const p = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state, resp, err) => {
+                if (state === SlidingSyncState.Complete) {
+                    callbackOrder.push("Lifecycle");
+                    return true;
+                }
+            });
+            await httpBackend.flushAllExpected();
+            await p;
+            expect(responseCalled).toBe(true);
+            expect(callbackOrder).toEqual(["Lifecycle", "onPostExtensionResponse"]);
+            slidingSync.stop();
+        });
+
+        it("is not possible to register the same extension name twice", async () => {
+            slidingSync = new SlidingSync(proxyBaseUrl, [], {}, client, 1);
+            slidingSync.registerExtension(extPre);
+            expect(() => { slidingSync.registerExtension(extPre); }).toThrow();
+        });
+    });
+});
+
+async function timeout(delayMs: number, reason: string): Promise<never> {
+    await sleep(delayMs);
+    throw new Error(`timeout: ${delayMs}ms - ${reason}`);
+}
+
+/**
+ * Listen until a callback returns data.
+ * @param {EventEmitter} emitter The event emitter
+ * @param {string} eventName The event to listen for
+ * @param {function} callback The callback which will be invoked when events fire. Return something truthy from this to resolve the promise.
+ * @param {number} timeoutMs The number of milliseconds to wait for the callback to return data. Default: 500ms.
+ * @returns {Promise} A promise which will be resolved when the callback returns data. If the callback throws or the timeout is reached,
+ * the promise is rejected.
+ */
+function listenUntil<T>(
+    emitter: EventEmitter,
+    eventName: string,
+    callback: (...args: any[]) => T,
+    timeoutMs = 500,
+): Promise<T> {
+    const trace = new Error().stack.split(`\n`)[2];
+    return Promise.race([new Promise<T>((resolve, reject) => {
+        const wrapper = (...args) => {
+            try {
+                const data = callback(...args);
+                if (data) {
+                    emitter.off(eventName, wrapper);
+                    resolve(data);
+                }
+            } catch (err) {
+                reject(err);
+            }
+        };
+        emitter.on(eventName, wrapper);
+    }), timeout(timeoutMs, "timed out waiting for event " + eventName + " " + trace)]);
+}

spec/test-utils/beacon.ts

@@ -27,6 +27,7 @@ type InfoContentProps = {
     isLive?: boolean;
     assetType?: LocationAssetType;
     description?: string;
+    timestamp?: number;
 };
 const DEFAULT_INFO_CONTENT_PROPS: InfoContentProps = {
     timeout: 3600000,
@@ -44,7 +45,11 @@ export const makeBeaconInfoEvent = (
     eventId?: string,
 ): MatrixEvent => {
     const {
-        timeout, isLive, description, assetType,
+        timeout,
+        isLive,
+        description,
+        assetType,
+        timestamp,
     } = {
         ...DEFAULT_INFO_CONTENT_PROPS,
         ...contentProps,
@@ -53,10 +58,10 @@ export const makeBeaconInfoEvent = (
         type: M_BEACON_INFO.name,
         room_id: roomId,
         state_key: sender,
-        content: makeBeaconInfoContent(timeout, isLive, description, assetType),
+        content: makeBeaconInfoContent(timeout, isLive, description, assetType, timestamp),
     });
 
-    event.event.origin_server_ts = Date.now();
+    event.event.origin_server_ts = timestamp || Date.now();
 
     // live beacons use the beacon_info event id
     // set or default this

spec/test-utils/test-utils.ts

@@ -6,7 +6,7 @@ import '../olm-loader';
 
 import { logger } from '../../src/logger';
 import { IContent, IEvent, IUnsigned, MatrixEvent, MatrixEventEvent } from "../../src/models/event";
-import { ClientEvent, EventType, MatrixClient } from "../../src";
+import { ClientEvent, EventType, MatrixClient, MsgType } from "../../src";
 import { SyncState } from "../../src/sync";
 import { eventMapperFor } from "../../src/event-mapper";
 
@@ -70,11 +70,10 @@ export function mock<T>(constr: { new(...args: any[]): T }, name: string): T {
 
 interface IEventOpts {
     type: EventType | string;
-    room: string;
+    room?: string;
     sender?: string;
     skey?: string;
     content: IContent;
-    event?: boolean;
     user?: string;
     unsigned?: IUnsigned;
     redacts?: string;
@@ -93,7 +92,9 @@ let testEventIndex = 1; // counter for events, easier for comparison of randomly
  * @param {MatrixClient} client If passed along with opts.event=true will be used to set up re-emitters.
  * @return {Object} a JSON object representing this event.
  */
-export function mkEvent(opts: IEventOpts, client?: MatrixClient): object | MatrixEvent {
+export function mkEvent(opts: IEventOpts & { event: true }, client?: MatrixClient): MatrixEvent;
+export function mkEvent(opts: IEventOpts & { event?: false }, client?: MatrixClient): Partial<IEvent>;
+export function mkEvent(opts: IEventOpts & { event?: boolean }, client?: MatrixClient): Partial<IEvent> | MatrixEvent {
     if (!opts.type || !opts.content) {
         throw new Error("Missing .type or .content =>" + JSON.stringify(opts));
     }
@@ -128,6 +129,21 @@ export function mkEvent(opts: IEventOpts, client?: MatrixClient): object | Matri
     return opts.event ? new MatrixEvent(event) : event;
 }
 
+type GeneratedMetadata = {
+    event_id: string;
+    txn_id: string;
+    origin_server_ts: number;
+};
+
+export function mkEventCustom<T>(base: T): T & GeneratedMetadata {
+    return {
+        event_id: "$" + testEventIndex++ + "-" + Math.random() + "-" + Math.random(),
+        txn_id: "~" + Math.random(),
+        origin_server_ts: Date.now(),
+        ...base,
+    };
+}
+
 interface IPresenceOpts {
     user?: string;
     sender?: string;
@@ -143,7 +159,9 @@ interface IPresenceOpts {
  * @param {Object} opts Values for the presence.
  * @return {Object|MatrixEvent} The event
  */
-export function mkPresence(opts: IPresenceOpts): object | MatrixEvent {
+export function mkPresence(opts: IPresenceOpts & { event: true }): MatrixEvent;
+export function mkPresence(opts: IPresenceOpts & { event?: false }): Partial<IEvent>;
+export function mkPresence(opts: IPresenceOpts & { event?: boolean }): Partial<IEvent> | MatrixEvent {
     const event = {
         event_id: "$" + Math.random() + "-" + Math.random(),
         type: "m.presence",
@@ -159,7 +177,7 @@ export function mkPresence(opts: IPresenceOpts): object | MatrixEvent {
 }
 
 interface IMembershipOpts {
-    room: string;
+    room?: string;
     mship: string;
     sender?: string;
     user?: string;
@@ -182,7 +200,9 @@ interface IMembershipOpts {
  * @param {boolean} opts.event True to make a MatrixEvent.
  * @return {Object|MatrixEvent} The event
  */
-export function mkMembership(opts: IMembershipOpts): object | MatrixEvent {
+export function mkMembership(opts: IMembershipOpts & { event: true }): MatrixEvent;
+export function mkMembership(opts: IMembershipOpts & { event?: false }): Partial<IEvent>;
+export function mkMembership(opts: IMembershipOpts & { event?: boolean }): Partial<IEvent> | MatrixEvent {
     const eventOpts: IEventOpts = {
         ...opts,
         type: EventType.RoomMember,
@@ -203,8 +223,20 @@ export function mkMembership(opts: IMembershipOpts): object | MatrixEvent {
     return mkEvent(eventOpts);
 }
 
+export function mkMembershipCustom<T>(
+    base: T & { membership: string, sender: string, content?: IContent },
+): T & { type: EventType, sender: string, state_key: string, content: IContent } & GeneratedMetadata {
+    const content = base.content || {};
+    return mkEventCustom({
+        ...base,
+        content: { ...content, membership: base.membership },
+        type: EventType.RoomMember,
+        state_key: base.sender,
+    });
+}
+
 interface IMessageOpts {
-    room: string;
+    room?: string;
     user: string;
     msg?: string;
     event?: boolean;
@@ -220,12 +252,17 @@ interface IMessageOpts {
  * @param {MatrixClient} client If passed along with opts.event=true will be used to set up re-emitters.
  * @return {Object|MatrixEvent} The event
  */
-export function mkMessage(opts: IMessageOpts, client?: MatrixClient): object | MatrixEvent {
+export function mkMessage(opts: IMessageOpts & { event: true }, client?: MatrixClient): MatrixEvent;
+export function mkMessage(opts: IMessageOpts & { event?: false }, client?: MatrixClient): Partial<IEvent>;
+export function mkMessage(
+    opts: IMessageOpts & { event?: boolean },
+    client?: MatrixClient,
+): Partial<IEvent> | MatrixEvent {
     const eventOpts: IEventOpts = {
         ...opts,
         type: EventType.RoomMessage,
         content: {
-            msgtype: "m.text",
+            msgtype: MsgType.Text,
             body: opts.msg,
         },
     };
@@ -236,6 +273,50 @@ export function mkMessage(opts: IMessageOpts, client?: MatrixClient): object | M
     return mkEvent(eventOpts, client);
 }
 
+interface IReplyMessageOpts extends IMessageOpts {
+    replyToMessage: MatrixEvent;
+}
+
+/**
+ * Create a reply message.
+ *
+ * @param {Object} opts Values for the message
+ * @param {string} opts.room The room ID for the event.
+ * @param {string} opts.user The user ID for the event.
+ * @param {string} opts.msg Optional. The content.body for the event.
+ * @param {MatrixEvent} opts.replyToMessage The replied message
+ * @param {boolean} opts.event True to make a MatrixEvent.
+ * @param {MatrixClient} client If passed along with opts.event=true will be used to set up re-emitters.
+ * @return {Object|MatrixEvent} The event
+ */
+export function mkReplyMessage(opts: IReplyMessageOpts & { event: true }, client?: MatrixClient): MatrixEvent;
+export function mkReplyMessage(opts: IReplyMessageOpts & { event?: false }, client?: MatrixClient): Partial<IEvent>;
+export function mkReplyMessage(
+    opts: IReplyMessageOpts & { event?: boolean },
+    client?: MatrixClient,
+): Partial<IEvent> | MatrixEvent {
+    const eventOpts: IEventOpts = {
+        ...opts,
+        type: EventType.RoomMessage,
+        content: {
+            "msgtype": MsgType.Text,
+            "body": opts.msg,
+            "m.relates_to": {
+                "rel_type": "m.in_reply_to",
+                "event_id": opts.replyToMessage.getId(),
+                "m.in_reply_to": {
+                    "event_id": opts.replyToMessage.getId(),
+                },
+            },
+        },
+    };
+
+    if (!eventOpts.content.body) {
+        eventOpts.content.body = "Random->" + Math.random();
+    }
+    return mkEvent(eventOpts, client);
+}
+
 /**
  * A mock implementation of webstorage
  *

spec/test-utils/webrtc.ts

@@ -0,0 +1,146 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export const DUMMY_SDP = (
+    "v=0\r\n" +
+    "o=- 5022425983810148698 2 IN IP4 127.0.0.1\r\n" +
+    "s=-\r\nt=0 0\r\na=group:BUNDLE 0\r\n" +
+    "a=msid-semantic: WMS h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA\r\n" +
+    "m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126\r\n" +
+    "c=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:hLDR\r\n" +
+    "a=ice-pwd:bMGD9aOldHWiI+6nAq/IIlRw\r\n" +
+    "a=ice-options:trickle\r\n" +
+    "a=fingerprint:sha-256 E4:94:84:F9:4A:98:8A:56:F5:5F:FD:AF:72:B9:32:89:49:5C:4B:9A:" +
+    "4A:15:8E:41:8A:F3:69:E4:39:52:DC:D6\r\n" +
+    "a=setup:active\r\n" +
+    "a=mid:0\r\n" +
+    "a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n" +
+    "a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n" +
+    "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n" +
+    "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n" +
+    "a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n" +
+    "a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n" +
+    "a=sendrecv\r\n" +
+    "a=msid:h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA 4357098f-3795-4131-bff4-9ba9c0348c49\r\n" +
+    "a=rtcp-mux\r\n" +
+    "a=rtpmap:111 opus/48000/2\r\n" +
+    "a=rtcp-fb:111 transport-cc\r\n" +
+    "a=fmtp:111 minptime=10;useinbandfec=1\r\n" +
+    "a=rtpmap:103 ISAC/16000\r\n" +
+    "a=rtpmap:104 ISAC/32000\r\n" +
+    "a=rtpmap:9 G722/8000\r\n" +
+    "a=rtpmap:0 PCMU/8000\r\n" +
+    "a=rtpmap:8 PCMA/8000\r\n" +
+    "a=rtpmap:106 CN/32000\r\n" +
+    "a=rtpmap:105 CN/16000\r\n" +
+    "a=rtpmap:13 CN/8000\r\n" +
+    "a=rtpmap:110 telephone-event/48000\r\n" +
+    "a=rtpmap:112 telephone-event/32000\r\n" +
+    "a=rtpmap:113 telephone-event/16000\r\n" +
+    "a=rtpmap:126 telephone-event/8000\r\n" +
+    "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n"
+);
+
+export class MockRTCPeerConnection {
+    localDescription: RTCSessionDescription;
+
+    constructor() {
+        this.localDescription = {
+            sdp: DUMMY_SDP,
+            type: 'offer',
+            toJSON: function() { },
+        };
+    }
+
+    addEventListener() { }
+    createDataChannel(label: string, opts: RTCDataChannelInit) { return { label, ...opts }; }
+    createOffer() {
+        return Promise.resolve({});
+    }
+    setRemoteDescription() {
+        return Promise.resolve();
+    }
+    setLocalDescription() {
+        return Promise.resolve();
+    }
+    close() { }
+    getStats() { return []; }
+    addTrack(track: MockMediaStreamTrack) { return new MockRTCRtpSender(track); }
+}
+
+export class MockRTCRtpSender {
+    constructor(public track: MockMediaStreamTrack) { }
+
+    replaceTrack(track: MockMediaStreamTrack) { this.track = track; }
+}
+
+export class MockMediaStreamTrack {
+    constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) { }
+
+    stop() { }
+}
+
+// XXX: Using EventTarget in jest doesn't seem to work, so we write our own
+// implementation
+export class MockMediaStream {
+    constructor(
+        public id: string,
+        private tracks: MockMediaStreamTrack[] = [],
+    ) {}
+
+    listeners: [string, (...args: any[]) => any][] = [];
+
+    dispatchEvent(eventType: string) {
+        this.listeners.forEach(([t, c]) => {
+            if (t !== eventType) return;
+            c();
+        });
+    }
+    getTracks() { return this.tracks; }
+    getAudioTracks() { return this.tracks.filter((track) => track.kind === "audio"); }
+    getVideoTracks() { return this.tracks.filter((track) => track.kind === "video"); }
+    addEventListener(eventType: string, callback: (...args: any[]) => any) {
+        this.listeners.push([eventType, callback]);
+    }
+    removeEventListener(eventType: string, callback: (...args: any[]) => any) {
+        this.listeners.filter(([t, c]) => {
+            return t !== eventType || c !== callback;
+        });
+    }
+    addTrack(track: MockMediaStreamTrack) {
+        this.tracks.push(track);
+        this.dispatchEvent("addtrack");
+    }
+    removeTrack(track: MockMediaStreamTrack) { this.tracks.splice(this.tracks.indexOf(track), 1); }
+}
+
+export class MockMediaDeviceInfo {
+    constructor(
+        public kind: "audio" | "video",
+    ) { }
+}
+
+export class MockMediaHandler {
+    getUserMediaStream(audio: boolean, video: boolean) {
+        const tracks = [];
+        if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio"));
+        if (video) tracks.push(new MockMediaStreamTrack("video_track", "video"));
+
+        return new MockMediaStream("mock_stream_from_media_handler", tracks);
+    }
+    stopUserMediaStream() { }
+    hasAudioDevice() { return true; }
+}

spec/unit/crypto.spec.js

@@ -3,7 +3,6 @@ import '../olm-loader';
 import { EventEmitter } from "events";
 
 import { Crypto } from "../../src/crypto";
-import { WebStorageSessionStore } from "../../src/store/session/webstorage";
 import { MemoryCryptoStore } from "../../src/crypto/store/memory-crypto-store";
 import { MockStorageApi } from "../MockStorageApi";
 import { TestClient } from "../TestClient";
@@ -14,9 +13,47 @@ import { sleep } from "../../src/utils";
 import { CRYPTO_ENABLED } from "../../src/client";
 import { DeviceInfo } from "../../src/crypto/deviceinfo";
 import { logger } from '../../src/logger';
+import { MemoryStore } from "../../src";
 
 const Olm = global.Olm;
 
+function awaitEvent(emitter, event) {
+    return new Promise((resolve, reject) => {
+        emitter.once(event, (result) => {
+            resolve(result);
+        });
+    });
+}
+
+async function keyshareEventForEvent(client, event, index) {
+    const roomId = event.getRoomId();
+    const eventContent = event.getWireContent();
+    const key = await client.crypto.olmDevice.getInboundGroupSessionKey(
+        roomId,
+        eventContent.sender_key,
+        eventContent.session_id,
+        index,
+    );
+    const ksEvent = new MatrixEvent({
+        type: "m.forwarded_room_key",
+        sender: client.getUserId(),
+        content: {
+            algorithm: olmlib.MEGOLM_ALGORITHM,
+            room_id: roomId,
+            sender_key: eventContent.sender_key,
+            sender_claimed_ed25519_key: key.sender_claimed_ed25519_key,
+            session_id: eventContent.session_id,
+            session_key: key.key,
+            chain_index: key.chain_index,
+            forwarding_curve25519_key_chain:
+            key.forwarding_curve_key_chain,
+        },
+    });
+    // make onRoomKeyEvent think this was an encrypted event
+    ksEvent.senderCurve25519Key = "akey";
+    return ksEvent;
+}
+
 describe("Crypto", function() {
     if (!CRYPTO_ENABLED) {
         return;
@@ -116,7 +153,7 @@ describe("Crypto", function() {
 
         beforeEach(async function() {
             const mockStorage = new MockStorageApi();
-            const sessionStore = new WebStorageSessionStore(mockStorage);
+            const clientStore = new MemoryStore({ localStorage: mockStorage });
             const cryptoStore = new MemoryCryptoStore(mockStorage);
 
             cryptoStore.storeEndToEndDeviceData({
@@ -143,10 +180,9 @@ describe("Crypto", function() {
 
             crypto = new Crypto(
                 mockBaseApis,
-                sessionStore,
                 "@alice:home.server",
                 "FLIBBLE",
-                sessionStore,
+                clientStore,
                 cryptoStore,
                 mockRoomList,
             );
@@ -203,136 +239,141 @@ describe("Crypto", function() {
             bobClient.stopClient();
         });
 
-        it(
-            "does not cancel keyshare requests if some messages are not decrypted",
-            async function() {
-                function awaitEvent(emitter, event) {
-                    return new Promise((resolve, reject) => {
-                        emitter.once(event, (result) => {
-                            resolve(result);
-                        });
-                    });
-                }
-
-                async function keyshareEventForEvent(event, index) {
-                    const eventContent = event.getWireContent();
-                    const key = await aliceClient.crypto.olmDevice
-                        .getInboundGroupSessionKey(
-                            roomId, eventContent.sender_key, eventContent.session_id,
-                            index,
-                        );
-                    const ksEvent = new MatrixEvent({
-                        type: "m.forwarded_room_key",
-                        sender: "@alice:example.com",
-                        content: {
-                            algorithm: olmlib.MEGOLM_ALGORITHM,
-                            room_id: roomId,
-                            sender_key: eventContent.sender_key,
-                            sender_claimed_ed25519_key: key.sender_claimed_ed25519_key,
-                            session_id: eventContent.session_id,
-                            session_key: key.key,
-                            chain_index: key.chain_index,
-                            forwarding_curve25519_key_chain:
-                            key.forwarding_curve_key_chain,
-                        },
-                    });
-                    // make onRoomKeyEvent think this was an encrypted event
-                    ksEvent.senderCurve25519Key = "akey";
-                    return ksEvent;
-                }
-
-                const encryptionCfg = {
-                    "algorithm": "m.megolm.v1.aes-sha2",
-                };
-                const roomId = "!someroom";
-                const aliceRoom = new Room(roomId, aliceClient, "@alice:example.com", {});
-                const bobRoom = new Room(roomId, bobClient, "@bob:example.com", {});
-                aliceClient.store.storeRoom(aliceRoom);
-                bobClient.store.storeRoom(bobRoom);
-                await aliceClient.setRoomEncryption(roomId, encryptionCfg);
-                await bobClient.setRoomEncryption(roomId, encryptionCfg);
-                const events = [
-                    new MatrixEvent({
-                        type: "m.room.message",
-                        sender: "@alice:example.com",
-                        room_id: roomId,
-                        event_id: "$1",
-                        content: {
-                            msgtype: "m.text",
-                            body: "1",
-                        },
-                    }),
-                    new MatrixEvent({
-                        type: "m.room.message",
-                        sender: "@alice:example.com",
-                        room_id: roomId,
-                        event_id: "$2",
-                        content: {
-                            msgtype: "m.text",
-                            body: "2",
-                        },
-                    }),
-                ];
-                await Promise.all(events.map(async (event) => {
-                    // alice encrypts each event, and then bob tries to decrypt
-                    // them without any keys, so that they'll be in pending
-                    await aliceClient.crypto.encryptEvent(event, aliceRoom);
-                    event.clearEvent = undefined;
-                    event.senderCurve25519Key = null;
-                    event.claimedEd25519Key = null;
-                    try {
-                        await bobClient.crypto.decryptEvent(event);
-                    } catch (e) {
-                        // we expect this to fail because we don't have the
-                        // decryption keys yet
-                    }
-                }));
-
-                const bobDecryptor = bobClient.crypto.getRoomDecryptor(
-                    roomId, olmlib.MEGOLM_ALGORITHM,
-                );
-
-                let eventPromise = Promise.all(events.map((ev) => {
-                    return awaitEvent(ev, "Event.decrypted");
-                }));
-
-                // keyshare the session key starting at the second message, so
-                // the first message can't be decrypted yet, but the second one
-                // can
-                let ksEvent = await keyshareEventForEvent(events[1], 1);
-                await bobDecryptor.onRoomKeyEvent(ksEvent);
-                await eventPromise;
-                expect(events[0].getContent().msgtype).toBe("m.bad.encrypted");
-                expect(events[1].getContent().msgtype).not.toBe("m.bad.encrypted");
-
-                const cryptoStore = bobClient.cryptoStore;
-                const eventContent = events[0].getWireContent();
-                const senderKey = eventContent.sender_key;
-                const sessionId = eventContent.session_id;
-                const roomKeyRequestBody = {
-                    algorithm: olmlib.MEGOLM_ALGORITHM,
+        it("does not cancel keyshare requests if some messages are not decrypted", async function() {
+            const encryptionCfg = {
+                "algorithm": "m.megolm.v1.aes-sha2",
+            };
+            const roomId = "!someroom";
+            const aliceRoom = new Room(roomId, aliceClient, "@alice:example.com", {});
+            const bobRoom = new Room(roomId, bobClient, "@bob:example.com", {});
+            aliceClient.store.storeRoom(aliceRoom);
+            bobClient.store.storeRoom(bobRoom);
+            await aliceClient.setRoomEncryption(roomId, encryptionCfg);
+            await bobClient.setRoomEncryption(roomId, encryptionCfg);
+            const events = [
+                new MatrixEvent({
+                    type: "m.room.message",
+                    sender: "@alice:example.com",
                     room_id: roomId,
-                    sender_key: senderKey,
-                    session_id: sessionId,
-                };
-                // the room key request should still be there, since we haven't
-                // decrypted everything
-                expect(await cryptoStore.getOutgoingRoomKeyRequest(roomKeyRequestBody))
-                    .toBeDefined();
+                    event_id: "$1",
+                    content: {
+                        msgtype: "m.text",
+                        body: "1",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.room.message",
+                    sender: "@alice:example.com",
+                    room_id: roomId,
+                    event_id: "$2",
+                    content: {
+                        msgtype: "m.text",
+                        body: "2",
+                    },
+                }),
+            ];
+            await Promise.all(events.map(async (event) => {
+                // alice encrypts each event, and then bob tries to decrypt
+                // them without any keys, so that they'll be in pending
+                await aliceClient.crypto.encryptEvent(event, aliceRoom);
+                event.clearEvent = undefined;
+                event.senderCurve25519Key = null;
+                event.claimedEd25519Key = null;
+                try {
+                    await bobClient.crypto.decryptEvent(event);
+                } catch (e) {
+                    // we expect this to fail because we don't have the
+                    // decryption keys yet
+                }
+            }));
 
-                // keyshare the session key starting at the first message, so
-                // that it can now be decrypted
-                eventPromise = awaitEvent(events[0], "Event.decrypted");
-                ksEvent = await keyshareEventForEvent(events[0], 0);
-                await bobDecryptor.onRoomKeyEvent(ksEvent);
-                await eventPromise;
-                expect(events[0].getContent().msgtype).not.toBe("m.bad.encrypted");
-                await sleep(1);
-                // the room key request should be gone since we've now decrypted everything
-                expect(await cryptoStore.getOutgoingRoomKeyRequest(roomKeyRequestBody))
-                    .toBeFalsy();
-            },
-        );
+            const bobDecryptor = bobClient.crypto.getRoomDecryptor(
+                roomId, olmlib.MEGOLM_ALGORITHM,
+            );
+
+            let eventPromise = Promise.all(events.map((ev) => {
+                return awaitEvent(ev, "Event.decrypted");
+            }));
+
+            // keyshare the session key starting at the second message, so
+            // the first message can't be decrypted yet, but the second one
+            // can
+            let ksEvent = await keyshareEventForEvent(aliceClient, events[1], 1);
+            await bobDecryptor.onRoomKeyEvent(ksEvent);
+            await eventPromise;
+            expect(events[0].getContent().msgtype).toBe("m.bad.encrypted");
+            expect(events[1].getContent().msgtype).not.toBe("m.bad.encrypted");
+
+            const cryptoStore = bobClient.cryptoStore;
+            const eventContent = events[0].getWireContent();
+            const senderKey = eventContent.sender_key;
+            const sessionId = eventContent.session_id;
+            const roomKeyRequestBody = {
+                algorithm: olmlib.MEGOLM_ALGORITHM,
+                room_id: roomId,
+                sender_key: senderKey,
+                session_id: sessionId,
+            };
+            // the room key request should still be there, since we haven't
+            // decrypted everything
+            expect(await cryptoStore.getOutgoingRoomKeyRequest(roomKeyRequestBody)).toBeDefined();
+
+            // keyshare the session key starting at the first message, so
+            // that it can now be decrypted
+            eventPromise = awaitEvent(events[0], "Event.decrypted");
+            ksEvent = await keyshareEventForEvent(aliceClient, events[0], 0);
+            await bobDecryptor.onRoomKeyEvent(ksEvent);
+            await eventPromise;
+            expect(events[0].getContent().msgtype).not.toBe("m.bad.encrypted");
+            await sleep(1);
+            // the room key request should be gone since we've now decrypted everything
+            expect(await cryptoStore.getOutgoingRoomKeyRequest(roomKeyRequestBody)).toBeFalsy();
+        });
+
+        it("should error if a forwarded room key lacks a content.sender_key", async function() {
+            const encryptionCfg = {
+                "algorithm": "m.megolm.v1.aes-sha2",
+            };
+            const roomId = "!someroom";
+            const aliceRoom = new Room(roomId, aliceClient, "@alice:example.com", {});
+            const bobRoom = new Room(roomId, bobClient, "@bob:example.com", {});
+            aliceClient.store.storeRoom(aliceRoom);
+            bobClient.store.storeRoom(bobRoom);
+            await aliceClient.setRoomEncryption(roomId, encryptionCfg);
+            await bobClient.setRoomEncryption(roomId, encryptionCfg);
+            const event = new MatrixEvent({
+                type: "m.room.message",
+                sender: "@alice:example.com",
+                room_id: roomId,
+                event_id: "$1",
+                content: {
+                    msgtype: "m.text",
+                    body: "1",
+                },
+            });
+            // alice encrypts each event, and then bob tries to decrypt
+            // them without any keys, so that they'll be in pending
+            await aliceClient.crypto.encryptEvent(event, aliceRoom);
+            event.clearEvent = undefined;
+            event.senderCurve25519Key = null;
+            event.claimedEd25519Key = null;
+            try {
+                await bobClient.crypto.decryptEvent(event);
+            } catch (e) {
+                // we expect this to fail because we don't have the
+                // decryption keys yet
+            }
+
+            const bobDecryptor = bobClient.crypto.getRoomDecryptor(
+                roomId, olmlib.MEGOLM_ALGORITHM,
+            );
+
+            const ksEvent = await keyshareEventForEvent(aliceClient, event, 1);
+            ksEvent.getContent().sender_key = undefined; // test
+            bobClient.crypto.addInboundGroupSession = jest.fn();
+            await bobDecryptor.onRoomKeyEvent(ksEvent);
+            expect(bobClient.crypto.addInboundGroupSession).not.toHaveBeenCalled();
+        });
 
         it("creates a new keyshare request if we request a keyshare", async function() {
             // make sure that cancelAndResend... creates a new keyshare request
@@ -423,6 +464,7 @@ describe("Crypto", function() {
             await client.crypto.bootstrapSecretStorage({
                 createSecretStorageKey,
             });
+            client.stopClient();
         });
     });
 });

spec/unit/crypto/CrossSigningInfo.spec.ts

@@ -66,23 +66,23 @@ describe("CrossSigningInfo.getCrossSigningKey", function() {
     });
 
     it.each(types)("should throw if the callback returns falsey",
-                   async ({ type, shouldCache }) => {
-        const info = new CrossSigningInfo(userId, {
-            getCrossSigningKey: () => false,
+        async ({ type, shouldCache }) => {
+            const info = new CrossSigningInfo(userId, {
+                getCrossSigningKey: async () => false as unknown as Uint8Array,
+            });
+            await expect(info.getCrossSigningKey(type)).rejects.toThrow("falsey");
         });
-        await expect(info.getCrossSigningKey(type)).rejects.toThrow("falsey");
-    });
 
     it("should throw if the expected key doesn't come back", async () => {
         const info = new CrossSigningInfo(userId, {
-            getCrossSigningKey: () => masterKeyPub,
+            getCrossSigningKey: async () => masterKeyPub as unknown as Uint8Array,
         });
         await expect(info.getCrossSigningKey("master", "")).rejects.toThrow();
     });
 
     it("should return a key from its callback", async () => {
         const info = new CrossSigningInfo(userId, {
-            getCrossSigningKey: () => testKey,
+            getCrossSigningKey: async () => testKey,
         });
         const [pubKey, pkSigning] = await info.getCrossSigningKey("master", masterKeyPub);
         expect(pubKey).toEqual(masterKeyPub);
@@ -99,7 +99,7 @@ describe("CrossSigningInfo.getCrossSigningKey", function() {
     it.each(types)("should request a key from the cache callback (if set)" +
                    " and does not call app if one is found" +
                    " %o",
-                   async ({ type, shouldCache }) => {
+    async ({ type, shouldCache }) => {
         const getCrossSigningKey = jest.fn().mockImplementation(() => {
             if (shouldCache) {
                 return Promise.reject(new Error("Regular callback called"));
@@ -122,58 +122,58 @@ describe("CrossSigningInfo.getCrossSigningKey", function() {
     });
 
     it.each(types)("should store a key with the cache callback (if set)",
-                   async ({ type, shouldCache }) => {
-        const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
-        const storeCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
-        const info = new CrossSigningInfo(
-            userId,
-            { getCrossSigningKey },
-            { storeCrossSigningKeyCache },
-        );
-        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
-        expect(pubKey).toEqual(masterKeyPub);
-        expect(storeCrossSigningKeyCache.mock.calls.length).toEqual(shouldCache ? 1 : 0);
-        if (shouldCache) {
-            expect(storeCrossSigningKeyCache.mock.calls[0][0]).toBe(type);
-            expect(storeCrossSigningKeyCache.mock.calls[0][1]).toBe(testKey);
-        }
-    });
+        async ({ type, shouldCache }) => {
+            const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
+            const storeCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+            const info = new CrossSigningInfo(
+                userId,
+                { getCrossSigningKey },
+                { storeCrossSigningKeyCache },
+            );
+            const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+            expect(pubKey).toEqual(masterKeyPub);
+            expect(storeCrossSigningKeyCache.mock.calls.length).toEqual(shouldCache ? 1 : 0);
+            if (shouldCache) {
+                expect(storeCrossSigningKeyCache.mock.calls[0][0]).toBe(type);
+                expect(storeCrossSigningKeyCache.mock.calls[0][1]).toBe(testKey);
+            }
+        });
 
     it.each(types)("does not store a bad key to the cache",
-                   async ({ type, shouldCache }) => {
-        const getCrossSigningKey = jest.fn().mockResolvedValue(badKey);
-        const storeCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
-        const info = new CrossSigningInfo(
-            userId,
-            { getCrossSigningKey },
-            { storeCrossSigningKeyCache },
-        );
-        await expect(info.getCrossSigningKey(type, masterKeyPub)).rejects.toThrow();
-        expect(storeCrossSigningKeyCache.mock.calls.length).toEqual(0);
-    });
+        async ({ type, shouldCache }) => {
+            const getCrossSigningKey = jest.fn().mockResolvedValue(badKey);
+            const storeCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+            const info = new CrossSigningInfo(
+                userId,
+                { getCrossSigningKey },
+                { storeCrossSigningKeyCache },
+            );
+            await expect(info.getCrossSigningKey(type, masterKeyPub)).rejects.toThrow();
+            expect(storeCrossSigningKeyCache.mock.calls.length).toEqual(0);
+        });
 
     it.each(types)("does not store a value to the cache if it came from the cache",
         async ({ type, shouldCache }) => {
-        const getCrossSigningKey = jest.fn().mockImplementation(() => {
-            if (shouldCache) {
-                return Promise.reject(new Error("Regular callback called"));
-            } else {
-                return Promise.resolve(testKey);
-            }
+            const getCrossSigningKey = jest.fn().mockImplementation(() => {
+                if (shouldCache) {
+                    return Promise.reject(new Error("Regular callback called"));
+                } else {
+                    return Promise.resolve(testKey);
+                }
+            });
+            const getCrossSigningKeyCache = jest.fn().mockResolvedValue(testKey);
+            const storeCrossSigningKeyCache = jest.fn().mockRejectedValue(
+                new Error("Tried to store a value from cache"),
+            );
+            const info = new CrossSigningInfo(
+                userId,
+                { getCrossSigningKey },
+                { getCrossSigningKeyCache, storeCrossSigningKeyCache },
+            );
+            expect(storeCrossSigningKeyCache.mock.calls.length).toBe(0);
+            const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+            expect(pubKey).toEqual(masterKeyPub);
         });
-        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(testKey);
-        const storeCrossSigningKeyCache = jest.fn().mockRejectedValue(
-            new Error("Tried to store a value from cache"),
-        );
-        const info = new CrossSigningInfo(
-            userId,
-            { getCrossSigningKey },
-            { getCrossSigningKeyCache, storeCrossSigningKeyCache },
-        );
-        expect(storeCrossSigningKeyCache.mock.calls.length).toBe(0);
-        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
-        expect(pubKey).toEqual(masterKeyPub);
-    });
 
     it.each(types)("requests a key from the cache callback (if set) and then calls app" +
         " if one is not found", async ({ type, shouldCache }) => {
@@ -220,12 +220,14 @@ describe("CrossSigningInfo.getCrossSigningKey", function() {
  */
 describe.each([
     ["IndexedDBCryptoStore",
-     () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+        () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
     ["LocalStorageCryptoStore",
-     () => new IndexedDBCryptoStore(undefined, "tests")],
+        () => new IndexedDBCryptoStore(undefined, "tests")],
     ["MemoryCryptoStore", () => {
         const store = new IndexedDBCryptoStore(undefined, "tests");
+        // @ts-ignore set private properties
         store._backend = new MemoryCryptoStore();
+        // @ts-ignore
         store._backendPromise = Promise.resolve(store._backend);
         return store;
     }],

spec/unit/crypto/DeviceList.spec.ts

@@ -1,7 +1,7 @@
 /*
 Copyright 2017 Vector Creations Ltd
 Copyright 2018, 2019 New Vector Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2022 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -20,8 +20,10 @@ import { logger } from "../../../src/logger";
 import * as utils from "../../../src/utils";
 import { MemoryCryptoStore } from "../../../src/crypto/store/memory-crypto-store";
 import { DeviceList } from "../../../src/crypto/DeviceList";
+import { IDownloadKeyResult, MatrixClient } from "../../../src";
+import { OlmDevice } from "../../../src/crypto/OlmDevice";
 
-const signedDeviceList = {
+const signedDeviceList: IDownloadKeyResult = {
     "failures": {},
     "device_keys": {
         "@test1:sw1v.org": {
@@ -45,13 +47,15 @@ const signedDeviceList = {
                     "m.megolm.v1.aes-sha2",
                 ],
                 "device_id": "HGKAWHRVJQ",
-                "unsigned": {},
+                "unsigned": {
+                    "device_display_name": "",
+                },
             },
         },
     },
 };
 
-const signedDeviceList2 = {
+const signedDeviceList2: IDownloadKeyResult = {
     "failures": {},
     "device_keys": {
         "@test2:sw1v.org": {
@@ -75,7 +79,9 @@ const signedDeviceList2 = {
                     "m.megolm.v1.aes-sha2",
                 ],
                 "device_id": "QJVRHWAKGH",
-                "unsigned": {},
+                "unsigned": {
+                    "device_display_name": "",
+                },
             },
         },
     },
@@ -104,10 +110,10 @@ describe('DeviceList', function() {
             downloadKeysForUsers: downloadSpy,
             getUserId: () => '@test1:sw1v.org',
             deviceId: 'HGKAWHRVJQ',
-        };
+        } as unknown as MatrixClient;
         const mockOlm = {
             verifySignature: function(key, message, signature) {},
-        };
+        } as unknown as OlmDevice;
         const dl = new DeviceList(baseApis, cryptoStore, mockOlm, keyDownloadChunkSize);
         deviceLists.push(dl);
         return dl;
@@ -118,7 +124,7 @@ describe('DeviceList', function() {
 
         dl.startTrackingDeviceList('@test1:sw1v.org');
 
-        const queryDefer1 = utils.defer();
+        const queryDefer1 = utils.defer<IDownloadKeyResult>();
         downloadSpy.mockReturnValue(queryDefer1.promise);
 
         const prom1 = dl.refreshOutdatedDeviceLists();
@@ -128,6 +134,7 @@ describe('DeviceList', function() {
         return prom1.then(() => {
             const storedKeys = dl.getRawStoredDevicesForUser('@test1:sw1v.org');
             expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
+            dl.stop();
         });
     });
 
@@ -137,7 +144,7 @@ describe('DeviceList', function() {
 
         dl.startTrackingDeviceList('@test1:sw1v.org');
 
-        const queryDefer1 = utils.defer();
+        const queryDefer1 = utils.defer<IDownloadKeyResult>();
         downloadSpy.mockReturnValue(queryDefer1.promise);
 
         const prom1 = dl.refreshOutdatedDeviceLists();
@@ -154,6 +161,7 @@ describe('DeviceList', function() {
         dl.saveIfDirty().then(() => {
             // the first request completes
             queryDefer1.resolve({
+                failures: {},
                 device_keys: {
                     '@test1:sw1v.org': {},
                 },
@@ -165,11 +173,12 @@ describe('DeviceList', function() {
             logger.log("Creating new devicelist to simulate app reload");
             downloadSpy.mockReset();
             const dl2 = createTestDeviceList();
-            const queryDefer3 = utils.defer();
+            const queryDefer3 = utils.defer<IDownloadKeyResult>();
             downloadSpy.mockReturnValue(queryDefer3.promise);
 
             const prom3 = dl2.refreshOutdatedDeviceLists();
             expect(downloadSpy).toHaveBeenCalledWith(['@test1:sw1v.org'], {});
+            dl2.stop();
 
             queryDefer3.resolve(utils.deepCopy(signedDeviceList));
 
@@ -178,6 +187,7 @@ describe('DeviceList', function() {
         }).then(() => {
             const storedKeys = dl.getRawStoredDevicesForUser('@test1:sw1v.org');
             expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
+            dl.stop();
         });
     });
 
@@ -187,9 +197,9 @@ describe('DeviceList', function() {
         dl.startTrackingDeviceList('@test1:sw1v.org');
         dl.startTrackingDeviceList('@test2:sw1v.org');
 
-        const queryDefer1 = utils.defer();
+        const queryDefer1 = utils.defer<IDownloadKeyResult>();
         downloadSpy.mockReturnValueOnce(queryDefer1.promise);
-        const queryDefer2 = utils.defer();
+        const queryDefer2 = utils.defer<IDownloadKeyResult>();
         downloadSpy.mockReturnValueOnce(queryDefer2.promise);
 
         const prom1 = dl.refreshOutdatedDeviceLists();
@@ -204,6 +214,7 @@ describe('DeviceList', function() {
             expect(Object.keys(storedKeys1)).toEqual(['HGKAWHRVJQ']);
             const storedKeys2 = dl.getRawStoredDevicesForUser('@test2:sw1v.org');
             expect(Object.keys(storedKeys2)).toEqual(['QJVRHWAKGH']);
+            dl.stop();
         });
     });
 });

spec/unit/crypto/algorithms/megolm.spec.js

@@ -257,6 +257,8 @@ describe("MegolmDecryption", function() {
         });
 
         describe("session reuse and key reshares", () => {
+            const rotationPeriodMs = 999 * 24 * 60 * 60 * 1000; // 999 days, so we don't have to deal with it
+
             let megolmEncryption;
             let aliceDeviceInfo;
             let mockRoom;
@@ -318,7 +320,7 @@ describe("MegolmDecryption", function() {
                     baseApis: mockBaseApis,
                     roomId: ROOM_ID,
                     config: {
-                        rotation_period_ms: 9999999999999,
+                        rotation_period_ms: rotationPeriodMs,
                     },
                 });
                 mockRoom = {
@@ -329,6 +331,31 @@ describe("MegolmDecryption", function() {
                 };
             });
 
+            it("should use larger otkTimeout when preparing to encrypt room", async () => {
+                megolmEncryption.prepareToEncrypt(mockRoom);
+                await megolmEncryption.encryptMessage(mockRoom, "a.fake.type", {
+                    body: "Some text",
+                });
+                expect(mockRoom.getEncryptionTargetMembers).toHaveBeenCalled();
+
+                expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalledWith(
+                    [['@alice:home.server', 'aliceDevice']], 'signed_curve25519', 10000,
+                );
+            });
+
+            it("should generate a new session if this one needs rotation", async () => {
+                const session = await megolmEncryption.prepareNewSession(false);
+                session.creationTime -= rotationPeriodMs + 10000; // a smidge over the rotation time
+                // Inject expired session which needs rotation
+                megolmEncryption.setupPromise = Promise.resolve(session);
+
+                const prepareNewSessionSpy = jest.spyOn(megolmEncryption, "prepareNewSession");
+                await megolmEncryption.encryptMessage(mockRoom, "a.fake.type", {
+                    body: "Some text",
+                });
+                expect(prepareNewSessionSpy).toHaveBeenCalledTimes(1);
+            });
+
             it("re-uses sessions for sequential messages", async function() {
                 const ct1 = await megolmEncryption.encryptMessage(mockRoom, "a.fake.type", {
                     body: "Some text",
@@ -596,6 +623,8 @@ describe("MegolmDecryption", function() {
         });
         await aliceClient.crypto.encryptEvent(event, aliceRoom);
         await sendPromise;
+        aliceClient.stopClient();
+        bobClient.stopClient();
     });
 
     it("throws an error describing why it doesn't have a key", async function() {
@@ -614,7 +643,7 @@ describe("MegolmDecryption", function() {
         const roomId = "!someroom";
 
         aliceClient.crypto.onToDeviceEvent(new MatrixEvent({
-            type: "org.matrix.room_key.withheld",
+            type: "m.room_key.withheld",
             sender: "@bob:example.com",
             content: {
                 algorithm: "m.megolm.v1.aes-sha2",
@@ -666,6 +695,8 @@ describe("MegolmDecryption", function() {
                 session_id: "session_id2",
             },
         }))).rejects.toThrow("The sender has blocked you.");
+        aliceClient.stopClient();
+        bobClient.stopClient();
     });
 
     it("throws an error describing the lack of an olm session", async function() {
@@ -687,7 +718,7 @@ describe("MegolmDecryption", function() {
         const now = Date.now();
 
         aliceClient.crypto.onToDeviceEvent(new MatrixEvent({
-            type: "org.matrix.room_key.withheld",
+            type: "m.room_key.withheld",
             sender: "@bob:example.com",
             content: {
                 algorithm: "m.megolm.v1.aes-sha2",
@@ -749,6 +780,8 @@ describe("MegolmDecryption", function() {
             },
             origin_server_ts: now,
         }))).rejects.toThrow("The sender was unable to establish a secure channel.");
+        aliceClient.stopClient();
+        bobClient.stopClient();
     });
 
     it("throws an error to indicate a wedged olm session", async function() {
@@ -799,5 +832,7 @@ describe("MegolmDecryption", function() {
             },
             origin_server_ts: now,
         }))).rejects.toThrow("The secure channel with the sender was corrupted.");
+        aliceClient.stopClient();
+        bobClient.stopClient();
     });
 });

spec/unit/crypto/backup.spec.ts

@@ -15,20 +15,22 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { MockedObject } from "jest-mock";
+
 import '../../olm-loader';
 import { logger } from "../../../src/logger";
 import * as olmlib from "../../../src/crypto/olmlib";
 import { MatrixClient } from "../../../src/client";
 import { MatrixEvent } from "../../../src/models/event";
 import * as algorithms from "../../../src/crypto/algorithms";
-import { WebStorageSessionStore } from "../../../src/store/session/webstorage";
 import { MemoryCryptoStore } from "../../../src/crypto/store/memory-crypto-store";
-import { MockStorageApi } from "../../MockStorageApi";
 import * as testUtils from "../../test-utils/test-utils";
 import { OlmDevice } from "../../../src/crypto/OlmDevice";
 import { Crypto } from "../../../src/crypto";
 import { resetCrossSigningKeys } from "./crypto-utils";
 import { BackupManager } from "../../../src/crypto/backup";
+import { StubStore } from "../../../src/store/stub";
+import { IAbortablePromise, MatrixScheduler } from '../../../src';
 
 const Olm = global.Olm;
 
@@ -93,8 +95,8 @@ const AES256_KEY_BACKUP_DATA = {
 };
 
 const CURVE25519_BACKUP_INFO = {
-    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
-    version: 1,
+    algorithm: olmlib.MEGOLM_BACKUP_ALGORITHM,
+    version: '1',
     auth_data: {
         public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
     },
@@ -102,7 +104,7 @@ const CURVE25519_BACKUP_INFO = {
 
 const AES256_BACKUP_INFO = {
     algorithm: "org.matrix.msc3270.v1.aes-hmac-sha2",
-    version: 1,
+    version: '1',
     auth_data: {
         // FIXME: add iv and mac
     },
@@ -118,30 +120,22 @@ function saveCrossSigningKeys(k) {
     Object.assign(keys, k);
 }
 
-function makeTestClient(sessionStore, cryptoStore) {
+function makeTestClient(cryptoStore) {
     const scheduler = [
         "getQueueForEvent", "queueEvent", "removeEventFromQueue",
         "setProcessFunction",
-    ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
-    const store = [
-        "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
-        "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom",
-        "storeUser", "getFilterIdByName", "setFilterIdByName", "getFilter",
-        "storeFilter", "getSyncAccumulator", "startup", "deleteAllData",
-    ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
-    store.getSavedSync = jest.fn().mockReturnValue(Promise.resolve(null));
-    store.getSavedSyncToken = jest.fn().mockReturnValue(Promise.resolve(null));
-    store.setSyncData = jest.fn().mockReturnValue(Promise.resolve(null));
+    ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {}) as MockedObject<MatrixScheduler>;
+    const store = new StubStore();
+
     return new MatrixClient({
         baseUrl: "https://my.home.server",
         idBaseUrl: "https://identity.server",
         accessToken: "my.access.token",
-        request: function() {}, // NOP
+        request: jest.fn(), // NOP
         store: store,
         scheduler: scheduler,
         userId: "@alice:bar",
         deviceId: "device",
-        sessionStore: sessionStore,
         cryptoStore: cryptoStore,
         cryptoCallbacks: { getCrossSigningKey, saveCrossSigningKeys },
     });
@@ -160,8 +154,6 @@ describe("MegolmBackup", function() {
     let olmDevice;
     let mockOlmLib;
     let mockCrypto;
-    let mockStorage;
-    let sessionStore;
     let cryptoStore;
     let megolmDecryption;
     beforeEach(async function() {
@@ -173,9 +165,7 @@ describe("MegolmBackup", function() {
         );
         mockCrypto.backupInfo = CURVE25519_BACKUP_INFO;
 
-        mockStorage = new MockStorageApi();
-        sessionStore = new WebStorageSessionStore(mockStorage);
-        cryptoStore = new MemoryCryptoStore(mockStorage);
+        cryptoStore = new MemoryCryptoStore();
 
         olmDevice = new OlmDevice(cryptoStore);
 
@@ -188,7 +178,6 @@ describe("MegolmBackup", function() {
 
     describe("backup", function() {
         let mockBaseApis;
-        let realSetTimeout;
 
         beforeEach(function() {
             mockBaseApis = {};
@@ -206,14 +195,14 @@ describe("MegolmBackup", function() {
             // clobber the setTimeout function to run 100x faster.
             // ideally we would use lolex, but we have no oportunity
             // to tick the clock between the first try and the retry.
-            realSetTimeout = global.setTimeout;
-            global.setTimeout = function(f, n) {
+            const realSetTimeout = global.setTimeout;
+            jest.spyOn(global, 'setTimeout').mockImplementation(function(f, n) {
                 return realSetTimeout(f, n/100);
-            };
+            });
         });
 
         afterEach(function() {
-            global.setTimeout = realSetTimeout;
+            jest.spyOn(global, 'setTimeout').mockRestore();
         });
 
         it('automatically calls the key back up', function() {
@@ -261,7 +250,7 @@ describe("MegolmBackup", function() {
             const ibGroupSession = new Olm.InboundGroupSession();
             ibGroupSession.create(groupSession.session_key());
 
-            const client = makeTestClient(sessionStore, cryptoStore);
+            const client = makeTestClient(cryptoStore);
 
             megolmDecryption = new MegolmDecryption({
                 userId: '@user:id',
@@ -293,16 +282,16 @@ describe("MegolmBackup", function() {
                                 txn);
                         });
                 })
-                .then(() => {
-                    client.enableKeyBackup({
-                        algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
-                        version: 1,
+                .then(async () => {
+                    await client.enableKeyBackup({
+                        algorithm: olmlib.MEGOLM_BACKUP_ALGORITHM,
+                        version: '1',
                         auth_data: {
                             public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
                         },
                     });
                     let numCalls = 0;
-                    return new Promise((resolve, reject) => {
+                    return new Promise<void>((resolve, reject) => {
                         client.http.authedRequest = function(
                             callback, method, path, queryParams, data, opts,
                         ) {
@@ -311,17 +300,17 @@ describe("MegolmBackup", function() {
                             if (numCalls >= 2) {
                                 // exit out of retry loop if there's something wrong
                                 reject(new Error("authedRequest called too many timmes"));
-                                return Promise.resolve({});
+                                return Promise.resolve({}) as IAbortablePromise<any>;
                             }
                             expect(method).toBe("PUT");
                             expect(path).toBe("/room_keys/keys");
-                            expect(queryParams.version).toBe(1);
+                            expect(queryParams.version).toBe('1');
                             expect(data.rooms[ROOM_ID].sessions).toBeDefined();
                             expect(data.rooms[ROOM_ID].sessions).toHaveProperty(
                                 groupSession.session_id(),
                             );
                             resolve();
-                            return Promise.resolve({});
+                            return Promise.resolve({}) as IAbortablePromise<any>;
                         };
                         client.crypto.backupManager.backupGroupSession(
                             "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI",
@@ -329,6 +318,7 @@ describe("MegolmBackup", function() {
                         );
                     }).then(() => {
                         expect(numCalls).toBe(1);
+                        client.stopClient();
                     });
                 });
         });
@@ -339,7 +329,7 @@ describe("MegolmBackup", function() {
             const ibGroupSession = new Olm.InboundGroupSession();
             ibGroupSession.create(groupSession.session_key());
 
-            const client = makeTestClient(sessionStore, cryptoStore);
+            const client = makeTestClient(cryptoStore);
 
             megolmDecryption = new MegolmDecryption({
                 userId: '@user:id',
@@ -374,17 +364,17 @@ describe("MegolmBackup", function() {
                                 txn);
                         });
                 })
-                .then(() => {
-                    client.enableKeyBackup({
+                .then(async () => {
+                    await client.enableKeyBackup({
                         algorithm: "org.matrix.msc3270.v1.aes-hmac-sha2",
-                        version: 1,
+                        version: '1',
                         auth_data: {
                             iv: "PsCAtR7gMc4xBd9YS3A9Ow",
                             mac: "ZSDsTFEZK7QzlauCLMleUcX96GQZZM7UNtk4sripSqQ",
                         },
                     });
                     let numCalls = 0;
-                    return new Promise((resolve, reject) => {
+                    return new Promise<void>((resolve, reject) => {
                         client.http.authedRequest = function(
                             callback, method, path, queryParams, data, opts,
                         ) {
@@ -393,17 +383,17 @@ describe("MegolmBackup", function() {
                             if (numCalls >= 2) {
                                 // exit out of retry loop if there's something wrong
                                 reject(new Error("authedRequest called too many timmes"));
-                                return Promise.resolve({});
+                                return Promise.resolve({}) as IAbortablePromise<any>;
                             }
                             expect(method).toBe("PUT");
                             expect(path).toBe("/room_keys/keys");
-                            expect(queryParams.version).toBe(1);
+                            expect(queryParams.version).toBe('1');
                             expect(data.rooms[ROOM_ID].sessions).toBeDefined();
                             expect(data.rooms[ROOM_ID].sessions).toHaveProperty(
                                 groupSession.session_id(),
                             );
                             resolve();
-                            return Promise.resolve({});
+                            return Promise.resolve({}) as IAbortablePromise<any>;
                         };
                         client.crypto.backupManager.backupGroupSession(
                             "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI",
@@ -411,6 +401,7 @@ describe("MegolmBackup", function() {
                         );
                     }).then(() => {
                         expect(numCalls).toBe(1);
+                        client.stopClient();
                     });
                 });
         });
@@ -421,7 +412,7 @@ describe("MegolmBackup", function() {
             const ibGroupSession = new Olm.InboundGroupSession();
             ibGroupSession.create(groupSession.session_key());
 
-            const client = makeTestClient(sessionStore, cryptoStore);
+            const client = makeTestClient(cryptoStore);
 
             megolmDecryption = new MegolmDecryption({
                 userId: '@user:id',
@@ -434,19 +425,12 @@ describe("MegolmBackup", function() {
             megolmDecryption.olmlib = mockOlmLib;
 
             await client.initCrypto();
-            let privateKeys;
-            client.uploadDeviceSigningKeys = async function(e) {return;};
-            client.uploadKeySignatures = async function(e) {return;};
-            client.on("crossSigning.saveCrossSigningKeys", function(e) {
-                privateKeys = e;
-            });
-            client.on("crossSigning.getKey", function(e) {
-                e.done(privateKeys[e.type]);
-            });
+            client.uploadDeviceSigningKeys = async function(e) {return {};};
+            client.uploadKeySignatures = async function(e) {return { failures: {} };};
             await resetCrossSigningKeys(client);
             let numCalls = 0;
             await Promise.all([
-                new Promise((resolve, reject) => {
+                new Promise<void>((resolve, reject) => {
                     let backupInfo;
                     client.http.authedRequest = function(
                         callback, method, path, queryParams, data, opts,
@@ -463,33 +447,34 @@ describe("MegolmBackup", function() {
                                 );
                             } catch (e) {
                                 reject(e);
-                                return Promise.resolve({});
+                                return Promise.resolve({}) as IAbortablePromise<any>;
                             }
                             backupInfo = data;
-                            return Promise.resolve({});
+                            return Promise.resolve({}) as IAbortablePromise<any>;
                         } else if (numCalls === 2) {
                             expect(method).toBe("GET");
                             expect(path).toBe("/room_keys/version");
                             resolve();
-                            return Promise.resolve(backupInfo);
+                            return Promise.resolve(backupInfo) as IAbortablePromise<any>;
                         } else {
                             // exit out of retry loop if there's something wrong
                             reject(new Error("authedRequest called too many times"));
-                            return Promise.resolve({});
+                            return Promise.resolve({}) as IAbortablePromise<any>;
                         }
                     };
                 }),
                 client.createKeyBackupVersion({
-                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    algorithm: olmlib.MEGOLM_BACKUP_ALGORITHM,
                     auth_data: {
                         public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
                     },
                 }),
             ]);
             expect(numCalls).toBe(2);
+            client.stopClient();
         });
 
-        it('retries when a backup fails', function() {
+        it('retries when a backup fails', async function() {
             const groupSession = new Olm.OutboundGroupSession();
             groupSession.create();
             const ibGroupSession = new Olm.InboundGroupSession();
@@ -498,26 +483,17 @@ describe("MegolmBackup", function() {
             const scheduler = [
                 "getQueueForEvent", "queueEvent", "removeEventFromQueue",
                 "setProcessFunction",
-            ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
-            const store = [
-                "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
-                "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom",
-                "storeUser", "getFilterIdByName", "setFilterIdByName", "getFilter",
-                "storeFilter", "getSyncAccumulator", "startup", "deleteAllData",
-            ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
-            store.getSavedSync = jest.fn().mockReturnValue(Promise.resolve(null));
-            store.getSavedSyncToken = jest.fn().mockReturnValue(Promise.resolve(null));
-            store.setSyncData = jest.fn().mockReturnValue(Promise.resolve(null));
+            ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {}) as MockedObject<MatrixScheduler>;
+            const store = new StubStore();
             const client = new MatrixClient({
                 baseUrl: "https://my.home.server",
                 idBaseUrl: "https://identity.server",
                 accessToken: "my.access.token",
-                request: function() {}, // NOP
+                request: jest.fn(), // NOP
                 store: store,
                 scheduler: scheduler,
                 userId: "@alice:bar",
                 deviceId: "device",
-                sessionStore: sessionStore,
                 cryptoStore: cryptoStore,
             });
 
@@ -531,70 +507,68 @@ describe("MegolmBackup", function() {
 
             megolmDecryption.olmlib = mockOlmLib;
 
-            return client.initCrypto()
-                .then(() => {
-                    return cryptoStore.doTxn(
-                        "readwrite",
-                        [cryptoStore.STORE_SESSION],
-                        (txn) => {
-                            cryptoStore.addEndToEndInboundGroupSession(
-                                "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI",
-                                groupSession.session_id(),
-                                {
-                                    forwardingCurve25519KeyChain: undefined,
-                                    keysClaimed: {
-                                        ed25519: "SENDER_ED25519",
-                                    },
-                                    room_id: ROOM_ID,
-                                    session: ibGroupSession.pickle(olmDevice.pickleKey),
-                                },
-                                txn);
-                        });
-                })
-                .then(() => {
-                    client.enableKeyBackup({
-                        algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
-                        version: 1,
-                        auth_data: {
-                            public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+            await client.initCrypto();
+            await cryptoStore.doTxn(
+                "readwrite",
+                [cryptoStore.STORE_SESSION],
+                (txn) => {
+                    cryptoStore.addEndToEndInboundGroupSession(
+                        "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI",
+                        groupSession.session_id(),
+                        {
+                            forwardingCurve25519KeyChain: undefined,
+                            keysClaimed: {
+                                ed25519: "SENDER_ED25519",
+                            },
+                            room_id: ROOM_ID,
+                            session: ibGroupSession.pickle(olmDevice.pickleKey),
                         },
-                    });
-                    let numCalls = 0;
-                    return new Promise((resolve, reject) => {
-                        client.http.authedRequest = function(
-                            callback, method, path, queryParams, data, opts,
-                        ) {
-                            ++numCalls;
-                            expect(numCalls).toBeLessThanOrEqual(2);
-                            if (numCalls >= 3) {
-                                // exit out of retry loop if there's something wrong
-                                reject(new Error("authedRequest called too many timmes"));
-                                return Promise.resolve({});
-                            }
-                            expect(method).toBe("PUT");
-                            expect(path).toBe("/room_keys/keys");
-                            expect(queryParams.version).toBe(1);
-                            expect(data.rooms[ROOM_ID].sessions).toBeDefined();
-                            expect(data.rooms[ROOM_ID].sessions).toHaveProperty(
-                                groupSession.session_id(),
-                            );
-                            if (numCalls > 1) {
-                                resolve();
-                                return Promise.resolve({});
-                            } else {
-                                return Promise.reject(
-                                    new Error("this is an expected failure"),
-                                );
-                            }
-                        };
-                        client.crypto.backupManager.backupGroupSession(
-                            "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI",
-                            groupSession.session_id(),
-                        );
-                    }).then(() => {
-                        expect(numCalls).toBe(2);
-                    });
+                        txn);
                 });
+
+            await client.enableKeyBackup({
+                algorithm: olmlib.MEGOLM_BACKUP_ALGORITHM,
+                version: '1',
+                auth_data: {
+                    public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+                },
+            });
+            let numCalls = 0;
+
+            await new Promise<void>((resolve, reject) => {
+                client.http.authedRequest = function(
+                    callback, method, path, queryParams, data, opts,
+                ) {
+                    ++numCalls;
+                    expect(numCalls).toBeLessThanOrEqual(2);
+                    if (numCalls >= 3) {
+                        // exit out of retry loop if there's something wrong
+                        reject(new Error("authedRequest called too many timmes"));
+                        return Promise.resolve({}) as IAbortablePromise<any>;
+                    }
+                    expect(method).toBe("PUT");
+                    expect(path).toBe("/room_keys/keys");
+                    expect(queryParams.version).toBe('1');
+                    expect(data.rooms[ROOM_ID].sessions).toBeDefined();
+                    expect(data.rooms[ROOM_ID].sessions).toHaveProperty(
+                        groupSession.session_id(),
+                    );
+                    if (numCalls > 1) {
+                        resolve();
+                        return Promise.resolve({}) as IAbortablePromise<any>;
+                    } else {
+                        return Promise.reject(
+                            new Error("this is an expected failure"),
+                        ) as IAbortablePromise<any>;
+                    }
+                };
+                return client.crypto.backupManager.backupGroupSession(
+                    "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI",
+                    groupSession.session_id(),
+                );
+            });
+            expect(numCalls).toBe(2);
+            client.stopClient();
         });
     });
 
@@ -602,7 +576,7 @@ describe("MegolmBackup", function() {
         let client;
 
         beforeEach(function() {
-            client = makeTestClient(sessionStore, cryptoStore);
+            client = makeTestClient(cryptoStore);
 
             megolmDecryption = new MegolmDecryption({
                 userId: '@user:id',

spec/unit/crypto/cross-signing.spec.ts

@@ -17,12 +17,16 @@ limitations under the License.
 
 import '../../olm-loader';
 import anotherjson from 'another-json';
+import { PkSigning } from '@matrix-org/olm';
 
 import * as olmlib from "../../../src/crypto/olmlib";
-import { TestClient } from '../../TestClient';
-import { resetCrossSigningKeys } from "./crypto-utils";
 import { MatrixError } from '../../../src/http-api';
 import { logger } from '../../../src/logger';
+import { ICrossSigningKey, ICreateClientOpts, ISignedKey } from '../../../src/client';
+import { CryptoEvent } from '../../../src/crypto';
+import { IDevice } from '../../../src/crypto/deviceinfo';
+import { TestClient } from '../../TestClient';
+import { resetCrossSigningKeys } from "./crypto-utils";
 
 const PUSH_RULES_RESPONSE = {
     method: "GET",
@@ -47,9 +51,11 @@ function setHttpResponses(httpBackend, responses) {
     });
 }
 
-async function makeTestClient(userInfo, options, keys) {
-    if (!keys) keys = {};
-
+async function makeTestClient(
+    userInfo: { userId: string, deviceId: string},
+    options: Partial<ICreateClientOpts> = {},
+    keys = {},
+) {
     function getCrossSigningKey(type) {
         return keys[type];
     }
@@ -58,7 +64,6 @@ async function makeTestClient(userInfo, options, keys) {
         Object.assign(keys, k);
     }
 
-    if (!options) options = {};
     options.cryptoCallbacks = Object.assign(
         {}, { getCrossSigningKey, saveCrossSigningKeys }, options.cryptoCallbacks || {},
     );
@@ -86,20 +91,21 @@ describe("Cross Signing", function() {
         const { client: alice } = await makeTestClient(
             { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
-        alice.uploadDeviceSigningKeys = jest.fn(async (auth, keys) => {
+        alice.uploadDeviceSigningKeys = jest.fn().mockImplementation(async (auth, keys) => {
             await olmlib.verifySignature(
                 alice.crypto.olmDevice, keys.master_key, "@alice:example.com",
                 "Osborne2", alice.crypto.olmDevice.deviceEd25519Key,
             );
         });
-        alice.uploadKeySignatures = async () => {};
-        alice.setAccountData = async () => {};
-        alice.getAccountDataFromServer = async () => {};
+        alice.uploadKeySignatures = async () => ({ failures: {} });
+        alice.setAccountData = async () => ({});
+        alice.getAccountDataFromServer = async <T>() => ({} as T);
         // set Alice's cross-signing key
         await alice.bootstrapCrossSigning({
-            authUploadDeviceSigningKeys: async func => await func({}),
+            authUploadDeviceSigningKeys: async func => { await func({}); },
         });
         expect(alice.uploadDeviceSigningKeys).toHaveBeenCalled();
+        alice.stopClient();
     });
 
     it("should abort bootstrap if device signing auth fails", async function() {
@@ -133,9 +139,9 @@ describe("Cross Signing", function() {
             error.httpStatus == 401;
             throw error;
         };
-        alice.uploadKeySignatures = async () => {};
-        alice.setAccountData = async () => {};
-        alice.getAccountDataFromServer = async () => { };
+        alice.uploadKeySignatures = async () => ({ failures: {} });
+        alice.setAccountData = async () => ({});
+        alice.getAccountDataFromServer = async <T extends {[k: string]: any}>(): Promise<T> => ({} as T);
         const authUploadDeviceSigningKeys = async func => await func({});
 
         // Try bootstrap, expecting `authUploadDeviceSigningKeys` to pass
@@ -151,14 +157,15 @@ describe("Cross Signing", function() {
             }
         }
         expect(bootstrapDidThrow).toBeTruthy();
+        alice.stopClient();
     });
 
     it("should upload a signature when a user is verified", async function() {
         const { client: alice } = await makeTestClient(
             { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
-        alice.uploadDeviceSigningKeys = async () => {};
-        alice.uploadKeySignatures = async () => {};
+        alice.uploadDeviceSigningKeys = async () => ({});
+        alice.uploadKeySignatures = async () => ({ failures: {} });
         // set Alice's cross-signing key
         await resetCrossSigningKeys(alice);
         // Alice downloads Bob's device key
@@ -172,16 +179,20 @@ describe("Cross Signing", function() {
                     },
                 },
             },
+            firstUse: false,
+            crossSigningVerifiedBefore: false,
         });
         // Alice verifies Bob's key
         const promise = new Promise((resolve, reject) => {
-            alice.uploadKeySignatures = (...args) => {
+            alice.uploadKeySignatures = async (...args) => {
                 resolve(...args);
+                return { failures: {} };
             };
         });
         await alice.setDeviceVerified("@bob:example.com", "bobs+master+pubkey", true);
         // Alice should send a signature of Bob's key to the server
         await promise;
+        alice.stopClient();
     });
 
     it.skip("should get cross-signing keys from sync", async function() {
@@ -203,7 +214,7 @@ describe("Cross Signing", function() {
             {
                 cryptoCallbacks: {
                     // will be called to sign our own device
-                    getCrossSigningKey: type => {
+                    getCrossSigningKey: async type => {
                         if (type === 'master') {
                             return masterKey;
                         } else {
@@ -215,7 +226,7 @@ describe("Cross Signing", function() {
         );
 
         const keyChangePromise = new Promise((resolve, reject) => {
-            alice.once("crossSigning.keysChanged", async (e) => {
+            alice.once(CryptoEvent.KeysChanged, async (e) => {
                 resolve(e);
                 await alice.checkOwnCrossSigningTrust({
                     allowPrivateKeyRequests: true,
@@ -223,14 +234,14 @@ describe("Cross Signing", function() {
             });
         });
 
-        const uploadSigsPromise = new Promise((resolve, reject) => {
-            alice.uploadKeySignatures = jest.fn(async (content) => {
+        const uploadSigsPromise = new Promise<void>((resolve, reject) => {
+            alice.uploadKeySignatures = jest.fn().mockImplementation(async (content) => {
                 try {
                     await olmlib.verifySignature(
                         alice.crypto.olmDevice,
                         content["@alice:example.com"][
                             "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
-                            ],
+                        ],
                         "@alice:example.com",
                         "Osborne2", alice.crypto.olmDevice.deviceEd25519Key,
                     );
@@ -246,16 +257,22 @@ describe("Cross Signing", function() {
             });
         });
 
+        // @ts-ignore private property
         const deviceInfo = alice.crypto.deviceList.devices["@alice:example.com"]
             .Osborne2;
         const aliceDevice = {
             user_id: "@alice:example.com",
             device_id: "Osborne2",
+            keys: deviceInfo.keys,
+            algorithms: deviceInfo.algorithms,
         };
-        aliceDevice.keys = deviceInfo.keys;
-        aliceDevice.algorithms = deviceInfo.algorithms;
         await alice.crypto.signObject(aliceDevice);
-        olmlib.pkSign(aliceDevice, selfSigningKey, "@alice:example.com");
+        olmlib.pkSign(
+            aliceDevice as ISignedKey,
+            selfSigningKey as unknown as PkSigning,
+            "@alice:example.com",
+            '',
+        );
 
         // feed sync result that includes master key, ssk, device key
         const responses = [
@@ -353,14 +370,15 @@ describe("Cross Signing", function() {
         expect(aliceDeviceTrust.isLocallyVerified()).toBeTruthy();
         expect(aliceDeviceTrust.isTofu()).toBeTruthy();
         expect(aliceDeviceTrust.isVerified()).toBeTruthy();
+        alice.stopClient();
     });
 
     it("should use trust chain to determine device verification", async function() {
         const { client: alice } = await makeTestClient(
             { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
-        alice.uploadDeviceSigningKeys = async () => {};
-        alice.uploadKeySignatures = async () => {};
+        alice.uploadDeviceSigningKeys = async () => ({});
+        alice.uploadKeySignatures = async () => ({ failures: {} });
         // set Alice's cross-signing key
         await resetCrossSigningKeys(alice);
         // Alice downloads Bob's ssk and device key
@@ -370,7 +388,7 @@ describe("Cross Signing", function() {
         const bobSigning = new global.Olm.PkSigning();
         const bobPrivkey = bobSigning.generate_seed();
         const bobPubkey = bobSigning.init_with_seed(bobPrivkey);
-        const bobSSK = {
+        const bobSSK: ICrossSigningKey = {
             user_id: "@bob:example.com",
             usage: ["self_signing"],
             keys: {
@@ -394,10 +412,10 @@ describe("Cross Signing", function() {
                 },
                 self_signing: bobSSK,
             },
-            firstUse: 1,
-            unsigned: {},
+            firstUse: true,
+            crossSigningVerifiedBefore: false,
         });
-        const bobDevice = {
+        const bobDeviceUnsigned = {
             user_id: "@bob:example.com",
             device_id: "Dynabook",
             algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
@@ -406,11 +424,16 @@ describe("Cross Signing", function() {
                 "ed25519:Dynabook": "someOtherPubkey",
             },
         };
-        const sig = bobSigning.sign(anotherjson.stringify(bobDevice));
-        bobDevice.signatures = {
-            "@bob:example.com": {
-                ["ed25519:" + bobPubkey]: sig,
+        const sig = bobSigning.sign(anotherjson.stringify(bobDeviceUnsigned));
+        const bobDevice: IDevice = {
+            ...bobDeviceUnsigned,
+            signatures: {
+                "@bob:example.com": {
+                    ["ed25519:" + bobPubkey]: sig,
+                },
             },
+            verified: 0,
+            known: false,
         };
         alice.crypto.deviceList.storeDevicesForUser("@bob:example.com", {
             Dynabook: bobDevice,
@@ -425,7 +448,7 @@ describe("Cross Signing", function() {
         expect(bobDeviceTrust.isTofu()).toBeTruthy();
 
         // Alice verifies Bob's SSK
-        alice.uploadKeySignatures = () => {};
+        alice.uploadKeySignatures = async () => ({ failures: {} });
         await alice.setDeviceVerified("@bob:example.com", bobMasterPubkey, true);
 
         // Bob's device key should be trusted
@@ -437,10 +460,11 @@ describe("Cross Signing", function() {
         expect(bobDeviceTrust2.isCrossSigningVerified()).toBeTruthy();
         expect(bobDeviceTrust2.isLocallyVerified()).toBeFalsy();
         expect(bobDeviceTrust2.isTofu()).toBeTruthy();
+        alice.stopClient();
     });
 
     it.skip("should trust signatures received from other devices", async function() {
-        const aliceKeys = {};
+        const aliceKeys: Record<string, PkSigning> = {};
         const { client: alice, httpBackend } = await makeTestClient(
             { userId: "@alice:example.com", deviceId: "Osborne2" },
             null,
@@ -448,8 +472,8 @@ describe("Cross Signing", function() {
         );
         alice.crypto.deviceList.startTrackingDeviceList("@bob:example.com");
         alice.crypto.deviceList.stopTrackingAllDeviceLists = () => {};
-        alice.uploadDeviceSigningKeys = async () => {};
-        alice.uploadKeySignatures = async () => {};
+        alice.uploadDeviceSigningKeys = async () => ({});
+        alice.uploadKeySignatures = async () => ({ failures: {} });
 
         // set Alice's cross-signing key
         await resetCrossSigningKeys(alice);
@@ -461,28 +485,29 @@ describe("Cross Signing", function() {
             0x34, 0xf2, 0x4b, 0x64, 0x9b, 0x52, 0xf8, 0x5f,
         ]);
 
-        const keyChangePromise = new Promise((resolve, reject) => {
-            alice.crypto.deviceList.once("userCrossSigningUpdated", (userId) => {
+        const keyChangePromise = new Promise<void>((resolve, reject) => {
+            alice.crypto.deviceList.once(CryptoEvent.UserCrossSigningUpdated, (userId) => {
                 if (userId === "@bob:example.com") {
                     resolve();
                 }
             });
         });
 
+        // @ts-ignore private property
         const deviceInfo = alice.crypto.deviceList.devices["@alice:example.com"]
             .Osborne2;
         const aliceDevice = {
             user_id: "@alice:example.com",
             device_id: "Osborne2",
+            keys: deviceInfo.keys,
+            algorithms: deviceInfo.algorithms,
         };
-        aliceDevice.keys = deviceInfo.keys;
-        aliceDevice.algorithms = deviceInfo.algorithms;
         await alice.crypto.signObject(aliceDevice);
 
         const bobOlmAccount = new global.Olm.Account();
         bobOlmAccount.create();
         const bobKeys = JSON.parse(bobOlmAccount.identity_keys());
-        const bobDevice = {
+        const bobDeviceUnsigned = {
             user_id: "@bob:example.com",
             device_id: "Dynabook",
             algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
@@ -491,15 +516,25 @@ describe("Cross Signing", function() {
                 "curve25519:Dynabook": bobKeys.curve25519,
             },
         };
-        const deviceStr = anotherjson.stringify(bobDevice);
-        bobDevice.signatures = {
-            "@bob:example.com": {
-                "ed25519:Dynabook": bobOlmAccount.sign(deviceStr),
+        const deviceStr = anotherjson.stringify(bobDeviceUnsigned);
+        const bobDevice: IDevice = {
+            ...bobDeviceUnsigned,
+            signatures: {
+                "@bob:example.com": {
+                    "ed25519:Dynabook": bobOlmAccount.sign(deviceStr),
+                },
             },
+            verified: 0,
+            known: false,
         };
-        olmlib.pkSign(bobDevice, selfSigningKey, "@bob:example.com");
+        olmlib.pkSign(
+            bobDevice,
+            selfSigningKey as unknown as PkSigning,
+            "@bob:example.com",
+            '',
+        );
 
-        const bobMaster = {
+        const bobMaster: ICrossSigningKey = {
             user_id: "@bob:example.com",
             usage: ["master"],
             keys: {
@@ -507,7 +542,7 @@ describe("Cross Signing", function() {
                 "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk",
             },
         };
-        olmlib.pkSign(bobMaster, aliceKeys.user_signing, "@alice:example.com");
+        olmlib.pkSign(bobMaster, aliceKeys.user_signing, "@alice:example.com", '');
 
         // Alice downloads Bob's keys
         // - device key
@@ -600,14 +635,15 @@ describe("Cross Signing", function() {
         expect(bobDeviceTrust.isCrossSigningVerified()).toBeTruthy();
         expect(bobDeviceTrust.isLocallyVerified()).toBeFalsy();
         expect(bobDeviceTrust.isTofu()).toBeTruthy();
+        alice.stopClient();
     });
 
     it("should dis-trust an unsigned device", async function() {
         const { client: alice } = await makeTestClient(
             { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
-        alice.uploadDeviceSigningKeys = async () => {};
-        alice.uploadKeySignatures = async () => {};
+        alice.uploadDeviceSigningKeys = async () => ({});
+        alice.uploadKeySignatures = async () => ({ failures: {} });
         // set Alice's cross-signing key
         await resetCrossSigningKeys(alice);
         // Alice downloads Bob's ssk and device key
@@ -618,7 +654,7 @@ describe("Cross Signing", function() {
         const bobSigning = new global.Olm.PkSigning();
         const bobPrivkey = bobSigning.generate_seed();
         const bobPubkey = bobSigning.init_with_seed(bobPrivkey);
-        const bobSSK = {
+        const bobSSK: ICrossSigningKey = {
             user_id: "@bob:example.com",
             usage: ["self_signing"],
             keys: {
@@ -642,8 +678,8 @@ describe("Cross Signing", function() {
                 },
                 self_signing: bobSSK,
             },
-            firstUse: 1,
-            unsigned: {},
+            firstUse: true,
+            crossSigningVerifiedBefore: false,
         });
         const bobDevice = {
             user_id: "@bob:example.com",
@@ -655,7 +691,7 @@ describe("Cross Signing", function() {
             },
         };
         alice.crypto.deviceList.storeDevicesForUser("@bob:example.com", {
-            Dynabook: bobDevice,
+            Dynabook: bobDevice as unknown as IDevice,
         });
         // Bob's device key should be untrusted
         const bobDeviceTrust = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
@@ -669,14 +705,15 @@ describe("Cross Signing", function() {
         const bobDeviceTrust2 = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
         expect(bobDeviceTrust2.isVerified()).toBeFalsy();
         expect(bobDeviceTrust2.isTofu()).toBeFalsy();
+        alice.stopClient();
     });
 
     it("should dis-trust a user when their ssk changes", async function() {
         const { client: alice } = await makeTestClient(
             { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
-        alice.uploadDeviceSigningKeys = async () => {};
-        alice.uploadKeySignatures = async () => {};
+        alice.uploadDeviceSigningKeys = async () => ({});
+        alice.uploadKeySignatures = async () => ({ failures: {} });
         await resetCrossSigningKeys(alice);
         // Alice downloads Bob's keys
         const bobMasterSigning = new global.Olm.PkSigning();
@@ -685,7 +722,7 @@ describe("Cross Signing", function() {
         const bobSigning = new global.Olm.PkSigning();
         const bobPrivkey = bobSigning.generate_seed();
         const bobPubkey = bobSigning.init_with_seed(bobPrivkey);
-        const bobSSK = {
+        const bobSSK: ICrossSigningKey = {
             user_id: "@bob:example.com",
             usage: ["self_signing"],
             keys: {
@@ -709,10 +746,10 @@ describe("Cross Signing", function() {
                 },
                 self_signing: bobSSK,
             },
-            firstUse: 1,
-            unsigned: {},
+            firstUse: true,
+            crossSigningVerifiedBefore: false,
         });
-        const bobDevice = {
+        const bobDeviceUnsigned = {
             user_id: "@bob:example.com",
             device_id: "Dynabook",
             algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
@@ -721,16 +758,23 @@ describe("Cross Signing", function() {
                 "ed25519:Dynabook": "someOtherPubkey",
             },
         };
-        const bobDeviceString = anotherjson.stringify(bobDevice);
+        const bobDeviceString = anotherjson.stringify(bobDeviceUnsigned);
         const sig = bobSigning.sign(bobDeviceString);
-        bobDevice.signatures = {};
-        bobDevice.signatures["@bob:example.com"] = {};
-        bobDevice.signatures["@bob:example.com"]["ed25519:" + bobPubkey] = sig;
+        const bobDevice: IDevice = {
+            ...bobDeviceUnsigned,
+            verified: 0,
+            known: false,
+            signatures: {
+                "@bob:example.com": {
+                    ["ed25519:" + bobPubkey]: sig,
+                },
+            },
+        };
         alice.crypto.deviceList.storeDevicesForUser("@bob:example.com", {
             Dynabook: bobDevice,
         });
         // Alice verifies Bob's SSK
-        alice.uploadKeySignatures = () => {};
+        alice.uploadKeySignatures = async () => ({ failures: {} });
         await alice.setDeviceVerified("@bob:example.com", bobMasterPubkey, true);
 
         // Bob's device key should be trusted
@@ -745,7 +789,7 @@ describe("Cross Signing", function() {
         const bobSigning2 = new global.Olm.PkSigning();
         const bobPrivkey2 = bobSigning2.generate_seed();
         const bobPubkey2 = bobSigning2.init_with_seed(bobPrivkey2);
-        const bobSSK2 = {
+        const bobSSK2: ICrossSigningKey = {
             user_id: "@bob:example.com",
             usage: ["self_signing"],
             keys: {
@@ -769,8 +813,8 @@ describe("Cross Signing", function() {
                 },
                 self_signing: bobSSK2,
             },
-            firstUse: 0,
-            unsigned: {},
+            firstUse: false,
+            crossSigningVerifiedBefore: false,
         });
         // Bob's and his device should be untrusted
         const bobTrust = alice.checkUserTrust("@bob:example.com");
@@ -782,7 +826,7 @@ describe("Cross Signing", function() {
         expect(bobDeviceTrust2.isTofu()).toBeFalsy();
 
         // Alice verifies Bob's SSK
-        alice.uploadKeySignatures = () => {};
+        alice.uploadKeySignatures = async () => ({ failures: {} });
         await alice.setDeviceVerified("@bob:example.com", bobMasterPubkey2, true);
 
         // Bob should be trusted but not his device
@@ -805,6 +849,7 @@ describe("Cross Signing", function() {
 
         const bobDeviceTrust4 = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
         expect(bobDeviceTrust4.isCrossSigningVerified()).toBeTruthy();
+        alice.stopClient();
     });
 
     it("should offer to upgrade device verifications to cross-signing", async function() {
@@ -814,20 +859,21 @@ describe("Cross Signing", function() {
             { userId: "@alice:example.com", deviceId: "Osborne2" },
             {
                 cryptoCallbacks: {
-                    shouldUpgradeDeviceVerifications: (verifs) => {
+                    shouldUpgradeDeviceVerifications: async (verifs) => {
                         expect(verifs.users["@bob:example.com"]).toBeDefined();
                         upgradeResolveFunc();
                         return ["@bob:example.com"];
                     },
                 },
             },
+
         );
         const { client: bob } = await makeTestClient(
             { userId: "@bob:example.com", deviceId: "Dynabook" },
         );
 
-        bob.uploadDeviceSigningKeys = async () => {};
-        bob.uploadKeySignatures = async () => {};
+        bob.uploadDeviceSigningKeys = async () => ({});
+        bob.uploadKeySignatures = async () => ({ failures: {} });
         // set Bob's cross-signing key
         await resetCrossSigningKeys(bob);
         alice.crypto.deviceList.storeDevicesForUser("@bob:example.com", {
@@ -846,8 +892,8 @@ describe("Cross Signing", function() {
             bob.crypto.crossSigningInfo.toStorage(),
         );
 
-        alice.uploadDeviceSigningKeys = async () => {};
-        alice.uploadKeySignatures = async () => {};
+        alice.uploadDeviceSigningKeys = async () => ({});
+        alice.uploadKeySignatures = async () => ({ failures: {} });
         // when alice sets up cross-signing, she should notice that bob's
         // cross-signing key is signed by his Dynabook, which alice has
         // verified, and ask if the device verification should be upgraded to a
@@ -873,15 +919,17 @@ describe("Cross Signing", function() {
         upgradePromise = new Promise((resolve) => {
             upgradeResolveFunc = resolve;
         });
-        alice.crypto.deviceList.emit("userCrossSigningUpdated", "@bob:example.com");
+        alice.crypto.deviceList.emit(CryptoEvent.UserCrossSigningUpdated, "@bob:example.com");
         await new Promise((resolve) => {
-            alice.crypto.on("userTrustStatusChanged", resolve);
+            alice.crypto.on(CryptoEvent.UserTrustStatusChanged, resolve);
         });
         await upgradePromise;
 
         const bobTrust3 = alice.checkUserTrust("@bob:example.com");
         expect(bobTrust3.isCrossSigningVerified()).toBeTruthy();
         expect(bobTrust3.isTofu()).toBeTruthy();
+        alice.stopClient();
+        bob.stopClient();
     });
 
     it(
@@ -890,8 +938,8 @@ describe("Cross Signing", function() {
             const { client: alice } = await makeTestClient(
                 { userId: "@alice:example.com", deviceId: "Osborne2" },
             );
-            alice.uploadDeviceSigningKeys = async () => {};
-            alice.uploadKeySignatures = async () => {};
+            alice.uploadDeviceSigningKeys = async () => ({});
+            alice.uploadKeySignatures = async () => ({ failures: {} });
 
             // Generate Alice's SSK etc
             const aliceMasterSigning = new global.Olm.PkSigning();
@@ -900,7 +948,7 @@ describe("Cross Signing", function() {
             const aliceSigning = new global.Olm.PkSigning();
             const alicePrivkey = aliceSigning.generate_seed();
             const alicePubkey = aliceSigning.init_with_seed(alicePrivkey);
-            const aliceSSK = {
+            const aliceSSK: ICrossSigningKey = {
                 user_id: "@alice:example.com",
                 usage: ["self_signing"],
                 keys: {
@@ -926,34 +974,42 @@ describe("Cross Signing", function() {
                     },
                     self_signing: aliceSSK,
                 },
-                firstUse: 1,
-                unsigned: {},
+                firstUse: true,
+                crossSigningVerifiedBefore: false,
             });
 
             // Alice has a second device that's cross-signed
-            const aliceCrossSignedDevice = {
+            const aliceDeviceId = 'Dynabook';
+            const aliceUnsignedDevice = {
                 user_id: "@alice:example.com",
-                device_id: "Dynabook",
+                device_id: aliceDeviceId,
                 algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
                 keys: {
                     "curve25519:Dynabook": "somePubkey",
                     "ed25519:Dynabook": "someOtherPubkey",
                 },
             };
-            const sig = aliceSigning.sign(anotherjson.stringify(aliceCrossSignedDevice));
-            aliceCrossSignedDevice.signatures = {
-                "@alice:example.com": {
-                    ["ed25519:" + alicePubkey]: sig,
-                },
-            };
+            const sig = aliceSigning.sign(anotherjson.stringify(aliceUnsignedDevice));
+            const aliceCrossSignedDevice: IDevice = {
+                ...aliceUnsignedDevice,
+                verified: 0,
+                known: false,
+                signatures: {
+                    "@alice:example.com": {
+                        ["ed25519:" + alicePubkey]: sig,
+                    },
+                } };
             alice.crypto.deviceList.storeDevicesForUser("@alice:example.com", {
-                Dynabook: aliceCrossSignedDevice,
+                [aliceDeviceId]: aliceCrossSignedDevice,
             });
 
             // We don't trust the cross-signing keys yet...
-            expect(alice.checkDeviceTrust(aliceCrossSignedDevice.device_id).isCrossSigningVerified()).toBeFalsy();
+            expect(
+                alice.checkDeviceTrust("@alice:example.com", aliceDeviceId).isCrossSigningVerified(),
+            ).toBeFalsy();
             // ... but we do acknowledge that the device is signed by them
-            expect(alice.checkIfOwnDeviceCrossSigned(aliceCrossSignedDevice.device_id)).toBeTruthy();
+            expect(alice.checkIfOwnDeviceCrossSigned(aliceDeviceId)).toBeTruthy();
+            alice.stopClient();
         },
     );
 
@@ -961,8 +1017,8 @@ describe("Cross Signing", function() {
         const { client: alice } = await makeTestClient(
             { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
-        alice.uploadDeviceSigningKeys = async () => {};
-        alice.uploadKeySignatures = async () => {};
+        alice.uploadDeviceSigningKeys = async () => ({});
+        alice.uploadKeySignatures = async () => ({ failures: {} });
 
         // Generate Alice's SSK etc
         const aliceMasterSigning = new global.Olm.PkSigning();
@@ -971,7 +1027,7 @@ describe("Cross Signing", function() {
         const aliceSigning = new global.Olm.PkSigning();
         const alicePrivkey = aliceSigning.generate_seed();
         const alicePubkey = aliceSigning.init_with_seed(alicePrivkey);
-        const aliceSSK = {
+        const aliceSSK: ICrossSigningKey = {
             user_id: "@alice:example.com",
             usage: ["self_signing"],
             keys: {
@@ -997,14 +1053,14 @@ describe("Cross Signing", function() {
                 },
                 self_signing: aliceSSK,
             },
-            firstUse: 1,
-            unsigned: {},
+            firstUse: true,
+            crossSigningVerifiedBefore: false,
         });
 
-        // Alice has a second device that's also not cross-signed
-        const aliceNotCrossSignedDevice = {
-            user_id: "@alice:example.com",
-            device_id: "Dynabook",
+        const deviceId = "Dynabook";
+        const aliceNotCrossSignedDevice: IDevice = {
+            verified: 0,
+            known: false,
             algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
             keys: {
                 "curve25519:Dynabook": "somePubkey",
@@ -1012,9 +1068,10 @@ describe("Cross Signing", function() {
             },
         };
         alice.crypto.deviceList.storeDevicesForUser("@alice:example.com", {
-            Dynabook: aliceNotCrossSignedDevice,
+            [deviceId]: aliceNotCrossSignedDevice,
         });
 
-        expect(alice.checkIfOwnDeviceCrossSigned(aliceNotCrossSignedDevice.device_id)).toBeFalsy();
+        expect(alice.checkIfOwnDeviceCrossSigned(deviceId)).toBeFalsy();
+        alice.stopClient();
     });
 });

spec/unit/crypto/crypto-utils.ts

@@ -1,11 +1,13 @@
+import { IRecoveryKey } from '../../../src/crypto/api';
+import { CrossSigningLevel } from '../../../src/crypto/CrossSigning';
 import { IndexedDBCryptoStore } from '../../../src/crypto/store/indexeddb-crypto-store';
 
 // needs to be phased out and replaced with bootstrapSecretStorage,
 // but that is doing too much extra stuff for it to be an easy transition.
-export async function resetCrossSigningKeys(client, {
-    level,
-    authUploadDeviceSigningKeys = async func => await func(),
-} = {}) {
+export async function resetCrossSigningKeys(
+    client,
+    { level }: { level?: CrossSigningLevel} = {},
+): Promise<void> {
     const crypto = client.crypto;
 
     const oldKeys = Object.assign({}, crypto.crossSigningInfo.keys);
@@ -30,14 +32,14 @@ export async function resetCrossSigningKeys(client, {
     await crypto.afterCrossSigningLocalKeyChange();
 }
 
-export async function createSecretStorageKey() {
+export async function createSecretStorageKey(): Promise<IRecoveryKey> {
     const decryption = new global.Olm.PkDecryption();
     const storagePublicKey = decryption.generate_key();
     const storagePrivateKey = decryption.get_private_key();
     decryption.free();
     return {
         // `pubkey` not used anymore with symmetric 4S
-        keyInfo: { pubkey: storagePublicKey },
+        keyInfo: { pubkey: storagePublicKey, key: undefined },
         privateKey: storagePrivateKey,
     };
 }

spec/unit/crypto/outgoing-room-key-requests.spec.ts

@@ -43,13 +43,15 @@ const requests = [
 
 describe.each([
     ["IndexedDBCryptoStore",
-     () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+        () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
     ["LocalStorageCryptoStore",
-     () => new IndexedDBCryptoStore(undefined, "tests")],
+        () => new IndexedDBCryptoStore(undefined, "tests")],
     ["MemoryCryptoStore", () => {
         const store = new IndexedDBCryptoStore(undefined, "tests");
-        store._backend = new MemoryCryptoStore();
-        store._backendPromise = Promise.resolve(store._backend);
+        // @ts-ignore set private properties
+        store.backend = new MemoryCryptoStore();
+        // @ts-ignore
+        store.backendPromise = Promise.resolve(store.backend);
         return store;
     }],
 ])("Outgoing room key requests [%s]", function(name, dbFactory) {
@@ -64,22 +66,22 @@ describe.each([
     });
 
     it("getAllOutgoingRoomKeyRequestsByState retrieves all entries in a given state",
-    async () => {
-        const r = await
+        async () => {
+            const r = await
             store.getAllOutgoingRoomKeyRequestsByState(RoomKeyRequestState.Sent);
-        expect(r).toHaveLength(2);
-        requests.filter((e) => e.state === RoomKeyRequestState.Sent).forEach((e) => {
-            expect(r).toContainEqual(e);
+            expect(r).toHaveLength(2);
+            requests.filter((e) => e.state === RoomKeyRequestState.Sent).forEach((e) => {
+                expect(r).toContainEqual(e);
+            });
         });
-    });
 
     test("getOutgoingRoomKeyRequestByState retrieves any entry in a given state",
-    async () => {
-        const r =
+        async () => {
+            const r =
             await store.getOutgoingRoomKeyRequestByState([RoomKeyRequestState.Sent]);
-        expect(r).not.toBeNull();
-        expect(r).not.toBeUndefined();
-        expect(r.state).toEqual(RoomKeyRequestState.Sent);
-        expect(requests).toContainEqual(r);
-    });
+            expect(r).not.toBeNull();
+            expect(r).not.toBeUndefined();
+            expect(r.state).toEqual(RoomKeyRequestState.Sent);
+            expect(requests).toContainEqual(r);
+        });
 });

spec/unit/crypto/secrets.spec.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2022 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -24,15 +24,18 @@ import { encryptAES } from "../../../src/crypto/aes";
 import { resetCrossSigningKeys, createSecretStorageKey } from "./crypto-utils";
 import { logger } from '../../../src/logger';
 import * as utils from "../../../src/utils";
+import { ICreateClientOpts } from '../../../src/client';
+import { ISecretStorageKeyInfo } from '../../../src/crypto/api';
 
 try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
     const crypto = require('crypto');
     utils.setCrypto(crypto);
 } catch (err) {
     logger.log('nodejs was compiled without crypto support');
 }
 
-async function makeTestClient(userInfo, options) {
+async function makeTestClient(userInfo: { userId: string, deviceId: string}, options: Partial<ICreateClientOpts> = {}) {
     const client = (new TestClient(
         userInfo.userId, userInfo.deviceId, undefined, undefined, options,
     )).client;
@@ -46,7 +49,7 @@ async function makeTestClient(userInfo, options) {
     await client.initCrypto();
 
     // No need to download keys for these tests
-    client.crypto.downloadKeys = async function() {};
+    jest.spyOn(client.crypto, 'downloadKeys').mockResolvedValue({});
 
     return client;
 }
@@ -54,7 +57,7 @@ async function makeTestClient(userInfo, options) {
 // Wrapper around pkSign to return a signed object. pkSign returns the
 // signature, rather than the signed object.
 function sign(obj, key, userId) {
-    olmlib.pkSign(obj, key, userId);
+    olmlib.pkSign(obj, key, userId, '');
     return obj;
 }
 
@@ -84,7 +87,7 @@ describe("Secrets", function() {
             },
         };
 
-        const getKey = jest.fn(e => {
+        const getKey = jest.fn().mockImplementation(async e => {
             expect(Object.keys(e.keys)).toEqual(["abc"]);
             return ['abc', key];
         });
@@ -93,7 +96,7 @@ describe("Secrets", function() {
             { userId: "@alice:example.com", deviceId: "Osborne2" },
             {
                 cryptoCallbacks: {
-                    getCrossSigningKey: t => signingKey,
+                    getCrossSigningKey: async t => signingKey,
                     getSecretStorageKey: getKey,
                 },
             },
@@ -104,17 +107,19 @@ describe("Secrets", function() {
 
         const secretStorage = alice.crypto.secretStorage;
 
-        alice.setAccountData = async function(eventType, contents, callback) {
-            alice.store.storeAccountDataEvents([
-                new MatrixEvent({
-                    type: eventType,
-                    content: contents,
-                }),
-            ]);
-            if (callback) {
-                callback();
-            }
-        };
+        jest.spyOn(alice, 'setAccountData').mockImplementation(
+            async function(eventType, contents, callback) {
+                alice.store.storeAccountDataEvents([
+                    new MatrixEvent({
+                        type: eventType,
+                        content: contents,
+                    }),
+                ]);
+                if (callback) {
+                    callback(undefined, undefined);
+                }
+                return {};
+            });
 
         const keyAccountData = {
             algorithm: SECRET_STORAGE_ALGORITHM_V1_AES,
@@ -136,6 +141,7 @@ describe("Secrets", function() {
         expect(await secretStorage.get("foo")).toBe("bar");
 
         expect(getKey).toHaveBeenCalled();
+        alice.stopClient();
     });
 
     it("should throw if given a key that doesn't exist", async function() {
@@ -150,6 +156,7 @@ describe("Secrets", function() {
             expect(true).toBeFalsy();
         } catch (e) {
         }
+        alice.stopClient();
     });
 
     it("should refuse to encrypt with zero keys", async function() {
@@ -162,12 +169,13 @@ describe("Secrets", function() {
             expect(true).toBeFalsy();
         } catch (e) {
         }
+        alice.stopClient();
     });
 
     it("should encrypt with default key if keys is null", async function() {
         const key = new Uint8Array(16);
         for (let i = 0; i < 16; i++) key[i] = i;
-        const getKey = jest.fn(e => {
+        const getKey = jest.fn().mockImplementation(async e => {
             expect(Object.keys(e.keys)).toEqual([newKeyId]);
             return [newKeyId, key];
         });
@@ -190,11 +198,12 @@ describe("Secrets", function() {
                     content: contents,
                 }),
             ]);
+            return {};
         };
         resetCrossSigningKeys(alice);
 
         const { keyId: newKeyId } = await alice.addSecretStorageKey(
-            SECRET_STORAGE_ALGORITHM_V1_AES,
+            SECRET_STORAGE_ALGORITHM_V1_AES, { pubkey: undefined, key: undefined },
         );
         // we don't await on this because it waits for the event to come down the sync
         // which won't happen in the test setup
@@ -203,6 +212,7 @@ describe("Secrets", function() {
 
         const accountData = alice.getAccountData('foo');
         expect(accountData.getContent().encrypted).toBeTruthy();
+        alice.stopClient();
     });
 
     it("should refuse to encrypt if no keys given and no default key", async function() {
@@ -215,10 +225,11 @@ describe("Secrets", function() {
             expect(true).toBeFalsy();
         } catch (e) {
         }
+        alice.stopClient();
     });
 
     it("should request secrets from other clients", async function() {
-        const [osborne2, vax] = await makeTestClients(
+        const [[osborne2, vax], clearTestClientTimeouts] = await makeTestClients(
             [
                 { userId: "@alice:example.com", deviceId: "Osborne2" },
                 { userId: "@alice:example.com", deviceId: "VAX" },
@@ -273,6 +284,9 @@ describe("Secrets", function() {
         const secret = await request.promise;
 
         expect(secret).toBe("bar");
+        osborne2.stop();
+        vax.stop();
+        clearTestClientTimeouts();
     });
 
     describe("bootstrap", function() {
@@ -298,7 +312,7 @@ describe("Secrets", function() {
         it("bootstraps when no storage or cross-signing keys locally", async function() {
             const key = new Uint8Array(16);
             for (let i = 0; i < 16; i++) key[i] = i;
-            const getKey = jest.fn(e => {
+            const getKey = jest.fn().mockImplementation(async e => {
                 return [Object.keys(e.keys)[0], key];
             });
 
@@ -313,8 +327,8 @@ describe("Secrets", function() {
                     },
                 },
             );
-            bob.uploadDeviceSigningKeys = async () => {};
-            bob.uploadKeySignatures = async () => {};
+            bob.uploadDeviceSigningKeys = async () => ({});
+            bob.uploadKeySignatures = jest.fn().mockResolvedValue(undefined);
             bob.setAccountData = async function(eventType, contents, callback) {
                 const event = new MatrixEvent({
                     type: eventType,
@@ -324,10 +338,11 @@ describe("Secrets", function() {
                     event,
                 ]);
                 this.emit("accountData", event);
+                return {};
             };
 
             await bob.bootstrapCrossSigning({
-                authUploadDeviceSigningKeys: async func => await func({}),
+                authUploadDeviceSigningKeys: async func => { await func({}); },
             });
             await bob.bootstrapSecretStorage({
                 createSecretStorageKey,
@@ -340,6 +355,7 @@ describe("Secrets", function() {
             expect(await crossSigning.isStoredInSecretStorage(secretStorage))
                 .toBeTruthy();
             expect(await secretStorage.hasKey()).toBeTruthy();
+            bob.stopClient();
         });
 
         it("bootstraps when cross-signing keys in secret storage", async function() {
@@ -406,10 +422,11 @@ describe("Secrets", function() {
             expect(await crossSigning.isStoredInSecretStorage(secretStorage))
                 .toBeTruthy();
             expect(await secretStorage.hasKey()).toBeTruthy();
+            bob.stopClient();
         });
 
         it("adds passphrase checking if it's lacking", async function() {
-            let crossSigningKeys = {
+            let crossSigningKeys: Record<string, Uint8Array> = {
                 master: XSK,
                 user_signing: USK,
                 self_signing: SSK,
@@ -421,9 +438,9 @@ describe("Secrets", function() {
                 { userId: "@alice:example.com", deviceId: "Osborne2" },
                 {
                     cryptoCallbacks: {
-                        getCrossSigningKey: t => crossSigningKeys[t],
+                        getCrossSigningKey: async t => crossSigningKeys[t],
                         saveCrossSigningKeys: k => crossSigningKeys = k,
-                        getSecretStorageKey: ({ keys }, name) => {
+                        getSecretStorageKey: async ({ keys }, name) => {
                             for (const keyId of Object.keys(keys)) {
                                 if (secretStorageKeys[keyId]) {
                                     return [keyId, secretStorageKeys[keyId]];
@@ -479,6 +496,8 @@ describe("Secrets", function() {
                 }),
             ]);
             alice.crypto.deviceList.storeCrossSigningForUser("@alice:example.com", {
+                firstUse: false,
+                crossSigningVerifiedBefore: false,
                 keys: {
                     master: {
                         user_id: "@alice:example.com",
@@ -519,14 +538,15 @@ describe("Secrets", function() {
                 });
                 alice.store.storeAccountDataEvents([event]);
                 this.emit("accountData", event);
+                return {};
             };
 
-            await alice.bootstrapSecretStorage();
+            await alice.bootstrapSecretStorage({});
 
             expect(alice.getAccountData("m.secret_storage.default_key").getContent())
                 .toEqual({ key: "key_id" });
             const keyInfo = alice.getAccountData("m.secret_storage.key.key_id")
-                .getContent();
+                .getContent() as ISecretStorageKeyInfo;
             expect(keyInfo.algorithm)
                 .toEqual("m.secret_storage.v1.aes-hmac-sha2");
             expect(keyInfo.passphrase).toEqual({
@@ -538,9 +558,10 @@ describe("Secrets", function() {
             expect(keyInfo).toHaveProperty("mac");
             expect(alice.checkSecretStorageKey(secretStorageKeys.key_id, keyInfo))
                 .toBeTruthy();
+            alice.stopClient();
         });
         it("fixes backup keys in the wrong format", async function() {
-            let crossSigningKeys = {
+            let crossSigningKeys: Record<string, Uint8Array> = {
                 master: XSK,
                 user_signing: USK,
                 self_signing: SSK,
@@ -552,9 +573,9 @@ describe("Secrets", function() {
                 { userId: "@alice:example.com", deviceId: "Osborne2" },
                 {
                     cryptoCallbacks: {
-                        getCrossSigningKey: t => crossSigningKeys[t],
+                        getCrossSigningKey: async t => crossSigningKeys[t],
                         saveCrossSigningKeys: k => crossSigningKeys = k,
-                        getSecretStorageKey: ({ keys }, name) => {
+                        getSecretStorageKey: async ({ keys }, name) => {
                             for (const keyId of Object.keys(keys)) {
                                 if (secretStorageKeys[keyId]) {
                                     return [keyId, secretStorageKeys[keyId]];
@@ -619,6 +640,8 @@ describe("Secrets", function() {
                 }),
             ]);
             alice.crypto.deviceList.storeCrossSigningForUser("@alice:example.com", {
+                firstUse: false,
+                crossSigningVerifiedBefore: false,
                 keys: {
                     master: {
                         user_id: "@alice:example.com",
@@ -659,15 +682,17 @@ describe("Secrets", function() {
                 });
                 alice.store.storeAccountDataEvents([event]);
                 this.emit("accountData", event);
+                return {};
             };
 
-            await alice.bootstrapSecretStorage();
+            await alice.bootstrapSecretStorage({});
 
             const backupKey = alice.getAccountData("m.megolm_backup.v1")
                 .getContent();
             expect(backupKey.encrypted).toHaveProperty("key_id");
             expect(await alice.getSecret("m.megolm_backup.v1"))
                 .toEqual("ey0GB1kB6jhOWgwiBUMIWg==");
+            alice.stopClient();
         });
     });
 });

spec/unit/crypto/verification/request.spec.js

@@ -40,7 +40,7 @@ describe("verification request integration tests with crypto layer", function()
     });
 
     it("should request and accept a verification", async function() {
-        const [alice, bob] = await makeTestClients(
+        const [[alice, bob], clearTestClientTimeouts] = await makeTestClients(
             [
                 { userId: "@alice:example.com", deviceId: "Osborne2" },
                 { userId: "@bob:example.com", deviceId: "Dynabook" },
@@ -78,5 +78,9 @@ describe("verification request integration tests with crypto layer", function()
 
         // XXX: Private function access (but it's a test, so we're okay)
         aliceVerifier.endTimer();
+
+        alice.stop();
+        bob.stop();
+        clearTestClientTimeouts();
     });
 });

spec/unit/crypto/verification/sas.spec.js

@@ -75,9 +75,10 @@ describe("SAS verification", function() {
         let bobSasEvent;
         let aliceVerifier;
         let bobPromise;
+        let clearTestClientTimeouts;
 
         beforeEach(async () => {
-            [alice, bob] = await makeTestClients(
+            [[alice, bob], clearTestClientTimeouts] = await makeTestClients(
                 [
                     { userId: "@alice:example.com", deviceId: "Osborne2" },
                     { userId: "@bob:example.com", deviceId: "Dynabook" },
@@ -178,6 +179,8 @@ describe("SAS verification", function() {
                 alice.stop(),
                 bob.stop(),
             ]);
+
+            clearTestClientTimeouts();
         });
 
         it("should verify a key", async () => {
@@ -215,7 +218,7 @@ describe("SAS verification", function() {
             ]);
 
             // make sure that it uses the preferred method
-            expect(macMethod).toBe("hkdf-hmac-sha256");
+            expect(macMethod).toBe("org.matrix.msc3783.hkdf-hmac-sha256");
             expect(keyAgreement).toBe("curve25519-hkdf-sha256");
 
             // make sure Alice and Bob verified each other
@@ -227,6 +230,62 @@ describe("SAS verification", function() {
             expect(aliceDevice.isVerified()).toBeTruthy();
         });
 
+        it("should be able to verify using the old base64", async () => {
+            // pretend that Alice can only understand the old (incorrect) base64
+            // encoding, and make sure that she can still verify with Bob
+            let macMethod;
+            const aliceOrigSendToDevice = alice.client.sendToDevice.bind(alice.client);
+            alice.client.sendToDevice = (type, map) => {
+                if (type === "m.key.verification.start") {
+                    // Note: this modifies not only the message that Bob
+                    // receives, but also the copy of the message that Alice
+                    // has, since it is the same object.  If this does not
+                    // happen, the verification will fail due to a hash
+                    // commitment mismatch.
+                    map[bob.client.getUserId()][bob.client.deviceId]
+                        .message_authentication_codes = ['hkdf-hmac-sha256'];
+                }
+                return aliceOrigSendToDevice(type, map);
+            };
+            const bobOrigSendToDevice = bob.client.sendToDevice.bind(bob.client);
+            bob.client.sendToDevice = (type, map) => {
+                if (type === "m.key.verification.accept") {
+                    macMethod = map[alice.client.getUserId()][alice.client.deviceId]
+                        .message_authentication_code;
+                }
+                return bobOrigSendToDevice(type, map);
+            };
+
+            alice.httpBackend.when('POST', '/keys/query').respond(200, {
+                failures: {},
+                device_keys: {
+                    "@bob:example.com": BOB_DEVICES,
+                },
+            });
+            bob.httpBackend.when('POST', '/keys/query').respond(200, {
+                failures: {},
+                device_keys: {
+                    "@alice:example.com": ALICE_DEVICES,
+                },
+            });
+
+            await Promise.all([
+                aliceVerifier.verify(),
+                bobPromise.then((verifier) => verifier.verify()),
+                alice.httpBackend.flush(),
+                bob.httpBackend.flush(),
+            ]);
+
+            expect(macMethod).toBe("hkdf-hmac-sha256");
+
+            const bobDevice
+                  = await alice.client.getStoredDevice("@bob:example.com", "Dynabook");
+            expect(bobDevice.isVerified()).toBeTruthy();
+            const aliceDevice
+                  = await bob.client.getStoredDevice("@alice:example.com", "Osborne2");
+            expect(aliceDevice.isVerified()).toBeTruthy();
+        });
+
         it("should be able to verify using the old MAC", async () => {
             // pretend that Alice can only understand the old (incorrect) MAC,
             // and make sure that she can still verify with Bob
@@ -334,7 +393,7 @@ describe("SAS verification", function() {
     });
 
     it("should send a cancellation message on error", async function() {
-        const [alice, bob] = await makeTestClients(
+        const [[alice, bob], clearTestClientTimeouts] = await makeTestClients(
             [
                 { userId: "@alice:example.com", deviceId: "Osborne2" },
                 { userId: "@bob:example.com", deviceId: "Dynabook" },
@@ -377,6 +436,10 @@ describe("SAS verification", function() {
             .not.toHaveBeenCalled();
         expect(bob.client.setDeviceVerified)
             .not.toHaveBeenCalled();
+
+        alice.stop();
+        bob.stop();
+        clearTestClientTimeouts();
     });
 
     describe("verification in DM", function() {
@@ -386,9 +449,10 @@ describe("SAS verification", function() {
         let bobSasEvent;
         let aliceVerifier;
         let bobPromise;
+        let clearTestClientTimeouts;
 
         beforeEach(async function() {
-            [alice, bob] = await makeTestClients(
+            [[alice, bob], clearTestClientTimeouts] = await makeTestClients(
                 [
                     { userId: "@alice:example.com", deviceId: "Osborne2" },
                     { userId: "@bob:example.com", deviceId: "Dynabook" },
@@ -488,6 +552,8 @@ describe("SAS verification", function() {
                 alice.stop(),
                 bob.stop(),
             ]);
+
+            clearTestClientTimeouts();
         });
 
         it("should verify a key", async function() {

spec/unit/crypto/verification/util.js

@@ -23,6 +23,7 @@ import { logger } from '../../../../src/logger';
 
 export async function makeTestClients(userInfos, options) {
     const clients = [];
+    const timeouts = [];
     const clientMap = {};
     const sendToDevice = function(type, map) {
         // logger.log(this.getUserId(), "sends", type, map);
@@ -66,7 +67,7 @@ export async function makeTestClients(userInfos, options) {
             },
         }));
 
-        setImmediate(() => {
+        const timeout = setTimeout(() => {
             for (const tc of clients) {
                 if (tc.client === this) { // eslint-disable-line @babel/no-invalid-this
                     logger.log("sending remote echo!!");
@@ -77,6 +78,8 @@ export async function makeTestClients(userInfos, options) {
             }
         });
 
+        timeouts.push(timeout);
+
         return Promise.resolve({ event_id: eventId });
     };
 
@@ -103,7 +106,11 @@ export async function makeTestClients(userInfos, options) {
 
     await Promise.all(clients.map((testClient) => testClient.client.initCrypto()));
 
-    return clients;
+    const destroy = () => {
+        timeouts.forEach((t) => clearTimeout(t));
+    };
+
+    return [clients, destroy];
 }
 
 export function setupWebcrypto() {

spec/unit/event-mapper.spec.ts

@@ -44,6 +44,10 @@ describe("eventMapperFor", function() {
         rooms = [];
     });
 
+    afterEach(() => {
+        client.stopClient();
+    });
+
     it("should de-duplicate MatrixEvent instances by means of findEventById on the room object", async () => {
         const roomId = "!room:example.org";
         const room = new Room(roomId, client, userId);

spec/unit/event-timeline-set.spec.ts

@@ -0,0 +1,294 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import * as utils from "../test-utils/test-utils";
+import {
+    EventTimeline,
+    EventTimelineSet,
+    EventType,
+    MatrixClient,
+    MatrixEvent,
+    MatrixEventEvent,
+    Room,
+    DuplicateStrategy,
+} from '../../src';
+import { Thread } from "../../src/models/thread";
+import { ReEmitter } from "../../src/ReEmitter";
+
+describe('EventTimelineSet', () => {
+    const roomId = '!foo:bar';
+    const userA = "@alice:bar";
+
+    let room: Room;
+    let eventTimeline: EventTimeline;
+    let eventTimelineSet: EventTimelineSet;
+    let client: MatrixClient;
+
+    let messageEvent: MatrixEvent;
+    let replyEvent: MatrixEvent;
+
+    const itShouldReturnTheRelatedEvents = () => {
+        it('should return the related events', () => {
+            eventTimelineSet.relations.aggregateChildEvent(messageEvent);
+            const relations = eventTimelineSet.relations.getChildEventsForEvent(
+                messageEvent.getId(),
+                "m.in_reply_to",
+                EventType.RoomMessage,
+            );
+            expect(relations).toBeDefined();
+            expect(relations.getRelations().length).toBe(1);
+            expect(relations.getRelations()[0].getId()).toBe(replyEvent.getId());
+        });
+    };
+
+    beforeEach(() => {
+        client = utils.mock(MatrixClient, 'MatrixClient');
+        client.reEmitter = utils.mock(ReEmitter, 'ReEmitter');
+        room = new Room(roomId, client, userA);
+        eventTimelineSet = new EventTimelineSet(room);
+        eventTimeline = new EventTimeline(eventTimelineSet);
+        messageEvent = utils.mkMessage({
+            room: roomId,
+            user: userA,
+            msg: 'Hi!',
+            event: true,
+        });
+        replyEvent = utils.mkReplyMessage({
+            room: roomId,
+            user: userA,
+            msg: 'Hoo!',
+            event: true,
+            replyToMessage: messageEvent,
+        });
+    });
+
+    describe('addLiveEvent', () => {
+        it("Adds event to the live timeline in the timeline set", () => {
+            const liveTimeline = eventTimelineSet.getLiveTimeline();
+            expect(liveTimeline.getEvents().length).toStrictEqual(0);
+            eventTimelineSet.addLiveEvent(messageEvent);
+            expect(liveTimeline.getEvents().length).toStrictEqual(1);
+        });
+
+        it("should replace a timeline event if dupe strategy is 'replace'", () => {
+            const liveTimeline = eventTimelineSet.getLiveTimeline();
+            expect(liveTimeline.getEvents().length).toStrictEqual(0);
+            eventTimelineSet.addLiveEvent(messageEvent, {
+                duplicateStrategy: DuplicateStrategy.Replace,
+            });
+            expect(liveTimeline.getEvents().length).toStrictEqual(1);
+
+            // make a duplicate
+            const duplicateMessageEvent = utils.mkMessage({
+                room: roomId, user: userA, msg: "dupe", event: true,
+            });
+            duplicateMessageEvent.event.event_id = messageEvent.getId();
+
+            // Adding the duplicate event should replace the `messageEvent`
+            // because it has the same `event_id` and duplicate strategy is
+            // replace.
+            eventTimelineSet.addLiveEvent(duplicateMessageEvent, {
+                duplicateStrategy: DuplicateStrategy.Replace,
+            });
+
+            const eventsInLiveTimeline = liveTimeline.getEvents();
+            expect(eventsInLiveTimeline.length).toStrictEqual(1);
+            expect(eventsInLiveTimeline[0]).toStrictEqual(duplicateMessageEvent);
+        });
+
+        it("Make sure legacy overload passing options directly as parameters still works", () => {
+            expect(() => eventTimelineSet.addLiveEvent(messageEvent, DuplicateStrategy.Replace, false)).not.toThrow();
+            expect(() => eventTimelineSet.addLiveEvent(messageEvent, DuplicateStrategy.Ignore, true)).not.toThrow();
+        });
+    });
+
+    describe('addEventToTimeline', () => {
+        it("Adds event to timeline", () => {
+            const liveTimeline = eventTimelineSet.getLiveTimeline();
+            expect(liveTimeline.getEvents().length).toStrictEqual(0);
+            eventTimelineSet.addEventToTimeline(messageEvent, liveTimeline, {
+                toStartOfTimeline: true,
+            });
+            expect(liveTimeline.getEvents().length).toStrictEqual(1);
+        });
+
+        it("Make sure legacy overload passing options directly as parameters still works", () => {
+            const liveTimeline = eventTimelineSet.getLiveTimeline();
+            expect(() => {
+                eventTimelineSet.addEventToTimeline(
+                    messageEvent,
+                    liveTimeline,
+                    true,
+                );
+            }).not.toThrow();
+            expect(() => {
+                eventTimelineSet.addEventToTimeline(
+                    messageEvent,
+                    liveTimeline,
+                    true,
+                    false,
+                );
+            }).not.toThrow();
+        });
+    });
+
+    describe('aggregateRelations', () => {
+        describe('with unencrypted events', () => {
+            beforeEach(() => {
+                eventTimelineSet.addEventsToTimeline(
+                    [
+                        messageEvent,
+                        replyEvent,
+                    ],
+                    true,
+                    eventTimeline,
+                    'foo',
+                );
+            });
+
+            itShouldReturnTheRelatedEvents();
+        });
+
+        describe('with events to be decrypted', () => {
+            let messageEventShouldAttemptDecryptionSpy: jest.SpyInstance;
+            let messageEventIsDecryptionFailureSpy: jest.SpyInstance;
+
+            let replyEventShouldAttemptDecryptionSpy: jest.SpyInstance;
+            let replyEventIsDecryptionFailureSpy: jest.SpyInstance;
+
+            beforeEach(() => {
+                messageEventShouldAttemptDecryptionSpy = jest.spyOn(messageEvent, 'shouldAttemptDecryption');
+                messageEventShouldAttemptDecryptionSpy.mockReturnValue(true);
+                messageEventIsDecryptionFailureSpy = jest.spyOn(messageEvent, 'isDecryptionFailure');
+
+                replyEventShouldAttemptDecryptionSpy = jest.spyOn(replyEvent, 'shouldAttemptDecryption');
+                replyEventShouldAttemptDecryptionSpy.mockReturnValue(true);
+                replyEventIsDecryptionFailureSpy = jest.spyOn(messageEvent, 'isDecryptionFailure');
+
+                eventTimelineSet.addEventsToTimeline(
+                    [
+                        messageEvent,
+                        replyEvent,
+                    ],
+                    true,
+                    eventTimeline,
+                    'foo',
+                );
+            });
+
+            it('should not return the related events', () => {
+                eventTimelineSet.relations.aggregateChildEvent(messageEvent);
+                const relations = eventTimelineSet.relations.getChildEventsForEvent(
+                    messageEvent.getId(),
+                    "m.in_reply_to",
+                    EventType.RoomMessage,
+                );
+                expect(relations).toBeUndefined();
+            });
+
+            describe('after decryption', () => {
+                beforeEach(() => {
+                    // simulate decryption failure once
+                    messageEventIsDecryptionFailureSpy.mockReturnValue(true);
+                    replyEventIsDecryptionFailureSpy.mockReturnValue(true);
+
+                    messageEvent.emit(MatrixEventEvent.Decrypted, messageEvent);
+                    replyEvent.emit(MatrixEventEvent.Decrypted, replyEvent);
+
+                    // simulate decryption
+                    messageEventIsDecryptionFailureSpy.mockReturnValue(false);
+                    replyEventIsDecryptionFailureSpy.mockReturnValue(false);
+
+                    messageEventShouldAttemptDecryptionSpy.mockReturnValue(false);
+                    replyEventShouldAttemptDecryptionSpy.mockReturnValue(false);
+
+                    messageEvent.emit(MatrixEventEvent.Decrypted, messageEvent);
+                    replyEvent.emit(MatrixEventEvent.Decrypted, replyEvent);
+                });
+
+                itShouldReturnTheRelatedEvents();
+            });
+        });
+    });
+
+    describe("canContain", () => {
+        const mkThreadResponse = (root: MatrixEvent) => utils.mkEvent({
+            event: true,
+            type: EventType.RoomMessage,
+            user: userA,
+            room: roomId,
+            content: {
+                "body": "Thread response :: " + Math.random(),
+                "m.relates_to": {
+                    "event_id": root.getId(),
+                    "m.in_reply_to": {
+                        "event_id": root.getId(),
+                    },
+                    "rel_type": "m.thread",
+                },
+            },
+        }, room.client);
+
+        let thread: Thread;
+
+        beforeEach(() => {
+            (client.supportsExperimentalThreads as jest.Mock).mockReturnValue(true);
+            thread = new Thread("!thread_id:server", messageEvent, { room, client });
+        });
+
+        it("should throw if timeline set has no room", () => {
+            const eventTimelineSet = new EventTimelineSet(undefined, {}, client);
+            expect(() => eventTimelineSet.canContain(messageEvent)).toThrowError();
+        });
+
+        it("should return false if timeline set is for thread but event is not threaded", () => {
+            const eventTimelineSet = new EventTimelineSet(room, {}, client, thread);
+            expect(eventTimelineSet.canContain(replyEvent)).toBeFalsy();
+        });
+
+        it("should return false if timeline set it for thread but event it for a different thread", () => {
+            const eventTimelineSet = new EventTimelineSet(room, {}, client, thread);
+            const event = mkThreadResponse(replyEvent);
+            expect(eventTimelineSet.canContain(event)).toBeFalsy();
+        });
+
+        it("should return false if timeline set is not for a thread but event is a thread response", () => {
+            const eventTimelineSet = new EventTimelineSet(room, {}, client);
+            const event = mkThreadResponse(replyEvent);
+            expect(eventTimelineSet.canContain(event)).toBeFalsy();
+        });
+
+        it("should return true if the timeline set is not for a thread and the event is a thread root", () => {
+            const eventTimelineSet = new EventTimelineSet(room, {}, client);
+            expect(eventTimelineSet.canContain(messageEvent)).toBeTruthy();
+        });
+
+        it("should return true if the timeline set is for a thread and the event is its thread root", () => {
+            const thread = new Thread(messageEvent.getId(), messageEvent, { room, client });
+            const eventTimelineSet = new EventTimelineSet(room, {}, client, thread);
+            messageEvent.setThread(thread);
+            expect(eventTimelineSet.canContain(messageEvent)).toBeTruthy();
+        });
+
+        it("should return true if the timeline set is for a thread and the event is a response to it", () => {
+            const thread = new Thread(messageEvent.getId(), messageEvent, { room, client });
+            const eventTimelineSet = new EventTimelineSet(room, {}, client, thread);
+            messageEvent.setThread(thread);
+            const event = mkThreadResponse(messageEvent);
+            expect(eventTimelineSet.canContain(event)).toBeTruthy();
+        });
+    });
+});

spec/unit/event-timeline.spec.js

@@ -50,9 +50,11 @@ describe("EventTimeline", function() {
             timeline.initialiseState(events);
             expect(timeline.startState.setStateEvents).toHaveBeenCalledWith(
                 events,
+                { timelineWasEmpty: undefined },
             );
             expect(timeline.endState.setStateEvents).toHaveBeenCalledWith(
                 events,
+                { timelineWasEmpty: undefined },
             );
         });
 
@@ -73,7 +75,7 @@ describe("EventTimeline", function() {
             expect(function() {
                 timeline.initialiseState(state);
             }).not.toThrow();
-            timeline.addEvent(event, false);
+            timeline.addEvent(event, { toStartOfTimeline: false });
             expect(function() {
                 timeline.initialiseState(state);
             }).toThrow();
@@ -149,9 +151,9 @@ describe("EventTimeline", function() {
         ];
 
         it("should be able to add events to the end", function() {
-            timeline.addEvent(events[0], false);
+            timeline.addEvent(events[0], { toStartOfTimeline: false });
             const initialIndex = timeline.getBaseIndex();
-            timeline.addEvent(events[1], false);
+            timeline.addEvent(events[1], { toStartOfTimeline: false });
             expect(timeline.getBaseIndex()).toEqual(initialIndex);
             expect(timeline.getEvents().length).toEqual(2);
             expect(timeline.getEvents()[0]).toEqual(events[0]);
@@ -159,9 +161,9 @@ describe("EventTimeline", function() {
         });
 
         it("should be able to add events to the start", function() {
-            timeline.addEvent(events[0], true);
+            timeline.addEvent(events[0], { toStartOfTimeline: true });
             const initialIndex = timeline.getBaseIndex();
-            timeline.addEvent(events[1], true);
+            timeline.addEvent(events[1], { toStartOfTimeline: true });
             expect(timeline.getBaseIndex()).toEqual(initialIndex + 1);
             expect(timeline.getEvents().length).toEqual(2);
             expect(timeline.getEvents()[0]).toEqual(events[1]);
@@ -203,9 +205,9 @@ describe("EventTimeline", function() {
                 content: { name: "Old Room Name" },
             });
 
-            timeline.addEvent(newEv, false);
+            timeline.addEvent(newEv, { toStartOfTimeline: false });
             expect(newEv.sender).toEqual(sentinel);
-            timeline.addEvent(oldEv, true);
+            timeline.addEvent(oldEv, { toStartOfTimeline: true });
             expect(oldEv.sender).toEqual(oldSentinel);
         });
 
@@ -242,9 +244,9 @@ describe("EventTimeline", function() {
             const oldEv = utils.mkMembership({
                 room: roomId, mship: "ban", user: userB, skey: userA, event: true,
             });
-            timeline.addEvent(newEv, false);
+            timeline.addEvent(newEv, { toStartOfTimeline: false });
             expect(newEv.target).toEqual(sentinel);
-            timeline.addEvent(oldEv, true);
+            timeline.addEvent(oldEv, { toStartOfTimeline: true });
             expect(oldEv.target).toEqual(oldSentinel);
         });
 
@@ -262,13 +264,13 @@ describe("EventTimeline", function() {
                 }),
             ];
 
-            timeline.addEvent(events[0], false);
-            timeline.addEvent(events[1], false);
+            timeline.addEvent(events[0], { toStartOfTimeline: false });
+            timeline.addEvent(events[1], { toStartOfTimeline: false });
 
             expect(timeline.getState(EventTimeline.FORWARDS).setStateEvents).
-                toHaveBeenCalledWith([events[0]]);
+                toHaveBeenCalledWith([events[0]], { timelineWasEmpty: undefined });
             expect(timeline.getState(EventTimeline.FORWARDS).setStateEvents).
-                toHaveBeenCalledWith([events[1]]);
+                toHaveBeenCalledWith([events[1]], { timelineWasEmpty: undefined });
 
             expect(events[0].forwardLooking).toBe(true);
             expect(events[1].forwardLooking).toBe(true);
@@ -291,13 +293,13 @@ describe("EventTimeline", function() {
                 }),
             ];
 
-            timeline.addEvent(events[0], true);
-            timeline.addEvent(events[1], true);
+            timeline.addEvent(events[0], { toStartOfTimeline: true });
+            timeline.addEvent(events[1], { toStartOfTimeline: true });
 
             expect(timeline.getState(EventTimeline.BACKWARDS).setStateEvents).
-                toHaveBeenCalledWith([events[0]]);
+                toHaveBeenCalledWith([events[0]], { timelineWasEmpty: undefined });
             expect(timeline.getState(EventTimeline.BACKWARDS).setStateEvents).
-                toHaveBeenCalledWith([events[1]]);
+                toHaveBeenCalledWith([events[1]], { timelineWasEmpty: undefined });
 
             expect(events[0].forwardLooking).toBe(false);
             expect(events[1].forwardLooking).toBe(false);
@@ -305,6 +307,11 @@ describe("EventTimeline", function() {
             expect(timeline.getState(EventTimeline.FORWARDS).setStateEvents).
                 not.toHaveBeenCalled();
         });
+
+        it("Make sure legacy overload passing options directly as parameters still works", () => {
+            expect(() => timeline.addEvent(events[0], { toStartOfTimeline: true })).not.toThrow();
+            expect(() => timeline.addEvent(events[0], { stateContext: new RoomState() })).not.toThrow();
+        });
     });
 
     describe("removeEvent", function() {
@@ -324,8 +331,8 @@ describe("EventTimeline", function() {
         ];
 
         it("should remove events", function() {
-            timeline.addEvent(events[0], false);
-            timeline.addEvent(events[1], false);
+            timeline.addEvent(events[0], { toStartOfTimeline: false });
+            timeline.addEvent(events[1], { toStartOfTimeline: false });
             expect(timeline.getEvents().length).toEqual(2);
 
             let ev = timeline.removeEvent(events[0].getId());
@@ -338,9 +345,9 @@ describe("EventTimeline", function() {
         });
 
         it("should update baseIndex", function() {
-            timeline.addEvent(events[0], false);
-            timeline.addEvent(events[1], true);
-            timeline.addEvent(events[2], false);
+            timeline.addEvent(events[0], { toStartOfTimeline: false });
+            timeline.addEvent(events[1], { toStartOfTimeline: true });
+            timeline.addEvent(events[2], { toStartOfTimeline: false });
             expect(timeline.getEvents().length).toEqual(3);
             expect(timeline.getBaseIndex()).toEqual(1);
 
@@ -358,11 +365,11 @@ describe("EventTimeline", function() {
         // further addEvent(ev, false) calls made the index increase.
         it("should not make baseIndex assplode when removing the last event",
            function() {
-               timeline.addEvent(events[0], true);
+               timeline.addEvent(events[0], { toStartOfTimeline: true });
                timeline.removeEvent(events[0].getId());
                const initialIndex = timeline.getBaseIndex();
-               timeline.addEvent(events[1], false);
-               timeline.addEvent(events[2], false);
+               timeline.addEvent(events[1], { toStartOfTimeline: false });
+               timeline.addEvent(events[2], { toStartOfTimeline: false });
                expect(timeline.getBaseIndex()).toEqual(initialIndex);
                expect(timeline.getEvents().length).toEqual(2);
            });

spec/unit/filter-component.spec.ts

@@ -1,7 +1,4 @@
-import {
-    MatrixEvent,
-    RelationType,
-} from "../../src";
+import { RelationType } from "../../src";
 import { FilterComponent } from "../../src/filter-component";
 import { mkEvent } from '../test-utils/test-utils';
 
@@ -14,7 +11,7 @@ describe("Filter Component", function() {
                 content: { },
                 room: 'roomId',
                 event: true,
-            }) as MatrixEvent;
+            });
 
             const checkResult = filter.check(event);
 
@@ -28,7 +25,7 @@ describe("Filter Component", function() {
                 content: { },
                 room: 'roomId',
                 event: true,
-            }) as MatrixEvent;
+            });
 
             const checkResult = filter.check(event);
 
@@ -55,7 +52,7 @@ describe("Filter Component", function() {
                         },
                     },
                 },
-            }) as MatrixEvent;
+            });
 
             expect(filter.check(threadRootNotParticipated)).toBe(false);
         });
@@ -80,7 +77,7 @@ describe("Filter Component", function() {
                 user: '@someone-else:server.org',
                 room: 'roomId',
                 event: true,
-            }) as MatrixEvent;
+            });
 
             expect(filter.check(threadRootParticipated)).toBe(true);
         });
@@ -100,7 +97,7 @@ describe("Filter Component", function() {
                         [RelationType.Reference]: {},
                     },
                 },
-            }) as MatrixEvent;
+            });
 
             expect(filter.check(referenceRelationEvent)).toBe(false);
         });
@@ -123,7 +120,7 @@ describe("Filter Component", function() {
                 },
                 room: 'roomId',
                 event: true,
-            }) as MatrixEvent;
+            });
 
             const eventWithMultipleRelations = mkEvent({
                 "type": "m.room.message",
@@ -148,7 +145,7 @@ describe("Filter Component", function() {
                 },
                 "room": 'roomId',
                 "event": true,
-            }) as MatrixEvent;
+            });
 
             const noMatchEvent = mkEvent({
                 "type": "m.room.message",
@@ -160,7 +157,7 @@ describe("Filter Component", function() {
                 },
                 "room": 'roomId',
                 "event": true,
-            }) as MatrixEvent;
+            });
 
             expect(filter.check(threadRootEvent)).toBe(true);
             expect(filter.check(eventWithMultipleRelations)).toBe(true);

spec/unit/matrix-client.spec.ts

@@ -87,7 +87,7 @@ describe("MatrixClient", function() {
         // }
         // items are popped off when processed and block if no items left.
     ];
-    let acceptKeepalives;
+    let acceptKeepalives: boolean;
     let pendingLookup = null;
     function httpReq(cb, method, path, qp, data, prefix) {
         if (path === KEEP_ALIVE_PATH && acceptKeepalives) {
@@ -118,6 +118,7 @@ describe("MatrixClient", function() {
                 method: method,
                 path: path,
             };
+            pendingLookup.promise.abort = () => {}; // to make it a valid IAbortablePromise
             return pendingLookup.promise;
         }
         if (next.path === path && next.method === method) {
@@ -126,7 +127,7 @@ describe("MatrixClient", function() {
                 (next.error ? "BAD" : "GOOD") + " response",
             );
             if (next.expectBody) {
-                expect(next.expectBody).toEqual(data);
+                expect(data).toEqual(next.expectBody);
             }
             if (next.expectQueryParams) {
                 Object.keys(next.expectQueryParams).forEach(function(k) {
@@ -150,6 +151,10 @@ describe("MatrixClient", function() {
             }
             return Promise.resolve(next.data);
         }
+        // Jest doesn't let us have custom expectation errors, so if you're seeing this then
+        // you forgot to handle at least 1 pending request. Check your tests to ensure your
+        // number of expectations lines up with your number of requests made, and that those
+        // requests match your expectations.
         expect(true).toBe(false);
         return new Promise(() => {});
     }
@@ -205,6 +210,7 @@ describe("MatrixClient", function() {
         client.http.authedRequest.mockImplementation(function() {
             return new Promise(() => {});
         });
+        client.stopClient();
     });
 
     it("should create (unstable) file trees", async () => {
@@ -725,18 +731,16 @@ describe("MatrixClient", function() {
     });
 
     describe("guest rooms", function() {
-        it("should only do /sync calls (without filter/pushrules)", function(done) {
-            httpLookups = []; // no /pushrules or /filterw
+        it("should only do /sync calls (without filter/pushrules)", async function() {
+            httpLookups = []; // no /pushrules or /filter
             httpLookups.push({
                 method: "GET",
                 path: "/sync",
                 data: SYNC_DATA,
-                thenCall: function() {
-                    done();
-                },
             });
             client.setGuest(true);
-            client.startClient();
+            await client.startClient();
+            expect(httpLookups.length).toBe(0);
         });
 
         xit("should be able to peek into a room using peekInRoom", function(done) {
@@ -773,7 +777,7 @@ describe("MatrixClient", function() {
                 expectBody: content,
             }];
 
-            await client.sendEvent(roomId, EventType.RoomMessage, content, txnId);
+            await client.sendEvent(roomId, EventType.RoomMessage, { ...content }, txnId);
         });
 
         it("overload with null threadId works", async () => {
@@ -786,20 +790,99 @@ describe("MatrixClient", function() {
                 expectBody: content,
             }];
 
-            await client.sendEvent(roomId, null, EventType.RoomMessage, content, txnId);
+            await client.sendEvent(roomId, null, EventType.RoomMessage, { ...content }, txnId);
         });
 
         it("overload with threadId works", async () => {
             const eventId = "$eventId:example.org";
             const txnId = client.makeTxnId();
+            const threadId = "$threadId:server";
             httpLookups = [{
                 method: "PUT",
                 path: `/rooms/${encodeURIComponent(roomId)}/send/m.room.message/${txnId}`,
                 data: { event_id: eventId },
-                expectBody: content,
+                expectBody: {
+                    ...content,
+                    "m.relates_to": {
+                        "event_id": threadId,
+                        "is_falling_back": true,
+                        "rel_type": "m.thread",
+                    },
+                },
             }];
 
-            await client.sendEvent(roomId, "$threadId:server", EventType.RoomMessage, content, txnId);
+            await client.sendEvent(roomId, threadId, EventType.RoomMessage, { ...content }, txnId);
+        });
+
+        it("should add thread relation if threadId is passed and the relation is missing", async () => {
+            const eventId = "$eventId:example.org";
+            const threadId = "$threadId:server";
+            const txnId = client.makeTxnId();
+
+            const room = new Room(roomId, client, userId);
+            store.getRoom.mockReturnValue(room);
+
+            const rootEvent = new MatrixEvent({ event_id: threadId });
+            room.createThread(threadId, rootEvent, [rootEvent], false);
+
+            httpLookups = [{
+                method: "PUT",
+                path: `/rooms/${encodeURIComponent(roomId)}/send/m.room.message/${txnId}`,
+                data: { event_id: eventId },
+                expectBody: {
+                    ...content,
+                    "m.relates_to": {
+                        "m.in_reply_to": {
+                            event_id: threadId,
+                        },
+                        "event_id": threadId,
+                        "is_falling_back": true,
+                        "rel_type": "m.thread",
+                    },
+                },
+            }];
+
+            await client.sendEvent(roomId, threadId, EventType.RoomMessage, { ...content }, txnId);
+        });
+
+        it("should add thread relation if threadId is passed and the relation is missing with reply", async () => {
+            const eventId = "$eventId:example.org";
+            const threadId = "$threadId:server";
+            const txnId = client.makeTxnId();
+
+            const content = {
+                body,
+                "m.relates_to": {
+                    "m.in_reply_to": {
+                        event_id: "$other:event",
+                    },
+                },
+            };
+
+            const room = new Room(roomId, client, userId);
+            store.getRoom.mockReturnValue(room);
+
+            const rootEvent = new MatrixEvent({ event_id: threadId });
+            room.createThread(threadId, rootEvent, [rootEvent], false);
+
+            httpLookups = [{
+                method: "PUT",
+                path: `/rooms/${encodeURIComponent(roomId)}/send/m.room.message/${txnId}`,
+                data: { event_id: eventId },
+                expectBody: {
+                    ...content,
+                    "m.relates_to": {
+                        "m.in_reply_to": {
+                            event_id: "$other:event",
+                        },
+                        "event_id": threadId,
+                        "is_falling_back": false,
+                        "rel_type": "m.thread",
+                    },
+                },
+            }];
+
+            await client.sendEvent(roomId, threadId, EventType.RoomMessage, { ...content }, txnId);
         });
     });
 
@@ -926,6 +1009,7 @@ describe("MatrixClient", function() {
             };
             client.crypto = { // mock crypto
                 encryptEvent: (event, room) => new Promise(() => {}),
+                stop: jest.fn(),
             };
         });
 
@@ -1191,4 +1275,26 @@ describe("MatrixClient", function() {
             passwordTest({ auth, new_password: newPassword, logout_devices: false }, callback);
         });
     });
+
+    describe("getLocalAliases", () => {
+        it("should call the right endpoint", async () => {
+            const response = {
+                aliases: ["#woop:example.org", "#another:example.org"],
+            };
+            client.http.authedRequest.mockClear().mockResolvedValue(response);
+
+            const roomId = "!whatever:example.org";
+            const result = await client.getLocalAliases(roomId);
+
+            // Current version of the endpoint we support is v3
+            const [callback, method, path, queryParams, data, opts] = client.http.authedRequest.mock.calls[0];
+            expect(callback).toBeFalsy();
+            expect(data).toBeFalsy();
+            expect(method).toBe('GET');
+            expect(path).toEqual(`/rooms/${encodeURIComponent(roomId)}/aliases`);
+            expect(opts).toMatchObject({ prefix: "/_matrix/client/v3" });
+            expect(queryParams).toBeFalsy();
+            expect(result!.aliases).toEqual(response.aliases);
+        });
+    });
 });

spec/unit/models/beacon.spec.ts

@@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { MatrixEvent } from "../../../src";
 import {
     isTimestampInDuration,
     Beacon,
@@ -65,33 +66,36 @@ describe('Beacon', () => {
         // beacon_info events
         // created 'an hour ago'
         // without timeout of 3 hours
-        let liveBeaconEvent;
-        let notLiveBeaconEvent;
-        let user2BeaconEvent;
+        let liveBeaconEvent: MatrixEvent;
+        let notLiveBeaconEvent: MatrixEvent;
+        let user2BeaconEvent: MatrixEvent;
 
         const advanceDateAndTime = (ms: number) => {
             // bc liveness check uses Date.now we have to advance this mock
-            jest.spyOn(global.Date, 'now').mockReturnValue(now + ms);
+            jest.spyOn(global.Date, 'now').mockReturnValue(Date.now() + ms);
             // then advance time for the interval by the same amount
             jest.advanceTimersByTime(ms);
         };
 
         beforeEach(() => {
-            // go back in time to create the beacon
-            jest.spyOn(global.Date, 'now').mockReturnValue(now - HOUR_MS);
             liveBeaconEvent = makeBeaconInfoEvent(
                 userId,
                 roomId,
                 {
                     timeout: HOUR_MS * 3,
                     isLive: true,
+                    timestamp: now - HOUR_MS,
                 },
                 '$live123',
             );
             notLiveBeaconEvent = makeBeaconInfoEvent(
                 userId,
                 roomId,
-                { timeout: HOUR_MS * 3, isLive: false },
+                {
+                    timeout: HOUR_MS * 3,
+                    isLive: false,
+                    timestamp: now - HOUR_MS,
+                },
                 '$dead123',
             );
             user2BeaconEvent = makeBeaconInfoEvent(
@@ -100,11 +104,12 @@ describe('Beacon', () => {
                 {
                     timeout: HOUR_MS * 3,
                     isLive: true,
+                    timestamp: now - HOUR_MS,
                 },
                 '$user2live123',
             );
 
-            // back to now
+            // back to 'now'
             jest.spyOn(global.Date, 'now').mockReturnValue(now);
         });
 
@@ -131,17 +136,81 @@ describe('Beacon', () => {
             });
 
             it('returns false when beacon is expired', () => {
-                // time travel to beacon creation + 3 hours
-                jest.spyOn(global.Date, 'now').mockReturnValue(now - 3 * HOUR_MS);
-                const beacon = new Beacon(liveBeaconEvent);
+                const expiredBeaconEvent = makeBeaconInfoEvent(
+                    userId2,
+                    roomId,
+                    {
+                        timeout: HOUR_MS,
+                        isLive: true,
+                        timestamp: now - HOUR_MS * 2,
+                    },
+                    '$user2live123',
+                );
+                const beacon = new Beacon(expiredBeaconEvent);
                 expect(beacon.isLive).toEqual(false);
             });
 
-            it('returns false when beacon timestamp is in future', () => {
-                // time travel to before beacon events timestamp
-                // event was created now - 1 hour
-                jest.spyOn(global.Date, 'now').mockReturnValue(now - HOUR_MS - HOUR_MS);
-                const beacon = new Beacon(liveBeaconEvent);
+            it('returns false when beacon timestamp is in future by an hour', () => {
+                const beaconStartsInHour = makeBeaconInfoEvent(
+                    userId2,
+                    roomId,
+                    {
+                        timeout: HOUR_MS,
+                        isLive: true,
+                        timestamp: now + HOUR_MS,
+                    },
+                    '$user2live123',
+                );
+                const beacon = new Beacon(beaconStartsInHour);
+                expect(beacon.isLive).toEqual(false);
+            });
+
+            it('returns true when beacon timestamp is one minute in the future', () => {
+                const beaconStartsInOneMin = makeBeaconInfoEvent(
+                    userId2,
+                    roomId,
+                    {
+                        timeout: HOUR_MS,
+                        isLive: true,
+                        timestamp: now + 60000,
+                    },
+                    '$user2live123',
+                );
+                const beacon = new Beacon(beaconStartsInOneMin);
+                expect(beacon.isLive).toEqual(true);
+            });
+
+            it('returns true when beacon timestamp is one minute before expiry', () => {
+                // this test case is to check the start time leniency doesn't affect
+                // strict expiry time checks
+                const expiresInOneMin = makeBeaconInfoEvent(
+                    userId2,
+                    roomId,
+                    {
+                        timeout: HOUR_MS,
+                        isLive: true,
+                        timestamp: now - HOUR_MS + 60000,
+                    },
+                    '$user2live123',
+                );
+                const beacon = new Beacon(expiresInOneMin);
+                expect(beacon.isLive).toEqual(true);
+            });
+
+            it('returns false when beacon timestamp is one minute after expiry', () => {
+                // this test case is to check the start time leniency doesn't affect
+                // strict expiry time checks
+                const expiredOneMinAgo = makeBeaconInfoEvent(
+                    userId2,
+                    roomId,
+                    {
+                        timeout: HOUR_MS,
+                        isLive: true,
+                        timestamp: now - HOUR_MS - 60000,
+                    },
+                    '$user2live123',
+                );
+                const beacon = new Beacon(expiredOneMinAgo);
                 expect(beacon.isLive).toEqual(false);
             });
 
@@ -224,13 +293,47 @@ describe('Beacon', () => {
                 beacon.monitorLiveness();
 
                 // @ts-ignore
-                expect(beacon.livenessWatchInterval).toBeFalsy();
+                expect(beacon.livenessWatchTimeout).toBeFalsy();
                 advanceDateAndTime(HOUR_MS * 2 + 1);
 
                 // no emit
                 expect(emitSpy).not.toHaveBeenCalled();
             });
 
+            it('checks liveness of beacon at expected start time', () => {
+                const futureBeaconEvent = makeBeaconInfoEvent(
+                    userId,
+                    roomId,
+                    {
+                        timeout: HOUR_MS * 3,
+                        isLive: true,
+                        // start timestamp hour in future
+                        timestamp: now + HOUR_MS,
+                    },
+                    '$live123',
+                );
+
+                const beacon = new Beacon(futureBeaconEvent);
+                expect(beacon.isLive).toBeFalsy();
+                const emitSpy = jest.spyOn(beacon, 'emit');
+
+                beacon.monitorLiveness();
+
+                // advance to the start timestamp of the beacon
+                advanceDateAndTime(HOUR_MS + 1);
+
+                // beacon is in live period now
+                expect(emitSpy).toHaveBeenCalledTimes(1);
+                expect(emitSpy).toHaveBeenCalledWith(BeaconEvent.LivenessChange, true, beacon);
+
+                // check the expiry monitor is still setup ok
+                // advance to the expiry
+                advanceDateAndTime(HOUR_MS * 3 + 100);
+
+                expect(emitSpy).toHaveBeenCalledTimes(2);
+                expect(emitSpy).toHaveBeenCalledWith(BeaconEvent.LivenessChange, false, beacon);
+            });
+
             it('checks liveness of beacon at expected expiry time', () => {
                 // live beacon was created an hour ago
                 // and has a 3hr duration
@@ -253,12 +356,12 @@ describe('Beacon', () => {
 
                 beacon.monitorLiveness();
                 // @ts-ignore
-                const oldMonitor = beacon.livenessWatchInterval;
+                const oldMonitor = beacon.livenessWatchTimeout;
 
                 beacon.monitorLiveness();
 
                 // @ts-ignore
-                expect(beacon.livenessWatchInterval).not.toEqual(oldMonitor);
+                expect(beacon.livenessWatchTimeout).not.toEqual(oldMonitor);
             });
 
             it('destroy kills liveness monitor and emits', () => {
@@ -309,6 +412,57 @@ describe('Beacon', () => {
                 expect(emitSpy).not.toHaveBeenCalled();
             });
 
+            describe('when beacon is live with a start timestamp is in the future', () => {
+                it('ignores locations before the beacon start timestamp', () => {
+                    const startTimestamp = now + 60000;
+                    const beacon = new Beacon(makeBeaconInfoEvent(
+                        userId,
+                        roomId,
+                        { isLive: true, timeout: 60000, timestamp: startTimestamp },
+                    ));
+                    const emitSpy = jest.spyOn(beacon, 'emit');
+
+                    beacon.addLocations([
+                        // beacon has now + 60000 live period
+                        makeBeaconEvent(
+                            userId,
+                            {
+                                beaconInfoId: beacon.beaconInfoId,
+                                // now < location timestamp < beacon timestamp
+                                timestamp: now + 10,
+                            },
+                        ),
+                    ]);
+
+                    expect(beacon.latestLocationState).toBeFalsy();
+                    expect(emitSpy).not.toHaveBeenCalled();
+                });
+                it('sets latest location when location timestamp is after startTimestamp', () => {
+                    const startTimestamp = now + 60000;
+                    const beacon = new Beacon(makeBeaconInfoEvent(
+                        userId,
+                        roomId,
+                        { isLive: true, timeout: 600000, timestamp: startTimestamp },
+                    ));
+                    const emitSpy = jest.spyOn(beacon, 'emit');
+
+                    beacon.addLocations([
+                        // beacon has now + 600000 live period
+                        makeBeaconEvent(
+                            userId,
+                            {
+                                beaconInfoId: beacon.beaconInfoId,
+                                // now < beacon timestamp < location timestamp
+                                timestamp: startTimestamp + 10,
+                            },
+                        ),
+                    ]);
+
+                    expect(beacon.latestLocationState).toBeTruthy();
+                    expect(emitSpy).toHaveBeenCalled();
+                });
+            });
+
             it('sets latest location state to most recent location', () => {
                 const beacon = new Beacon(makeBeaconInfoEvent(userId, roomId, { isLive: true, timeout: 60000 }));
                 const emitSpy = jest.spyOn(beacon, 'emit');
@@ -338,6 +492,7 @@ describe('Beacon', () => {
 
                 // the newest valid location
                 expect(beacon.latestLocationState).toEqual(expectedLatestLocation);
+                expect(beacon.latestLocationEvent).toEqual(locations[1]);
                 expect(emitSpy).toHaveBeenCalledWith(BeaconEvent.LocationUpdate, expectedLatestLocation);
             });
 
@@ -356,6 +511,7 @@ describe('Beacon', () => {
                 expect(beacon.latestLocationState).toEqual(expect.objectContaining({
                     uri: 'geo:bar',
                 }));
+                expect(beacon.latestLocationEvent).toEqual(newerLocation);
 
                 const emitSpy = jest.spyOn(beacon, 'emit').mockClear();
 

spec/unit/pushprocessor.spec.js

@@ -302,6 +302,7 @@ describe('NotificationService', function() {
             type: EventType.RoomServerAcl,
             room: testRoomId,
             user: "@alfred:localhost",
+            skey: "",
             event: true,
             content: {},
         });

spec/unit/realtime-callbacks.spec.js

@@ -1,7 +1,7 @@
 import * as callbacks from "../../src/realtime-callbacks";
 
 let wallTime = 1234567890;
-jest.useFakeTimers();
+jest.useFakeTimers().setSystemTime(wallTime);
 
 describe("realtime-callbacks", function() {
     function tick(millis) {
@@ -9,14 +9,6 @@ describe("realtime-callbacks", function() {
         jest.advanceTimersByTime(millis);
     }
 
-    beforeEach(function() {
-        callbacks.setNow(() => wallTime);
-    });
-
-    afterEach(function() {
-        callbacks.setNow();
-    });
-
     describe("setTimeout", function() {
         it("should call the callback after the timeout", function() {
             const callback = jest.fn();

spec/unit/relations.spec.ts

@@ -96,19 +96,14 @@ describe("Relations", function() {
             },
         });
 
-        // Stub the room
-
-        const room = new Room("room123", null, null);
-
         // Add the target event first, then the relation event
         {
+            const room = new Room("room123", null, null);
             const relationsCreated = new Promise(resolve => {
                 targetEvent.once(MatrixEventEvent.RelationsCreated, resolve);
             });
 
-            const timelineSet = new EventTimelineSet(room, {
-                unstableClientRelationAggregation: true,
-            });
+            const timelineSet = new EventTimelineSet(room);
             timelineSet.addLiveEvent(targetEvent);
             timelineSet.addLiveEvent(relationEvent);
 
@@ -117,13 +112,12 @@ describe("Relations", function() {
 
         // Add the relation event first, then the target event
         {
+            const room = new Room("room123", null, null);
             const relationsCreated = new Promise(resolve => {
                 targetEvent.once(MatrixEventEvent.RelationsCreated, resolve);
             });
 
-            const timelineSet = new EventTimelineSet(room, {
-                unstableClientRelationAggregation: true,
-            });
+            const timelineSet = new EventTimelineSet(room);
             timelineSet.addLiveEvent(relationEvent);
             timelineSet.addLiveEvent(targetEvent);
 
@@ -131,6 +125,14 @@ describe("Relations", function() {
         }
     });
 
+    it("should re-use Relations between all timeline sets in a room", async () => {
+        const room = new Room("room123", null, null);
+        const timelineSet1 = new EventTimelineSet(room);
+        const timelineSet2 = new EventTimelineSet(room);
+        expect(room.relations).toBe(timelineSet1.relations);
+        expect(room.relations).toBe(timelineSet2.relations);
+    });
+
     it("should ignore m.replace for state events", async () => {
         const userId = "@bob:example.com";
         const room = new Room("room123", null, userId);
@@ -168,6 +170,8 @@ describe("Relations", function() {
         await relations.setTargetEvent(originalTopic);
         expect(originalTopic.replacingEvent()).toBe(null);
         expect(originalTopic.getContent().topic).toBe("orig");
+        expect(badlyEditedTopic.isRelation()).toBe(false);
+        expect(badlyEditedTopic.isRelation("m.replace")).toBe(false);
 
         await relations.addEvent(badlyEditedTopic);
         expect(originalTopic.replacingEvent()).toBe(null);

spec/unit/room-state.spec.js

@@ -3,7 +3,7 @@ import { makeBeaconEvent, makeBeaconInfoEvent } from "../test-utils/beacon";
 import { filterEmitCallsByEventType } from "../test-utils/emitter";
 import { RoomState, RoomStateEvent } from "../../src/models/room-state";
 import { BeaconEvent, getBeaconInfoIdentifier } from "../../src/models/beacon";
-import { EventType, RelationType } from "../../src/@types/event";
+import { EventType, RelationType, UNSTABLE_MSC2716_MARKER } from "../../src/@types/event";
 import {
     MatrixEvent,
     MatrixEventEvent,
@@ -258,6 +258,29 @@ describe("RoomState", function() {
             );
         });
 
+        it("should emit `RoomStateEvent.Marker` for each marker event", function() {
+            const events = [
+                utils.mkEvent({
+                    event: true,
+                    type: UNSTABLE_MSC2716_MARKER.name,
+                    room: roomId,
+                    user: userA,
+                    skey: "",
+                    content: {
+                        "m.insertion_id": "$abc",
+                    },
+                }),
+            ];
+            let emitCount = 0;
+            state.on("RoomState.Marker", function(markerEvent, markerFoundOptions) {
+                expect(markerEvent).toEqual(events[emitCount]);
+                expect(markerFoundOptions).toEqual({ timelineWasEmpty: true });
+                emitCount += 1;
+            });
+            state.setStateEvents(events, { timelineWasEmpty: true });
+            expect(emitCount).toEqual(1);
+        });
+
         describe('beacon events', () => {
             it('adds new beacon info events to state and emits', () => {
                 const beaconEvent = makeBeaconInfoEvent(userA, roomId);

spec/unit/room.spec.ts

@@ -52,7 +52,7 @@ describe("Room", function() {
         event: true,
         user: userA,
         room: roomId,
-    }, room.client) as MatrixEvent;
+    }, room.client);
 
     const mkReply = (target: MatrixEvent) => utils.mkEvent({
         event: true,
@@ -67,7 +67,7 @@ describe("Room", function() {
                 },
             },
         },
-    }, room.client) as MatrixEvent;
+    }, room.client);
 
     const mkEdit = (target: MatrixEvent, salt = Math.random()) => utils.mkEvent({
         event: true,
@@ -84,7 +84,7 @@ describe("Room", function() {
                 event_id: target.getId(),
             },
         },
-    }, room.client) as MatrixEvent;
+    }, room.client);
 
     const mkThreadResponse = (root: MatrixEvent) => utils.mkEvent({
         event: true,
@@ -101,7 +101,7 @@ describe("Room", function() {
                 "rel_type": "m.thread",
             },
         },
-    }, room.client) as MatrixEvent;
+    }, room.client);
 
     const mkReaction = (target: MatrixEvent) => utils.mkEvent({
         event: true,
@@ -115,7 +115,7 @@ describe("Room", function() {
                 "key": Math.random().toString(),
             },
         },
-    }, room.client) as MatrixEvent;
+    }, room.client);
 
     const mkRedaction = (target: MatrixEvent) => utils.mkEvent({
         event: true,
@@ -124,7 +124,7 @@ describe("Room", function() {
         room: roomId,
         redacts: target.getId(),
         content: {},
-    }, room.client) as MatrixEvent;
+    }, room.client);
 
     beforeEach(function() {
         room = new Room(roomId, new TestClient(userA, "device").client, userA);
@@ -133,6 +133,27 @@ describe("Room", function() {
         room.currentState = room.getLiveTimeline().endState = utils.mock(RoomState, "currentState");
     });
 
+    describe('getCreator', () => {
+        it("should return the creator from m.room.create", function() {
+            room.currentState.getStateEvents.mockImplementation(function(type, key) {
+                if (type === EventType.RoomCreate && key === "") {
+                    return utils.mkEvent({
+                        event: true,
+                        type: EventType.RoomCreate,
+                        skey: "",
+                        room: roomId,
+                        user: userA,
+                        content: {
+                            creator: userA,
+                        },
+                    });
+                }
+            });
+            const roomCreator = room.getCreator();
+            expect(roomCreator).toStrictEqual(userA);
+        });
+    });
+
     describe("getAvatarUrl", function() {
         const hsUrl = "https://my.home.server";
 
@@ -189,29 +210,24 @@ describe("Room", function() {
         const events: MatrixEvent[] = [
             utils.mkMessage({
                 room: roomId, user: userA, msg: "changing room name", event: true,
-            }) as MatrixEvent,
+            }),
             utils.mkEvent({
                 type: EventType.RoomName, room: roomId, user: userA, event: true,
                 content: { name: "New Room Name" },
-            }) as MatrixEvent,
+            }),
         ];
 
-        it("should call RoomState.setTypingEvent on m.typing events", function() {
-            const typing = utils.mkEvent({
-                room: roomId,
-                type: EventType.Typing,
-                event: true,
-                content: {
-                    user_ids: [userA],
-                },
-            });
-            room.addEphemeralEvents([typing]);
-            expect(room.currentState.setTypingEvent).toHaveBeenCalledWith(typing);
+        it("Make sure legacy overload passing options directly as parameters still works", () => {
+            expect(() => room.addLiveEvents(events, DuplicateStrategy.Replace, false)).not.toThrow();
+            expect(() => room.addLiveEvents(events, DuplicateStrategy.Ignore, true)).not.toThrow();
+            expect(() => room.addLiveEvents(events, "shouldfailbecauseinvalidduplicatestrategy", false)).toThrow();
         });
 
         it("should throw if duplicateStrategy isn't 'replace' or 'ignore'", function() {
             expect(function() {
-                room.addLiveEvents(events, "foo");
+                room.addLiveEvents(events, {
+                    duplicateStrategy: "foo",
+                });
             }).toThrow();
         });
 
@@ -219,11 +235,13 @@ describe("Room", function() {
             // make a duplicate
             const dupe = utils.mkMessage({
                 room: roomId, user: userA, msg: "dupe", event: true,
-            }) as MatrixEvent;
+            });
             dupe.event.event_id = events[0].getId();
             room.addLiveEvents(events);
             expect(room.timeline[0]).toEqual(events[0]);
-            room.addLiveEvents([dupe], DuplicateStrategy.Replace);
+            room.addLiveEvents([dupe], {
+                duplicateStrategy: DuplicateStrategy.Replace,
+            });
             expect(room.timeline[0]).toEqual(dupe);
         });
 
@@ -231,11 +249,13 @@ describe("Room", function() {
             // make a duplicate
             const dupe = utils.mkMessage({
                 room: roomId, user: userA, msg: "dupe", event: true,
-            }) as MatrixEvent;
+            });
             dupe.event.event_id = events[0].getId();
             room.addLiveEvents(events);
             expect(room.timeline[0]).toEqual(events[0]);
-            room.addLiveEvents([dupe], "ignore");
+            room.addLiveEvents([dupe], {
+                duplicateStrategy: "ignore",
+            });
             expect(room.timeline[0]).toEqual(events[0]);
         });
 
@@ -257,20 +277,22 @@ describe("Room", function() {
                 const events: MatrixEvent[] = [
                     utils.mkMembership({
                         room: roomId, mship: "invite", user: userB, skey: userA, event: true,
-                    }) as MatrixEvent,
+                    }),
                     utils.mkEvent({
                         type: EventType.RoomName, room: roomId, user: userB, event: true,
                         content: {
                             name: "New room",
                         },
-                    }) as MatrixEvent,
+                    }),
                 ];
                 room.addLiveEvents(events);
                 expect(room.currentState.setStateEvents).toHaveBeenCalledWith(
                     [events[0]],
+                    { timelineWasEmpty: undefined },
                 );
                 expect(room.currentState.setStateEvents).toHaveBeenCalledWith(
                     [events[1]],
+                    { timelineWasEmpty: undefined },
                 );
                 expect(events[0].forwardLooking).toBe(true);
                 expect(events[1].forwardLooking).toBe(true);
@@ -296,13 +318,13 @@ describe("Room", function() {
         it("should emit Room.localEchoUpdated when a local echo is updated", function() {
             const localEvent = utils.mkMessage({
                 room: roomId, user: userA, event: true,
-            }) as MatrixEvent;
+            });
             localEvent.status = EventStatus.SENDING;
             const localEventId = localEvent.getId();
 
             const remoteEvent = utils.mkMessage({
                 room: roomId, user: userA, event: true,
-            }) as MatrixEvent;
+            });
             remoteEvent.event.unsigned = { transaction_id: "TXN_ID" };
             const remoteEventId = remoteEvent.getId();
 
@@ -339,6 +361,86 @@ describe("Room", function() {
 
             expect(callCount).toEqual(2);
         });
+
+        it("should be able to update local echo without a txn ID (/send then /sync)", function() {
+            const eventJson = utils.mkMessage({
+                room: roomId, user: userA, event: false,
+            }) as object;
+            delete eventJson["txn_id"];
+            delete eventJson["event_id"];
+            const localEvent = new MatrixEvent(Object.assign({ event_id: "$temp" }, eventJson));
+            localEvent.status = EventStatus.SENDING;
+            expect(localEvent.getTxnId()).toBeNull();
+            expect(room.timeline.length).toEqual(0);
+
+            // first add the local echo. This is done before the /send request is even sent.
+            const txnId = "My_txn_id";
+            room.addPendingEvent(localEvent, txnId);
+            expect(room.getEventForTxnId(txnId)).toEqual(localEvent);
+            expect(room.timeline.length).toEqual(1);
+
+            // now the /send request returns the true event ID.
+            const realEventId = "$real-event-id";
+            room.updatePendingEvent(localEvent, EventStatus.SENT, realEventId);
+
+            // then /sync returns the remoteEvent, it should de-dupe based on the event ID.
+            const remoteEvent = new MatrixEvent(Object.assign({ event_id: realEventId }, eventJson));
+            expect(remoteEvent.getTxnId()).toBeNull();
+            room.addLiveEvents([remoteEvent]);
+            // the duplicate strategy code should ensure we don't add a 2nd event to the live timeline
+            expect(room.timeline.length).toEqual(1);
+            // but without the event ID matching we will still have the local event in pending events
+            expect(room.getEventForTxnId(txnId)).toBeUndefined();
+        });
+
+        it("should be able to update local echo without a txn ID (/sync then /send)", function() {
+            const eventJson = utils.mkMessage({
+                room: roomId, user: userA, event: false,
+            }) as object;
+            delete eventJson["txn_id"];
+            delete eventJson["event_id"];
+            const txnId = "My_txn_id";
+            const localEvent = new MatrixEvent(Object.assign({ event_id: "$temp", txn_id: txnId }, eventJson));
+            localEvent.status = EventStatus.SENDING;
+            expect(localEvent.getTxnId()).toEqual(txnId);
+            expect(room.timeline.length).toEqual(0);
+
+            // first add the local echo. This is done before the /send request is even sent.
+            room.addPendingEvent(localEvent, txnId);
+            expect(room.getEventForTxnId(txnId)).toEqual(localEvent);
+            expect(room.timeline.length).toEqual(1);
+
+            // now the /sync returns the remoteEvent, it is impossible for the JS SDK to de-dupe this.
+            const realEventId = "$real-event-id";
+            const remoteEvent = new MatrixEvent(Object.assign({ event_id: realEventId }, eventJson));
+            expect(remoteEvent.getUnsigned().transaction_id).toBeUndefined();
+            room.addLiveEvents([remoteEvent]);
+            expect(room.timeline.length).toEqual(2); // impossible to de-dupe as no txn ID or matching event ID
+
+            // then the /send request returns the real event ID.
+            // Now it is possible for the JS SDK to de-dupe this.
+            room.updatePendingEvent(localEvent, EventStatus.SENT, realEventId);
+
+            // the 2nd event should be removed from the timeline.
+            expect(room.timeline.length).toEqual(1);
+            // but without the event ID matching we will still have the local event in pending events
+            expect(room.getEventForTxnId(txnId)).toBeUndefined();
+        });
+    });
+
+    describe('addEphemeralEvents', () => {
+        it("should call RoomState.setTypingEvent on m.typing events", function() {
+            const typing = utils.mkEvent({
+                room: roomId,
+                type: EventType.Typing,
+                event: true,
+                content: {
+                    user_ids: [userA],
+                },
+            });
+            room.addEphemeralEvents([typing]);
+            expect(room.currentState.setTypingEvent).toHaveBeenCalledWith(typing);
+        });
     });
 
     describe("addEventsToTimeline", function() {
@@ -408,11 +510,11 @@ describe("Room", function() {
             const newEv = utils.mkEvent({
                 type: EventType.RoomName, room: roomId, user: userA, event: true,
                 content: { name: "New Room Name" },
-            }) as MatrixEvent;
+            });
             const oldEv = utils.mkEvent({
                 type: EventType.RoomName, room: roomId, user: userA, event: true,
                 content: { name: "Old Room Name" },
-            }) as MatrixEvent;
+            });
             room.addLiveEvents([newEv]);
             expect(newEv.sender).toEqual(sentinel);
             room.addEventsToTimeline([oldEv], true, room.getLiveTimeline());
@@ -445,10 +547,10 @@ describe("Room", function() {
 
             const newEv = utils.mkMembership({
                 room: roomId, mship: "invite", user: userB, skey: userA, event: true,
-            }) as MatrixEvent;
+            });
             const oldEv = utils.mkMembership({
                 room: roomId, mship: "ban", user: userB, skey: userA, event: true,
-            }) as MatrixEvent;
+            });
             room.addLiveEvents([newEv]);
             expect(newEv.target).toEqual(sentinel);
             room.addEventsToTimeline([oldEv], true, room.getLiveTimeline());
@@ -460,21 +562,23 @@ describe("Room", function() {
             const events: MatrixEvent[] = [
                 utils.mkMembership({
                     room: roomId, mship: "invite", user: userB, skey: userA, event: true,
-                }) as MatrixEvent,
+                }),
                 utils.mkEvent({
                     type: EventType.RoomName, room: roomId, user: userB, event: true,
                     content: {
                         name: "New room",
                     },
-                }) as MatrixEvent,
+                }),
             ];
 
             room.addEventsToTimeline(events, true, room.getLiveTimeline());
             expect(room.oldState.setStateEvents).toHaveBeenCalledWith(
                 [events[0]],
+                { timelineWasEmpty: undefined },
             );
             expect(room.oldState.setStateEvents).toHaveBeenCalledWith(
                 [events[1]],
+                { timelineWasEmpty: undefined },
             );
             expect(events[0].forwardLooking).toBe(false);
             expect(events[1].forwardLooking).toBe(false);
@@ -520,6 +624,23 @@ describe("Room", function() {
         it("should reset the legacy timeline fields", function() {
             room.addLiveEvents([events[0], events[1]]);
             expect(room.timeline.length).toEqual(2);
+
+            const oldStateBeforeRunningReset = room.oldState;
+            let oldStateUpdateEmitCount = 0;
+            room.on(RoomEvent.OldStateUpdated, function(room, previousOldState, oldState) {
+                expect(previousOldState).toBe(oldStateBeforeRunningReset);
+                expect(oldState).toBe(room.oldState);
+                oldStateUpdateEmitCount += 1;
+            });
+
+            const currentStateBeforeRunningReset = room.currentState;
+            let currentStateUpdateEmitCount = 0;
+            room.on(RoomEvent.CurrentStateUpdated, function(room, previousCurrentState, currentState) {
+                expect(previousCurrentState).toBe(currentStateBeforeRunningReset);
+                expect(currentState).toBe(room.currentState);
+                currentStateUpdateEmitCount += 1;
+            });
+
             room.resetLiveTimeline('sometoken', 'someothertoken');
 
             room.addLiveEvents([events[2]]);
@@ -529,6 +650,10 @@ describe("Room", function() {
                 newLiveTimeline.getState(EventTimeline.BACKWARDS));
             expect(room.currentState).toEqual(
                 newLiveTimeline.getState(EventTimeline.FORWARDS));
+            // Make sure `RoomEvent.OldStateUpdated` was emitted
+            expect(oldStateUpdateEmitCount).toEqual(1);
+            // Make sure `RoomEvent.OldStateUpdated` was emitted if necessary
+            expect(currentStateUpdateEmitCount).toEqual(timelineSupport ? 1 : 0);
         });
 
         it("should emit Room.timelineReset event and set the correct " +
@@ -571,13 +696,13 @@ describe("Room", function() {
         const events: MatrixEvent[] = [
             utils.mkMessage({
                 room: roomId, user: userA, msg: "1111", event: true,
-            }) as MatrixEvent,
+            }),
             utils.mkMessage({
                 room: roomId, user: userA, msg: "2222", event: true,
-            }) as MatrixEvent,
+            }),
             utils.mkMessage({
                 room: roomId, user: userA, msg: "3333", event: true,
-            }) as MatrixEvent,
+            }),
         ];
 
         it("should handle events in the same timeline", function() {
@@ -718,26 +843,26 @@ describe("Room", function() {
                 type: EventType.RoomJoinRules, room: roomId, user: userA, content: {
                     join_rule: rule,
                 }, event: true,
-            }) as MatrixEvent]);
+            })]);
         };
         const setAltAliases = function(aliases: string[]) {
             room.addLiveEvents([utils.mkEvent({
                 type: EventType.RoomCanonicalAlias, room: roomId, skey: "", content: {
                     alt_aliases: aliases,
                 }, event: true,
-            }) as MatrixEvent]);
+            })]);
         };
         const setAlias = function(alias: string) {
             room.addLiveEvents([utils.mkEvent({
                 type: EventType.RoomCanonicalAlias, room: roomId, skey: "", content: { alias }, event: true,
-            }) as MatrixEvent]);
+            })]);
         };
         const setRoomName = function(name: string) {
             room.addLiveEvents([utils.mkEvent({
                 type: EventType.RoomName, room: roomId, user: userA, content: {
                     name: name,
                 }, event: true,
-            }) as MatrixEvent]);
+            })]);
         };
         const addMember = function(userId: string, state = "join", opts: any = {}) {
             opts.room = roomId;
@@ -745,7 +870,7 @@ describe("Room", function() {
             opts.user = opts.user || userId;
             opts.skey = userId;
             opts.event = true;
-            const event = utils.mkMembership(opts) as MatrixEvent;
+            const event = utils.mkMembership(opts);
             room.addLiveEvents([event]);
             return event;
         };
@@ -1053,7 +1178,7 @@ describe("Room", function() {
         const eventToAck = utils.mkMessage({
             room: roomId, user: userA, msg: "PLEASE ACKNOWLEDGE MY EXISTENCE",
             event: true,
-        }) as MatrixEvent;
+        });
 
         function mkReceipt(roomId: string, records) {
             const content = {};
@@ -1119,7 +1244,7 @@ describe("Room", function() {
                 const nextEventToAck = utils.mkMessage({
                     room: roomId, user: userA, msg: "I AM HERE YOU KNOW",
                     event: true,
-                }) as MatrixEvent;
+                });
                 const ts = 13787898424;
                 room.addReceipt(mkReceipt(roomId, [
                     mkRecord(eventToAck.getId(), "m.read", userB, ts),
@@ -1154,11 +1279,11 @@ describe("Room", function() {
                 const eventTwo = utils.mkMessage({
                     room: roomId, user: userA, msg: "2222",
                     event: true,
-                }) as MatrixEvent;
+                });
                 const eventThree = utils.mkMessage({
                     room: roomId, user: userA, msg: "3333",
                     event: true,
-                }) as MatrixEvent;
+                });
                 const ts = 13787898424;
                 room.addReceipt(mkReceipt(roomId, [
                     mkRecord(eventToAck.getId(), "m.read", userB, ts),
@@ -1206,15 +1331,15 @@ describe("Room", function() {
                     utils.mkMessage({
                         room: roomId, user: userA, msg: "1111",
                         event: true,
-                    }) as MatrixEvent,
+                    }),
                     utils.mkMessage({
                         room: roomId, user: userA, msg: "2222",
                         event: true,
-                    }) as MatrixEvent,
+                    }),
                     utils.mkMessage({
                         room: roomId, user: userA, msg: "3333",
                         event: true,
-                    }) as MatrixEvent,
+                    }),
                 ];
 
                 room.addLiveEvents(events);
@@ -1244,15 +1369,15 @@ describe("Room", function() {
                     utils.mkMessage({
                         room: roomId, user: userA, msg: "1111",
                         event: true,
-                    }) as MatrixEvent,
+                    }),
                     utils.mkMessage({
                         room: roomId, user: userA, msg: "2222",
                         event: true,
-                    }) as MatrixEvent,
+                    }),
                     utils.mkMessage({
                         room: roomId, user: userA, msg: "3333",
                         event: true,
-                    }) as MatrixEvent,
+                    }),
                 ];
 
                 room.addLiveEvents(events);
@@ -1344,14 +1469,14 @@ describe("Room", function() {
             });
             const eventA = utils.mkMessage({
                 room: roomId, user: userA, msg: "remote 1", event: true,
-            }) as MatrixEvent;
+            });
             const eventB = utils.mkMessage({
                 room: roomId, user: userA, msg: "local 1", event: true,
-            }) as MatrixEvent;
+            });
             eventB.status = EventStatus.SENDING;
             const eventC = utils.mkMessage({
                 room: roomId, user: userA, msg: "remote 2", event: true,
-            }) as MatrixEvent;
+            });
             room.addLiveEvents([eventA]);
             room.addPendingEvent(eventB, "TXN1");
             room.addLiveEvents([eventC]);
@@ -1370,14 +1495,14 @@ describe("Room", function() {
             });
             const eventA = utils.mkMessage({
                 room: roomId, user: userA, msg: "remote 1", event: true,
-            }) as MatrixEvent;
+            });
             const eventB = utils.mkMessage({
                 room: roomId, user: userA, msg: "local 1", event: true,
-            }) as MatrixEvent;
+            });
             eventB.status = EventStatus.SENDING;
             const eventC = utils.mkMessage({
                 room: roomId, user: userA, msg: "remote 2", event: true,
-            }) as MatrixEvent;
+            });
             room.addLiveEvents([eventA]);
             room.addPendingEvent(eventB, "TXN1");
             room.addLiveEvents([eventC]);
@@ -1397,7 +1522,7 @@ describe("Room", function() {
             });
             const eventA = utils.mkMessage({
                 room: roomId, user: userA, event: true,
-            }) as MatrixEvent;
+            });
             eventA.status = EventStatus.SENDING;
             const eventId = eventA.getId();
 
@@ -1430,7 +1555,7 @@ describe("Room", function() {
             const room = new Room(roomId, null, userA);
             const eventA = utils.mkMessage({
                 room: roomId, user: userA, event: true,
-            }) as MatrixEvent;
+            });
             eventA.status = EventStatus.SENDING;
             const eventId = eventA.getId();
 
@@ -1495,6 +1620,8 @@ describe("Room", function() {
                         return Promise.resolve();
                     },
                     getSyncToken: () => "sync_token",
+                    getPendingEvents: jest.fn().mockResolvedValue([]),
+                    setPendingEvents: jest.fn().mockResolvedValue(undefined),
                 },
             };
         }
@@ -1505,7 +1632,7 @@ describe("Room", function() {
             room: roomId,
             event: true,
             name: "User A",
-        }) as MatrixEvent;
+        });
 
         it("should load members from server on first call", async function() {
             const client = createClientMock([memberEvent]);
@@ -1525,7 +1652,7 @@ describe("Room", function() {
                 room: roomId,
                 event: true,
                 name: "Ms A",
-            }) as MatrixEvent;
+            });
             const client = createClientMock([memberEvent2], [memberEvent]);
             const room = new Room(roomId, client as any, null, { lazyLoadMembers: true });
 
@@ -1596,7 +1723,7 @@ describe("Room", function() {
                 mship: "join",
                 room: roomId,
                 event: true,
-            }) as MatrixEvent]);
+            })]);
             expect(room.guessDMUserId()).toEqual(userB);
         });
         it("should return self if only member present", function() {
@@ -1629,11 +1756,11 @@ describe("Room", function() {
                 utils.mkMembership({
                     user: userA, mship: "join",
                     room: roomId, event: true, name: "User A",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userB, mship: "join",
                     room: roomId, event: true, name: "User B",
-                }) as MatrixEvent,
+                }),
             ]);
             expect(room.getDefaultRoomName(userA)).toEqual("User B");
         });
@@ -1644,11 +1771,11 @@ describe("Room", function() {
                 utils.mkMembership({
                     user: userA, mship: "join",
                     room: roomId, event: true, name: "User A",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userB, mship: "ban",
                     room: roomId, event: true, name: "User B",
-                }) as MatrixEvent,
+                }),
             ]);
             expect(room.getDefaultRoomName(userA)).toEqual("Empty room (was User B)");
         });
@@ -1659,11 +1786,11 @@ describe("Room", function() {
                 utils.mkMembership({
                     user: userA, mship: "join",
                     room: roomId, event: true, name: "User A",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userB, mship: "invite",
                     room: roomId, event: true, name: "User B",
-                }) as MatrixEvent,
+                }),
             ]);
             expect(room.getDefaultRoomName(userA)).toEqual("User B");
         });
@@ -1674,11 +1801,11 @@ describe("Room", function() {
                 utils.mkMembership({
                     user: userA, mship: "join",
                     room: roomId, event: true, name: "User A",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userB, mship: "leave",
                     room: roomId, event: true, name: "User B",
-                }) as MatrixEvent,
+                }),
             ]);
             expect(room.getDefaultRoomName(userA)).toEqual("Empty room (was User B)");
         });
@@ -1689,15 +1816,15 @@ describe("Room", function() {
                 utils.mkMembership({
                     user: userA, mship: "join",
                     room: roomId, event: true, name: "User A",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userB, mship: "join",
                     room: roomId, event: true, name: "User B",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userC, mship: "join",
                     room: roomId, event: true, name: "User C",
-                }) as MatrixEvent,
+                }),
             ]);
             expect(room.getDefaultRoomName(userA)).toEqual("User B and User C");
         });
@@ -1708,19 +1835,19 @@ describe("Room", function() {
                 utils.mkMembership({
                     user: userA, mship: "join",
                     room: roomId, event: true, name: "User A",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userB, mship: "join",
                     room: roomId, event: true, name: "User B",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userC, mship: "join",
                     room: roomId, event: true, name: "User C",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userD, mship: "join",
                     room: roomId, event: true, name: "User D",
-                }) as MatrixEvent,
+                }),
             ]);
             expect(room.getDefaultRoomName(userA)).toEqual("User B and 2 others");
         });
@@ -1733,18 +1860,18 @@ describe("Room", function() {
                 utils.mkMembership({
                     user: userA, mship: "join",
                     room: roomId, event: true, name: "User A",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userB, mship: "join",
                     room: roomId, event: true, name: "User B",
-                }) as MatrixEvent,
+                }),
                 utils.mkEvent({
                     type: UNSTABLE_ELEMENT_FUNCTIONAL_USERS.name, skey: "",
                     room: roomId, event: true,
                     content: {
                         service_members: [],
                     },
-                }) as MatrixEvent,
+                }),
             ]);
             expect(room.getDefaultRoomName(userA)).toEqual("User B");
         });
@@ -1755,11 +1882,11 @@ describe("Room", function() {
                 utils.mkMembership({
                     user: userA, mship: "join",
                     room: roomId, event: true, name: "User A",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userB, mship: "join",
                     room: roomId, event: true, name: "User B",
-                }) as MatrixEvent,
+                }),
                 utils.mkEvent({
                     type: UNSTABLE_ELEMENT_FUNCTIONAL_USERS.name,
                     skey: "",
@@ -1768,7 +1895,7 @@ describe("Room", function() {
                     content: {
                         service_members: 1,
                     },
-                }) as MatrixEvent,
+                }),
             ]);
             expect(room.getDefaultRoomName(userA)).toEqual("User B");
         });
@@ -1779,18 +1906,18 @@ describe("Room", function() {
                 utils.mkMembership({
                     user: userA, mship: "join",
                     room: roomId, event: true, name: "User A",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userB, mship: "join",
                     room: roomId, event: true, name: "User B",
-                }) as MatrixEvent,
+                }),
                 utils.mkEvent({
                     type: UNSTABLE_ELEMENT_FUNCTIONAL_USERS.name, skey: "",
                     room: roomId, event: true,
                     content: {
                         service_members: userB,
                     },
-                }) as MatrixEvent,
+                }),
             ]);
             expect(room.getDefaultRoomName(userA)).toEqual("User B");
         });
@@ -1801,18 +1928,18 @@ describe("Room", function() {
                 utils.mkMembership({
                     user: userA, mship: "join",
                     room: roomId, event: true, name: "User A",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userB, mship: "join",
                     room: roomId, event: true, name: "User B",
-                }) as MatrixEvent,
+                }),
                 utils.mkEvent({
                     type: UNSTABLE_ELEMENT_FUNCTIONAL_USERS.name, skey: "",
                     room: roomId, event: true,
                     content: {
                         service_members: [userB],
                     },
-                }) as MatrixEvent,
+                }),
             ]);
             expect(room.getDefaultRoomName(userA)).toEqual("Empty room");
         });
@@ -1823,22 +1950,22 @@ describe("Room", function() {
                 utils.mkMembership({
                     user: userA, mship: "join",
                     room: roomId, event: true, name: "User A",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userB, mship: "join",
                     room: roomId, event: true, name: "User B",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userC, mship: "join",
                     room: roomId, event: true, name: "User C",
-                }) as MatrixEvent,
+                }),
                 utils.mkEvent({
                     type: UNSTABLE_ELEMENT_FUNCTIONAL_USERS.name, skey: "",
                     room: roomId, event: true, user: userA,
                     content: {
                         service_members: [userC],
                     },
-                }) as MatrixEvent,
+                }),
             ]);
             expect(room.getDefaultRoomName(userA)).toEqual("User B");
         });
@@ -1849,22 +1976,22 @@ describe("Room", function() {
                 utils.mkMembership({
                     user: userA, mship: "join",
                     room: roomId, event: true, name: "User A",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userB, mship: "join",
                     room: roomId, event: true, name: "User B",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userC, mship: "join",
                     room: roomId, event: true, name: "User C",
-                }) as MatrixEvent,
+                }),
                 utils.mkEvent({
                     type: UNSTABLE_ELEMENT_FUNCTIONAL_USERS.name, skey: "",
                     room: roomId, event: true, user: userA,
                     content: {
                         service_members: [userB, userC],
                     },
-                }) as MatrixEvent,
+                }),
             ]);
             expect(room.getDefaultRoomName(userA)).toEqual("Empty room");
         });
@@ -1875,18 +2002,18 @@ describe("Room", function() {
                 utils.mkMembership({
                     user: userA, mship: "join",
                     room: roomId, event: true, name: "User A",
-                }) as MatrixEvent,
+                }),
                 utils.mkMembership({
                     user: userB, mship: "join",
                     room: roomId, event: true, name: "User B",
-                }) as MatrixEvent,
+                }),
                 utils.mkEvent({
                     type: UNSTABLE_ELEMENT_FUNCTIONAL_USERS.name, skey: "",
                     room: roomId, event: true, user: userA,
                     content: {
                         service_members: [userC],
                     },
-                }) as MatrixEvent,
+                }),
             ]);
             expect(room.getDefaultRoomName(userA)).toEqual("User B");
         });
@@ -2274,7 +2401,7 @@ describe("Room", function() {
             const thread = threadRoot.getThread();
             expect(thread.rootEvent).toBe(threadRoot);
 
-            const rootRelations = thread.timelineSet.getRelationsForEvent(
+            const rootRelations = thread.timelineSet.relations.getChildEventsForEvent(
                 threadRoot.getId(),
                 RelationType.Annotation,
                 EventType.Reaction,
@@ -2284,7 +2411,7 @@ describe("Room", function() {
             expect(rootRelations[0][1].size).toEqual(1);
             expect(rootRelations[0][1].has(rootReaction)).toBeTruthy();
 
-            const responseRelations = thread.timelineSet.getRelationsForEvent(
+            const responseRelations = thread.timelineSet.relations.getChildEventsForEvent(
                 threadResponse.getId(),
                 RelationType.Annotation,
                 EventType.Reaction,

spec/unit/stores/indexeddb.spec.ts

@@ -0,0 +1,114 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import 'fake-indexeddb/auto';
+import 'jest-localstorage-mock';
+
+import { IndexedDBStore, IStateEventWithRoomId, MemoryStore } from "../../../src";
+import { emitPromise } from "../../test-utils/test-utils";
+import { LocalIndexedDBStoreBackend } from "../../../src/store/indexeddb-local-backend";
+
+describe("IndexedDBStore", () => {
+    afterEach(() => {
+        jest.clearAllMocks();
+    });
+
+    const roomId = "!room:id";
+    it("should degrade to MemoryStore on IDB errors", async () => {
+        const store = new IndexedDBStore({
+            indexedDB: indexedDB,
+            dbName: "database",
+            localStorage,
+        });
+        await store.startup();
+
+        const member1: IStateEventWithRoomId = {
+            room_id: roomId,
+            event_id: "!ev1:id",
+            sender: "@user1:id",
+            state_key: "@user1:id",
+            type: "m.room.member",
+            origin_server_ts: 123,
+            content: {},
+        };
+        const member2: IStateEventWithRoomId = {
+            room_id: roomId,
+            event_id: "!ev2:id",
+            sender: "@user2:id",
+            state_key: "@user2:id",
+            type: "m.room.member",
+            origin_server_ts: 123,
+            content: {},
+        };
+
+        expect(await store.getOutOfBandMembers(roomId)).toBe(null);
+        await store.setOutOfBandMembers(roomId, [member1]);
+        expect(await store.getOutOfBandMembers(roomId)).toHaveLength(1);
+
+        // Simulate a broken IDB
+        (store.backend as LocalIndexedDBStoreBackend)["db"].transaction = (): IDBTransaction => {
+            const err = new Error("Failed to execute 'transaction' on 'IDBDatabase': " +
+                "The database connection is closing.");
+            err.name = "InvalidStateError";
+            throw err;
+        };
+
+        expect(await store.getOutOfBandMembers(roomId)).toHaveLength(1);
+        await Promise.all([
+            emitPromise(store["emitter"], "degraded"),
+            store.setOutOfBandMembers(roomId, [member1, member2]),
+        ]);
+        expect(await store.getOutOfBandMembers(roomId)).toHaveLength(2);
+    });
+
+    it("should use MemoryStore methods for pending events if no localStorage", async () => {
+        jest.spyOn(MemoryStore.prototype, "setPendingEvents");
+        jest.spyOn(MemoryStore.prototype, "getPendingEvents");
+
+        const store = new IndexedDBStore({
+            indexedDB: indexedDB,
+            dbName: "database",
+            localStorage: undefined,
+        });
+
+        const events = [{ type: "test" }];
+        await store.setPendingEvents(roomId, events);
+        expect(MemoryStore.prototype.setPendingEvents).toHaveBeenCalledWith(roomId, events);
+        await expect(store.getPendingEvents(roomId)).resolves.toEqual(events);
+        expect(MemoryStore.prototype.getPendingEvents).toHaveBeenCalledWith(roomId);
+    });
+
+    it("should persist pending events to localStorage if available", async () => {
+        jest.spyOn(MemoryStore.prototype, "setPendingEvents");
+        jest.spyOn(MemoryStore.prototype, "getPendingEvents");
+
+        const store = new IndexedDBStore({
+            indexedDB: indexedDB,
+            dbName: "database",
+            localStorage,
+        });
+
+        await expect(store.getPendingEvents(roomId)).resolves.toEqual([]);
+        const events = [{ type: "test" }];
+        await store.setPendingEvents(roomId, events);
+        expect(MemoryStore.prototype.setPendingEvents).not.toHaveBeenCalled();
+        await expect(store.getPendingEvents(roomId)).resolves.toEqual(events);
+        expect(MemoryStore.prototype.getPendingEvents).not.toHaveBeenCalled();
+        expect(localStorage.getItem("mx_pending_events_" + roomId)).toBe(JSON.stringify(events));
+        await store.setPendingEvents(roomId, []);
+        expect(localStorage.getItem("mx_pending_events_" + roomId)).toBeNull();
+    });
+});

spec/unit/timeline-window.spec.js

@@ -35,13 +35,14 @@ function createTimeline(numEvents, baseIndex) {
     return timeline;
 }
 
-function addEventsToTimeline(timeline, numEvents, atStart) {
+function addEventsToTimeline(timeline, numEvents, toStartOfTimeline) {
     for (let i = 0; i < numEvents; i++) {
         timeline.addEvent(
             utils.mkMessage({
                 room: ROOM_ID, user: USER_ID,
                 event: true,
-            }), atStart,
+            }),
+            { toStartOfTimeline },
         );
     }
 }

spec/unit/utils.spec.ts

@@ -109,18 +109,6 @@ describe("utils", function() {
         });
     });
 
-    describe("checkObjectHasNoAdditionalKeys", function() {
-        it("should throw for extra keys", function() {
-            expect(function() {
-                utils.checkObjectHasNoAdditionalKeys({ foo: "bar", baz: 4 }, ["foo"]);
-            }).toThrow();
-
-            expect(function() {
-                utils.checkObjectHasNoAdditionalKeys({ foo: "bar" }, ["foo"]);
-            }).not.toThrow();
-        });
-    });
-
     describe("deepCompare", function() {
         const assert = {
             isTrue: function(x: any) {

spec/unit/webrtc/call.spec.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Matrix.org Foundation C.I.C.
+Copyright 2020 - 2022 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,119 +15,25 @@ limitations under the License.
 */
 
 import { TestClient } from '../../TestClient';
-import { MatrixCall, CallErrorCode, CallEvent, supportsMatrixCall } from '../../../src/webrtc/call';
+import { MatrixCall, CallErrorCode, CallEvent, supportsMatrixCall, CallType } from '../../../src/webrtc/call';
 import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes';
-import { RoomMember } from "../../../src";
+import {
+    DUMMY_SDP,
+    MockMediaHandler,
+    MockMediaStream,
+    MockMediaStreamTrack,
+    MockMediaDeviceInfo,
+    MockRTCPeerConnection,
+} from "../../test-utils/webrtc";
+import { CallFeed } from "../../../src/webrtc/callFeed";
 
-const DUMMY_SDP = (
-    "v=0\r\n" +
-    "o=- 5022425983810148698 2 IN IP4 127.0.0.1\r\n" +
-    "s=-\r\nt=0 0\r\na=group:BUNDLE 0\r\n" +
-    "a=msid-semantic: WMS h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA\r\n" +
-    "m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126\r\n" +
-    "c=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:hLDR\r\n" +
-    "a=ice-pwd:bMGD9aOldHWiI+6nAq/IIlRw\r\n" +
-    "a=ice-options:trickle\r\n" +
-    "a=fingerprint:sha-256 E4:94:84:F9:4A:98:8A:56:F5:5F:FD:AF:72:B9:32:89:49:5C:4B:9A:" +
-        "4A:15:8E:41:8A:F3:69:E4:39:52:DC:D6\r\n" +
-    "a=setup:active\r\n" +
-    "a=mid:0\r\n" +
-    "a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n" +
-    "a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n" +
-    "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n" +
-    "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n" +
-    "a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n" +
-    "a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n" +
-    "a=sendrecv\r\n" +
-    "a=msid:h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA 4357098f-3795-4131-bff4-9ba9c0348c49\r\n" +
-    "a=rtcp-mux\r\n" +
-    "a=rtpmap:111 opus/48000/2\r\n" +
-    "a=rtcp-fb:111 transport-cc\r\n" +
-    "a=fmtp:111 minptime=10;useinbandfec=1\r\n" +
-    "a=rtpmap:103 ISAC/16000\r\n" +
-    "a=rtpmap:104 ISAC/32000\r\n" +
-    "a=rtpmap:9 G722/8000\r\n" +
-    "a=rtpmap:0 PCMU/8000\r\n" +
-    "a=rtpmap:8 PCMA/8000\r\n" +
-    "a=rtpmap:106 CN/32000\r\n" +
-    "a=rtpmap:105 CN/16000\r\n" +
-    "a=rtpmap:13 CN/8000\r\n" +
-    "a=rtpmap:110 telephone-event/48000\r\n" +
-    "a=rtpmap:112 telephone-event/32000\r\n" +
-    "a=rtpmap:113 telephone-event/16000\r\n" +
-    "a=rtpmap:126 telephone-event/8000\r\n" +
-    "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n"
-);
+const startVoiceCall = async (client: TestClient, call: MatrixCall): Promise<void> => {
+    const callPromise = call.placeVoiceCall();
+    await client.httpBackend.flush("");
+    await callPromise;
 
-class MockRTCPeerConnection {
-    localDescription: RTCSessionDescription;
-
-    constructor() {
-        this.localDescription = {
-            sdp: DUMMY_SDP,
-            type: 'offer',
-            toJSON: function() {},
-        };
-    }
-
-    addEventListener() {}
-    createOffer() {
-        return Promise.resolve({});
-    }
-    setRemoteDescription() {
-        return Promise.resolve();
-    }
-    setLocalDescription() {
-        return Promise.resolve();
-    }
-    close() {}
-    getStats() { return []; }
-    addTrack(track: MockMediaStreamTrack) {return new MockRTCRtpSender(track);}
-}
-
-class MockRTCRtpSender {
-    constructor(public track: MockMediaStreamTrack) {}
-
-    replaceTrack(track: MockMediaStreamTrack) {this.track = track;}
-}
-
-class MockMediaStreamTrack {
-    constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) {}
-
-    stop() {}
-}
-
-class MockMediaStream {
-    constructor(
-        public id: string,
-        private tracks: MockMediaStreamTrack[] = [],
-    ) {}
-
-    getTracks() { return this.tracks; }
-    getAudioTracks() { return this.tracks.filter((track) => track.kind === "audio"); }
-    getVideoTracks() { return this.tracks.filter((track) => track.kind === "video"); }
-    addEventListener() {}
-    removeEventListener() { }
-    addTrack(track: MockMediaStreamTrack) {this.tracks.push(track);}
-    removeTrack(track: MockMediaStreamTrack) {this.tracks.splice(this.tracks.indexOf(track), 1);}
-}
-
-class MockMediaDeviceInfo {
-    constructor(
-        public kind: "audio" | "video",
-    ) {}
-}
-
-class MockMediaHandler {
-    getUserMediaStream(audio: boolean, video: boolean) {
-        const tracks = [];
-        if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio"));
-        if (video) tracks.push(new MockMediaStreamTrack("video_track", "video"));
-
-        return new MockMediaStream("mock_stream_from_media_handler", tracks);
-    }
-    stopUserMediaStream() {}
-}
+    call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
+};
 
 describe('Call', function() {
     let client;
@@ -185,9 +91,8 @@ describe('Call', function() {
     });
 
     it('should ignore candidate events from non-matching party ID', async function() {
-        const callPromise = call.placeVoiceCall();
-        await client.httpBackend.flush();
-        await callPromise;
+        await startVoiceCall(client, call);
+
         await call.onAnswerReceived({
             getContent: () => {
                 return {
@@ -241,9 +146,7 @@ describe('Call', function() {
     });
 
     it('should add candidates received before answer if party ID is correct', async function() {
-        const callPromise = call.placeVoiceCall();
-        await client.httpBackend.flush();
-        await callPromise;
+        await startVoiceCall(client, call);
         call.peerConn.addIceCandidate = jest.fn();
 
         call.onRemoteIceCandidatesReceived({
@@ -301,9 +204,7 @@ describe('Call', function() {
     });
 
     it('should map asserted identity messages to remoteAssertedIdentity', async function() {
-        const callPromise = call.placeVoiceCall();
-        await client.httpBackend.flush();
-        await callPromise;
+        await startVoiceCall(client, call);
         await call.onAnswerReceived({
             getContent: () => {
                 return {
@@ -345,13 +246,7 @@ describe('Call', function() {
     });
 
     it("should map SDPStreamMetadata to feeds", async () => {
-        const callPromise = call.placeVoiceCall();
-        await client.httpBackend.flush();
-        await callPromise;
-
-        call.getOpponentMember = () => {
-            return { userId: "@bob:bar.uk" };
-        };
+        await startVoiceCall(client, call);
 
         await call.onAnswerReceived({
             getContent: () => {
@@ -389,13 +284,7 @@ describe('Call', function() {
     });
 
     it("should fallback to replaceTrack() if the other side doesn't support SPDStreamMetadata", async () => {
-        const callPromise = call.placeVoiceCall();
-        await client.httpBackend.flush();
-        await callPromise;
-
-        call.getOpponentMember = () => {
-            return { userId: "@bob:bar.uk" } as RoomMember;
-        };
+        await startVoiceCall(client, call);
 
         await call.onAnswerReceived({
             getContent: () => {
@@ -438,9 +327,7 @@ describe('Call', function() {
             ),
         );
 
-        const callPromise = call.placeVideoCall();
-        await client.httpBackend.flush();
-        await callPromise;
+        await startVoiceCall(client, call);
 
         await call.onAnswerReceived({
             getContent: () => {
@@ -476,9 +363,7 @@ describe('Call', function() {
     });
 
     it("should handle upgrade to video call", async () => {
-        const callPromise = call.placeVoiceCall();
-        await client.httpBackend.flush();
-        await callPromise;
+        await startVoiceCall(client, call);
 
         await call.onAnswerReceived({
             getContent: () => {
@@ -506,6 +391,338 @@ describe('Call', function() {
         }).track.id).toBe("video_track");
     });
 
+    describe("should handle stream replacement", () => {
+        it("with both purpose and id", async () => {
+            await startVoiceCall(client, call);
+
+            call.updateRemoteSDPStreamMetadata({
+                "remote_stream1": {
+                    purpose: SDPStreamMetadataPurpose.Usermedia,
+                },
+            });
+            call.pushRemoteFeed(new MockMediaStream("remote_stream1", []));
+            const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream1");
+
+            call.updateRemoteSDPStreamMetadata({
+                "remote_stream2": {
+                    purpose: SDPStreamMetadataPurpose.Usermedia,
+                },
+            });
+            call.pushRemoteFeed(new MockMediaStream("remote_stream2", []));
+
+            expect(feed?.stream?.id).toBe("remote_stream2");
+        });
+
+        it("with just purpose", async () => {
+            await startVoiceCall(client, call);
+
+            call.updateRemoteSDPStreamMetadata({
+                "remote_stream1": {
+                    purpose: SDPStreamMetadataPurpose.Usermedia,
+                },
+            });
+            call.pushRemoteFeed(new MockMediaStream("remote_stream1", []));
+            const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream1");
+
+            call.updateRemoteSDPStreamMetadata({
+                "remote_stream2": {
+                    purpose: SDPStreamMetadataPurpose.Usermedia,
+                },
+            });
+            call.pushRemoteFeed(new MockMediaStream("remote_stream2", []));
+
+            expect(feed?.stream?.id).toBe("remote_stream2");
+        });
+
+        it("should not replace purpose is different", async () => {
+            await startVoiceCall(client, call);
+
+            call.updateRemoteSDPStreamMetadata({
+                "remote_stream1": {
+                    purpose: SDPStreamMetadataPurpose.Usermedia,
+                },
+            });
+            call.pushRemoteFeed(new MockMediaStream("remote_stream1", []));
+            const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream1");
+
+            call.updateRemoteSDPStreamMetadata({
+                "remote_stream2": {
+                    purpose: SDPStreamMetadataPurpose.Screenshare,
+                },
+            });
+            call.pushRemoteFeed(new MockMediaStream("remote_stream2", []));
+
+            expect(feed?.stream?.id).toBe("remote_stream1");
+        });
+    });
+
+    it("should handle SDPStreamMetadata changes", async () => {
+        await startVoiceCall(client, call);
+
+        call.updateRemoteSDPStreamMetadata({
+            "remote_stream": {
+                purpose: SDPStreamMetadataPurpose.Usermedia,
+                audio_muted: false,
+                video_muted: false,
+            },
+        });
+        call.pushRemoteFeed(new MockMediaStream("remote_stream", []));
+        const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream");
+
+        call.onSDPStreamMetadataChangedReceived({
+            getContent: () => ({
+                [SDPStreamMetadataKey]: {
+                    "remote_stream": {
+                        purpose: SDPStreamMetadataPurpose.Screenshare,
+                        audio_muted: true,
+                        video_muted: true,
+                        id: "feed_id2",
+                    },
+                },
+            }),
+        });
+
+        expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Screenshare);
+        expect(feed?.audioMuted).toBe(true);
+        expect(feed?.videoMuted).toBe(true);
+    });
+
+    it("should choose opponent member", async () => {
+        const callPromise = call.placeVoiceCall();
+        await client.httpBackend.flush();
+        await callPromise;
+
+        const opponentMember = {
+            roomId: call.roomId,
+            userId: "opponentUserId",
+        };
+        const opponentCaps = {
+            "m.call.transferee": true,
+            "m.call.dtmf": false,
+        };
+        call.chooseOpponent({
+            getContent: () => ({
+                version: 1,
+                party_id: "party_id",
+                capabilities: opponentCaps,
+            }),
+            sender: opponentMember,
+        });
+
+        expect(call.getOpponentMember()).toBe(opponentMember);
+        expect(call.opponentPartyId).toBe("party_id");
+        expect(call.opponentCaps).toBe(opponentCaps);
+        expect(call.opponentCanBeTransferred()).toBe(true);
+        expect(call.opponentSupportsDTMF()).toBe(false);
+    });
+
+    describe("should deduce the call type correctly", () => {
+        it("if no video", async () => {
+            call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
+
+            call.pushRemoteFeed(new MockMediaStream("remote_stream1", []));
+            expect(call.type).toBe(CallType.Voice);
+        });
+
+        it("if remote video", async () => {
+            call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
+
+            call.pushRemoteFeed(new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")]));
+            expect(call.type).toBe(CallType.Video);
+        });
+
+        it("if local video", async () => {
+            call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
+
+            call.pushNewLocalFeed(
+                new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")]),
+                SDPStreamMetadataPurpose.Usermedia,
+                false,
+            );
+            expect(call.type).toBe(CallType.Video);
+        });
+    });
+
+    it("should correctly generate local SDPStreamMetadata", async () => {
+        const callPromise = call.placeCallWithCallFeeds([new CallFeed({
+            client,
+            // @ts-ignore Mock
+            stream: new MockMediaStream("local_stream1", [new MockMediaStreamTrack("track_id", "audio")]),
+            roomId: call.roomId,
+            userId: client.getUserId(),
+            purpose: SDPStreamMetadataPurpose.Usermedia,
+            audioMuted: false,
+            videoMuted: false,
+        })]);
+        await client.httpBackend.flush();
+        await callPromise;
+        call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
+
+        call.pushNewLocalFeed(
+            new MockMediaStream("local_stream2", [new MockMediaStreamTrack("track_id", "video")]),
+            SDPStreamMetadataPurpose.Screenshare, "feed_id2",
+        );
+        await call.setMicrophoneMuted(true);
+
+        expect(call.getLocalSDPStreamMetadata()).toStrictEqual({
+            "local_stream1": {
+                "purpose": SDPStreamMetadataPurpose.Usermedia,
+                "audio_muted": true,
+                "video_muted": true,
+            },
+            "local_stream2": {
+                "purpose": SDPStreamMetadataPurpose.Screenshare,
+                "audio_muted": true,
+                "video_muted": false,
+            },
+        });
+    });
+
+    it("feed and stream getters return correctly", async () => {
+        const localUsermediaStream = new MockMediaStream("local_usermedia_stream_id", []);
+        const localScreensharingStream = new MockMediaStream("local_screensharing_stream_id", []);
+        const remoteUsermediaStream = new MockMediaStream("remote_usermedia_stream_id", []);
+        const remoteScreensharingStream = new MockMediaStream("remote_screensharing_stream_id", []);
+
+        const callPromise = call.placeCallWithCallFeeds([
+            new CallFeed({
+                client,
+                userId: client.getUserId(),
+                // @ts-ignore Mock
+                stream: localUsermediaStream,
+                purpose: SDPStreamMetadataPurpose.Usermedia,
+                id: "local_usermedia_feed_id",
+                audioMuted: false,
+                videoMuted: false,
+            }),
+            new CallFeed({
+                client,
+                userId: client.getUserId(),
+                // @ts-ignore Mock
+                stream: localScreensharingStream,
+                purpose: SDPStreamMetadataPurpose.Screenshare,
+                id: "local_screensharing_feed_id",
+                audioMuted: false,
+                videoMuted: false,
+            }),
+        ]);
+        await client.httpBackend.flush();
+        await callPromise;
+        call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
+
+        call.updateRemoteSDPStreamMetadata({
+            "remote_usermedia_stream_id": {
+                purpose: SDPStreamMetadataPurpose.Usermedia,
+                id: "remote_usermedia_feed_id",
+                audio_muted: false,
+                video_muted: false,
+            },
+            "remote_screensharing_stream_id": {
+                purpose: SDPStreamMetadataPurpose.Screenshare,
+                id: "remote_screensharing_feed_id",
+                audio_muted: false,
+                video_muted: false,
+            },
+        });
+        call.pushRemoteFeed(remoteUsermediaStream);
+        call.pushRemoteFeed(remoteScreensharingStream);
+
+        expect(call.localUsermediaFeed.stream).toBe(localUsermediaStream);
+        expect(call.localUsermediaStream).toBe(localUsermediaStream);
+        expect(call.localScreensharingFeed.stream).toBe(localScreensharingStream);
+        expect(call.localScreensharingStream).toBe(localScreensharingStream);
+        expect(call.remoteUsermediaFeed.stream).toBe(remoteUsermediaStream);
+        expect(call.remoteUsermediaStream).toBe(remoteUsermediaStream);
+        expect(call.remoteScreensharingFeed.stream).toBe(remoteScreensharingStream);
+        expect(call.remoteScreensharingStream).toBe(remoteScreensharingStream);
+        expect(call.hasRemoteUserMediaAudioTrack).toBe(false);
+    });
+
+    it("should end call after receiving a select event with a different party id", async () => {
+        const callPromise = call.initWithInvite({
+            getContent: () => ({
+                version: 1,
+                call_id: "call_id",
+                party_id: "remote_party_id",
+                offer: {
+                    sdp: DUMMY_SDP,
+                },
+            }),
+            getLocalAge: () => null,
+        });
+        call.feeds.push(new CallFeed({
+            client,
+            userId: "remote_user_id",
+            // @ts-ignore Mock
+            stream: new MockMediaStream("remote_stream_id", [new MockMediaStreamTrack("remote_tack_id")]),
+            id: "remote_feed_id",
+            purpose: SDPStreamMetadataPurpose.Usermedia,
+        }));
+        await client.httpBackend.flush();
+        await callPromise;
+
+        const callHangupCallback = jest.fn();
+        call.on(CallEvent.Hangup, callHangupCallback);
+
+        await call.onSelectAnswerReceived({
+            getContent: () => ({
+                version: 1,
+                call_id: call.callId,
+                party_id: 'party_id',
+                selected_party_id: "different_party_id",
+            }),
+        });
+
+        expect(callHangupCallback).toHaveBeenCalled();
+    });
+
+    describe("turn servers", () => {
+        it("should fallback if allowed", async () => {
+            client.client.isFallbackICEServerAllowed = () => true;
+            const localCall = new MatrixCall({
+                client: client.client,
+                roomId: '!room_id',
+            });
+
+            expect((localCall as any).turnServers).toStrictEqual([{ urls: ["stun:turn.matrix.org"] }]);
+        });
+
+        it("should not fallback if not allowed", async () => {
+            client.client.isFallbackICEServerAllowed = () => false;
+            const localCall = new MatrixCall({
+                client: client.client,
+                roomId: '!room_id',
+            });
+
+            expect((localCall as any).turnServers).toStrictEqual([]);
+        });
+
+        it("should not fallback if we supplied turn servers", async () => {
+            client.client.isFallbackICEServerAllowed = () => true;
+            const turnServers = [{ urls: ["turn.server.org"] }];
+            const localCall = new MatrixCall({
+                client: client.client,
+                roomId: '!room_id',
+                turnServers,
+            });
+
+            expect((localCall as any).turnServers).toStrictEqual(turnServers);
+        });
+    });
+
+    it("should handle creating a data channel", async () => {
+        await startVoiceCall(client, call);
+
+        const dataChannelCallback = jest.fn();
+        call.on(CallEvent.DataChannel, dataChannelCallback);
+
+        const dataChannel = call.createDataChannel("data_channel_label", { id: 123 });
+
+        expect(dataChannelCallback).toHaveBeenCalledWith(dataChannel);
+        expect(dataChannel.label).toBe("data_channel_label");
+        expect(dataChannel.id).toBe(123);
+    });
+
     describe("supportsMatrixCall", () => {
         it("should return true when the environment is right", () => {
             expect(supportsMatrixCall()).toBe(true);

spec/unit/webrtc/callEventHandler.spec.ts

@@ -15,7 +15,16 @@ limitations under the License.
 */
 
 import { TestClient } from '../../TestClient';
-import { ClientEvent, EventType, MatrixEvent, RoomEvent } from "../../../src";
+import {
+    ClientEvent,
+    EventTimeline,
+    EventTimelineSet,
+    EventType,
+    IRoomTimelineData,
+    MatrixEvent,
+    Room,
+    RoomEvent,
+} from "../../../src";
 import { CallEventHandler, CallEventHandlerEvent } from "../../../src/webrtc/callEventHandler";
 import { SyncState } from "../../../src/sync";
 
@@ -23,6 +32,8 @@ describe("callEventHandler", () => {
     it("should ignore a call if invite & hangup come within a single sync", () => {
         const testClient = new TestClient();
         const client = testClient.client;
+        const room = new Room("!room:id", client, "@user:id");
+        const timelineData: IRoomTimelineData = { timeline: new EventTimeline(new EventTimelineSet(room, {})) };
         client.callEventHandler = new CallEventHandler(client);
         client.callEventHandler.start();
 
@@ -33,7 +44,7 @@ describe("callEventHandler", () => {
                 call_id: "123",
             },
         });
-        client.emit(RoomEvent.Timeline, callInvite);
+        client.emit(RoomEvent.Timeline, callInvite, room, false, false, timelineData);
 
         const callHangup = new MatrixEvent({
             type: EventType.CallHangup,
@@ -41,13 +52,13 @@ describe("callEventHandler", () => {
                 call_id: "123",
             },
         });
-        client.emit(RoomEvent.Timeline, callHangup);
+        client.emit(RoomEvent.Timeline, callHangup, room, false, false, timelineData);
 
         const incomingCallEmitted = jest.fn();
         client.on(CallEventHandlerEvent.Incoming, incomingCallEmitted);
 
         client.getSyncState = jest.fn().mockReturnValue(SyncState.Syncing);
-        client.emit(ClientEvent.Sync);
+        client.emit(ClientEvent.Sync, SyncState.Syncing);
 
         expect(incomingCallEmitted).not.toHaveBeenCalled();
     });

spec/unit/webrtc/callFeed.spec.ts

@@ -0,0 +1,61 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes";
+import { CallFeed, CallFeedEvent } from "../../../src/webrtc/callFeed";
+import { MockMediaStream, MockMediaStreamTrack } from "../../test-utils/webrtc";
+import { TestClient } from "../../TestClient";
+
+describe("CallFeed", () => {
+    const roomId = "room_id";
+
+    let client;
+
+    beforeEach(() => {
+        client = new TestClient("@alice:foo", "somedevice", "token", undefined, {});
+    });
+
+    afterEach(() => {
+        client.stop();
+    });
+
+    it("should handle stream replacement", () => {
+        const feedNewStreamCallback = jest.fn();
+        const feed = new CallFeed({
+            client,
+            roomId,
+            userId: "user1",
+            // @ts-ignore Mock
+            stream: new MockMediaStream("stream1"),
+            id: "id",
+            purpose: SDPStreamMetadataPurpose.Usermedia,
+            audioMuted: false,
+            videoMuted: false,
+        });
+        feed.on(CallFeedEvent.NewStream, feedNewStreamCallback);
+
+        const replacementStream = new MockMediaStream("stream2");
+        // @ts-ignore Mock
+        feed.setNewStream(replacementStream);
+        expect(feedNewStreamCallback).toHaveBeenCalledWith(replacementStream);
+        expect(feed.stream).toBe(replacementStream);
+
+        feedNewStreamCallback.mockReset();
+
+        replacementStream.addTrack(new MockMediaStreamTrack("track_id", "audio"));
+        expect(feedNewStreamCallback).toHaveBeenCalledWith(replacementStream);
+    });
+});

src/@types/event.ts

@@ -151,6 +151,14 @@ export const UNSTABLE_MSC3089_LEAF = new UnstableValue("m.leaf", "org.matrix.msc
  */
 export const UNSTABLE_MSC3089_BRANCH = new UnstableValue("m.branch", "org.matrix.msc3089.branch");
 
+/**
+ * Marker event type to point back at imported historical content in a room. See
+ * [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716).
+ * Note that this reference is UNSTABLE and subject to breaking changes,
+ * including its eventual removal.
+ */
+export const UNSTABLE_MSC2716_MARKER = new UnstableValue("m.room.marker", "org.matrix.msc2716.marker");
+
 /**
  * Functional members type for declaring a purpose of room members (e.g. helpful bots).
  * Note that this reference is UNSTABLE and subject to breaking changes, including its

src/@types/requests.ts

@@ -22,6 +22,7 @@ import { IRoomEventFilter } from "../filter";
 import { Direction } from "../models/event-timeline";
 import { PushRuleAction } from "./PushRules";
 import { IRoomEvent } from "../sync-accumulator";
+import { RoomType } from "./event";
 
 // allow camelcase as these are things that go onto the wire
 /* eslint-disable camelcase */
@@ -111,7 +112,8 @@ export interface IRoomDirectoryOptions {
     limit?: number;
     since?: string;
     filter?: {
-        generic_search_term: string;
+        generic_search_term?: string;
+        "org.matrix.msc3827.room_types"?: Array<RoomType | null>;
     };
     include_all_networks?: boolean;
     third_party_instance_id?: string;

src/client.ts

@@ -61,6 +61,7 @@ import {
     PREFIX_R0,
     PREFIX_UNSTABLE,
     PREFIX_V1,
+    PREFIX_V3,
     retryNetworkOperation,
     UploadContentResponseType,
 } from "./http-api";
@@ -167,7 +168,6 @@ import {
 import { IAbortablePromise, IdServerUnbindResult, IImageInfo, Preset, Visibility } from "./@types/partials";
 import { EventMapper, eventMapperFor, MapperOpts } from "./event-mapper";
 import { randomString } from "./randomstring";
-import { WebStorageSessionStore } from "./store/session/webstorage";
 import { BackupManager, IKeyBackup, IKeyBackupCheck, IPreparedKeyBackupVersion, TrustInfo } from "./crypto/backup";
 import { DEFAULT_TREE_POWER_LEVELS_TEMPLATE, MSC3089TreeSpace } from "./models/MSC3089TreeSpace";
 import { ISignatures } from "./@types/signed";
@@ -190,11 +190,12 @@ import { MediaHandler } from "./webrtc/mediaHandler";
 import { IRefreshTokenResponse } from "./@types/auth";
 import { TypedEventEmitter } from "./models/typed-event-emitter";
 import { ReceiptType } from "./@types/read_receipts";
+import { MSC3575SlidingSyncRequest, MSC3575SlidingSyncResponse, SlidingSync } from "./sliding-sync";
+import { SlidingSyncSdk } from "./sliding-sync-sdk";
 import { Thread, THREAD_RELATION_TYPE } from "./models/thread";
 import { MBeaconInfoEventContent, M_BEACON_INFO } from "./@types/beacon";
 
 export type Store = IStore;
-export type SessionStore = WebStorageSessionStore;
 
 export type Callback<T = any> = (err: Error | any | null, data?: T) => void;
 export type ResetTimelineCallback = (roomId: string) => boolean;
@@ -314,21 +315,6 @@ export interface ICreateClientOpts {
      */
     pickleKey?: string;
 
-    /**
-     * A store to be used for end-to-end crypto session data. Most data has been
-     * migrated out of here to `cryptoStore` instead. If not specified,
-     * end-to-end crypto will be disabled. The `createClient` helper
-     * _will not_ create this store at the moment.
-     */
-    sessionStore?: SessionStore;
-
-    /**
-     * Set to true to enable client-side aggregation of event relations
-     * via `EventTimelineSet#getRelationsForEvent`.
-     * This feature is currently unstable and the API may change without notice.
-     */
-    unstableClientRelationAggregation?: boolean;
-
     verificationMethods?: Array<VerificationMethod>;
 
     /**
@@ -427,6 +413,11 @@ export interface IStartClientOpts {
      * @experimental
      */
     experimentalThreadSupport?: boolean;
+
+    /**
+     * @experimental
+     */
+     slidingSync?: SlidingSync;
 }
 
 export interface IStoredClientOpts extends IStartClientOpts {
@@ -439,15 +430,9 @@ export enum RoomVersionStability {
     Unstable = "unstable",
 }
 
-export interface IRoomCapability { // MSC3244
-    preferred: string | null;
-    support: string[];
-}
-
 export interface IRoomVersionsCapability {
     default: string;
     available: Record<string, RoomVersionStability>;
-    "org.matrix.msc3244.room_capabilities"?: Record<string, IRoomCapability>; // MSC3244
 }
 
 export interface ICapability {
@@ -589,13 +574,9 @@ export interface IRequestMsisdnTokenResponse extends IRequestTokenResponse {
     intl_fmt: string;
 }
 
-interface IUploadKeysRequest {
+export interface IUploadKeysRequest {
     device_keys?: Required<IDeviceKeys>;
-    one_time_keys?: {
-        [userId: string]: {
-            [deviceId: string]: number;
-        };
-    };
+    one_time_keys?: Record<string, IOneTimeKey>;
     "org.matrix.msc2732.fallback_keys"?: Record<string, IOneTimeKey>;
 }
 
@@ -633,6 +614,19 @@ interface IJoinedMembersResponse {
     };
 }
 
+export interface IRegisterRequestParams {
+    auth?: IAuthData;
+    username?: string;
+    password?: string;
+    refresh_token?: boolean;
+    guest_access_token?: string;
+    x_show_msisdn?: boolean;
+    bind_msisdn?: boolean;
+    bind_email?: boolean;
+    inhibit_login?: boolean;
+    initial_device_display_name?: string;
+}
+
 export interface IPublicRoomsChunkRoom {
     room_id: string;
     name?: string;
@@ -801,6 +795,7 @@ type RoomEvents = RoomEvent.Name
     | RoomEvent.Receipt
     | RoomEvent.Tags
     | RoomEvent.LocalEchoUpdated
+    | RoomEvent.HistoryImportedWithinTimeline
     | RoomEvent.AccountData
     | RoomEvent.MyMembership
     | RoomEvent.Timeline
@@ -810,6 +805,7 @@ type RoomStateEvents = RoomStateEvent.Events
     | RoomStateEvent.Members
     | RoomStateEvent.NewMember
     | RoomStateEvent.Update
+    | RoomStateEvent.Marker
     ;
 
 type CryptoEvents = CryptoEvent.KeySignatureUploadFailure
@@ -891,9 +887,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     public clientRunning = false;
     public timelineSupport = false;
     public urlPreviewCache: { [key: string]: Promise<IPreviewUrlResponse> } = {};
-    public unstableClientRelationAggregation = false;
     public identityServer: IIdentityServerProvider;
-    public sessionStore: SessionStore; // XXX: Intended private, used in code.
     public http: MatrixHttpApi; // XXX: Intended private, used in code.
     public crypto: Crypto; // XXX: Intended private, used in code.
     public cryptoCallbacks: ICryptoCallbacks; // XXX: Intended private, used in code.
@@ -916,7 +910,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     protected verificationMethods: VerificationMethod[];
     protected fallbackICEServerAllowed = false;
     protected roomList: RoomList;
-    protected syncApi: SyncApi;
+    protected syncApi: SlidingSyncSdk | SyncApi;
     public pushRules: IPushRules;
     protected syncLeftRoomsPromise: Promise<Room[]>;
     protected syncedLeftRooms = false;
@@ -953,6 +947,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         this.baseUrl = opts.baseUrl;
         this.idBaseUrl = opts.idBaseUrl;
+        this.identityServer = opts.identityServer;
 
         this.usingExternalCrypto = opts.usingExternalCrypto;
         this.store = opts.store || new StubStore();
@@ -1023,10 +1018,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         }
 
         this.timelineSupport = Boolean(opts.timelineSupport);
-        this.unstableClientRelationAggregation = !!opts.unstableClientRelationAggregation;
 
         this.cryptoStore = opts.cryptoStore;
-        this.sessionStore = opts.sessionStore;
         this.verificationMethods = opts.verificationMethods;
         this.cryptoCallbacks = opts.cryptoCallbacks || {};
 
@@ -1190,7 +1183,11 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             }
             return this.canResetTimelineCallback(roomId);
         };
-        this.syncApi = new SyncApi(this, this.clientOpts);
+        if (this.clientOpts.slidingSync) {
+            this.syncApi = new SlidingSyncSdk(this.clientOpts.slidingSync, this, this.clientOpts);
+        } else {
+            this.syncApi = new SyncApi(this, this.clientOpts);
+        }
         this.syncApi.sync();
 
         if (this.clientOpts.clientWellKnownPollPeriod !== undefined) {
@@ -1206,6 +1203,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * clean shutdown.
      */
     public stopClient() {
+        this.crypto?.stop(); // crypto might have been initialised even if the client wasn't fully started
+
         if (!this.clientRunning) return; // already stopped
 
         logger.log('stopping MatrixClient');
@@ -1215,7 +1214,6 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         this.syncApi?.stop();
         this.syncApi = null;
 
-        this.crypto?.stop();
         this.peekSync?.stopPeeking();
 
         this.callEventHandler?.stop();
@@ -1411,7 +1409,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      *
      * @return {?string} MXID for the logged-in user, or null if not logged in
      */
-    public getUserId(): string {
+    public getUserId(): string | null {
         if (this.credentials && this.credentials.userId) {
             return this.credentials.userId;
         }
@@ -1433,7 +1431,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * Get the local part of the current user ID e.g. "foo" in "@foo:bar".
      * @return {?string} The user ID localpart or null.
      */
-    public getUserIdLocalpart(): string {
+    public getUserIdLocalpart(): string | null {
         if (this.credentials && this.credentials.userId) {
             return this.credentials.userId.split(":")[0].substring(1);
         }
@@ -1488,7 +1486,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * @param {string} roomId The room the call is to be placed in.
      * @return {MatrixCall} the call or null if the browser doesn't support calling.
      */
-    public createCall(roomId: string): MatrixCall {
+    public createCall(roomId: string): MatrixCall | null {
         return createNewMatrixCall(this, roomId);
     }
 
@@ -1650,10 +1648,6 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             return;
         }
 
-        if (!this.sessionStore) {
-            // this is temporary, the sessionstore is supposed to be going away
-            throw new Error(`Cannot enable encryption: no sessionStore provided`);
-        }
         if (!this.cryptoStore) {
             // the cryptostore is provided by sdk.createClient, so this shouldn't happen
             throw new Error(`Cannot enable encryption: no cryptoStore provided`);
@@ -1682,8 +1676,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         const crypto = new Crypto(
             this,
-            this.sessionStore,
-            userId, this.deviceId,
+            userId,
+            this.deviceId,
             this.store,
             this.cryptoStore,
             this.roomList,
@@ -1800,7 +1794,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      *
      * @return {module:crypto/deviceinfo} device or null
      */
-    public getStoredDevice(userId: string, deviceId: string): DeviceInfo {
+    public getStoredDevice(userId: string, deviceId: string): DeviceInfo | null {
         if (!this.crypto) {
             throw new Error("End-to-end encryption disabled");
         }
@@ -3325,7 +3319,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * @return {?User} A user or null if there is no data store or the user does
      * not exist.
      */
-    public getUser(userId: string): User {
+    public getUser(userId: string): User | null {
         return this.store.getUser(userId);
     }
 
@@ -3759,17 +3753,20 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         // If we expect that an event is part of a thread but is missing the relation
         // we need to add it manually, as well as the reply fallback
         if (threadId && !content["m.relates_to"]?.rel_type) {
+            const isReply = !!content["m.relates_to"]?.["m.in_reply_to"];
             content["m.relates_to"] = {
                 ...content["m.relates_to"],
                 "rel_type": THREAD_RELATION_TYPE.name,
                 "event_id": threadId,
+                // Set is_falling_back to true unless this is actually intended to be a reply
+                "is_falling_back": !isReply,
             };
             const thread = this.getRoom(roomId)?.getThread(threadId);
-            if (thread) {
+            if (thread && !isReply) {
                 content["m.relates_to"]["m.in_reply_to"] = {
                     "event_id": thread.lastReply((ev: MatrixEvent) => {
                         return ev.isRelation(THREAD_RELATION_TYPE.name) && !ev.status;
-                    })?.getId(),
+                    })?.getId() ?? threadId,
                 };
             }
         }
@@ -4018,7 +4015,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             $txnId: txnId,
         };
 
-        let path;
+        let path: string;
 
         if (event.isState()) {
             let pathTemplate = "/rooms/$roomId/state/$eventType";
@@ -4823,8 +4820,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         };
 
         if (
-            this.identityServer &&
-            this.identityServer.getAccessToken &&
+            this.identityServer?.getAccessToken &&
             await this.doesServerAcceptIdentityAccessToken()
         ) {
             const identityAccessToken = await this.identityServer.getAccessToken();
@@ -5238,14 +5234,15 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * <p>If the EventTimelineSet object already has the given event in its store, the
      * corresponding timeline will be returned. Otherwise, a /context request is
      * made, and used to construct an EventTimeline.
+     * If the event does not belong to this EventTimelineSet then undefined will be returned.
      *
-     * @param {EventTimelineSet} timelineSet  The timelineSet to look for the event in
+     * @param {EventTimelineSet} timelineSet  The timelineSet to look for the event in, must be bound to a room
      * @param {string} eventId  The ID of the event to look for
      *
      * @return {Promise} Resolves:
      *    {@link module:models/event-timeline~EventTimeline} including the given event
      */
-    public async getEventTimeline(timelineSet: EventTimelineSet, eventId: string): Promise<EventTimeline> {
+    public async getEventTimeline(timelineSet: EventTimelineSet, eventId: string): Promise<EventTimeline | undefined> {
         // don't allow any timeline support unless it's been enabled.
         if (!this.timelineSupport) {
             throw new Error("timeline support is disabled. Set the 'timelineSupport'" +
@@ -5282,45 +5279,44 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         const mapper = this.getEventMapper();
         const event = mapper(res.event);
         const events = [
-            // we start with the last event, since that's the point at which we have known state.
+            // Order events from most recent to oldest (reverse-chronological).
+            // We start with the last event, since that's the point at which we have known state.
             // events_after is already backwards; events_before is forwards.
             ...res.events_after.reverse().map(mapper),
             event,
             ...res.events_before.map(mapper),
         ];
 
-        // Where the event is a thread reply (not a root) and running in MSC-enabled mode the Thread timeline only
-        // functions contiguously, so we have to jump through some hoops to get our target event in it.
-        // XXX: workaround for https://github.com/vector-im/element-meta/issues/150
-        if (Thread.hasServerSideSupport &&
-            this.supportsExperimentalThreads() &&
-            event.isRelation(THREAD_RELATION_TYPE.name)
-        ) {
-            const [, threadedEvents] = timelineSet.room.partitionThreadedEvents(events);
-            let thread = timelineSet.room.getThread(event.threadRootId);
-            if (!thread) {
-                thread = timelineSet.room.createThread(event.threadRootId, undefined, threadedEvents, true);
+        if (this.supportsExperimentalThreads()) {
+            if (!timelineSet.canContain(event)) {
+                return undefined;
             }
 
-            const opts: IRelationsRequestOpts = {
-                direction: Direction.Backward,
-                limit: 50,
-            };
+            // Where the event is a thread reply (not a root) and running in MSC-enabled mode the Thread timeline only
+            // functions contiguously, so we have to jump through some hoops to get our target event in it.
+            // XXX: workaround for https://github.com/vector-im/element-meta/issues/150
+            if (Thread.hasServerSideSupport && timelineSet.thread) {
+                const thread = timelineSet.thread;
+                const opts: IRelationsRequestOpts = {
+                    direction: Direction.Backward,
+                    limit: 50,
+                };
 
-            await thread.fetchInitialEvents();
-            let nextBatch = thread.liveTimeline.getPaginationToken(Direction.Backward);
+                await thread.fetchInitialEvents();
+                let nextBatch = thread.liveTimeline.getPaginationToken(Direction.Backward);
 
-            // Fetch events until we find the one we were asked for, or we run out of pages
-            while (!thread.findEventById(eventId)) {
-                if (nextBatch) {
-                    opts.from = nextBatch;
+                // Fetch events until we find the one we were asked for, or we run out of pages
+                while (!thread.findEventById(eventId)) {
+                    if (nextBatch) {
+                        opts.from = nextBatch;
+                    }
+
+                    ({ nextBatch } = await thread.fetchEvents(opts));
+                    if (!nextBatch) break;
                 }
 
-                ({ nextBatch } = await thread.fetchEvents(opts));
-                if (!nextBatch) break;
+                return thread.liveTimeline;
             }
-
-            return thread.liveTimeline;
         }
 
         // Here we handle non-thread timelines only, but still process any thread events to populate thread summaries.
@@ -5347,6 +5343,45 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             ?? timeline;
     }
 
+    /**
+     * Get an EventTimeline for the latest events in the room. This will just
+     * call `/messages` to get the latest message in the room, then use
+     * `client.getEventTimeline(...)` to construct a new timeline from it.
+     *
+     * @param {EventTimelineSet} timelineSet  The timelineSet to find or add the timeline to
+     *
+     * @return {Promise} Resolves:
+     *    {@link module:models/event-timeline~EventTimeline} timeline with the latest events in the room
+     */
+    public async getLatestTimeline(timelineSet: EventTimelineSet): Promise<EventTimeline> {
+        // don't allow any timeline support unless it's been enabled.
+        if (!this.timelineSupport) {
+            throw new Error("timeline support is disabled. Set the 'timelineSupport'" +
+                " parameter to true when creating MatrixClient to enable it.");
+        }
+
+        const messagesPath = utils.encodeUri(
+            "/rooms/$roomId/messages", {
+                $roomId: timelineSet.room.roomId,
+            },
+        );
+
+        const params: Record<string, string | string[]> = {
+            dir: 'b',
+        };
+        if (this.clientOpts.lazyLoadMembers) {
+            params.filter = JSON.stringify(Filter.LAZY_LOADING_MESSAGES_FILTER);
+        }
+
+        const res = await this.http.authedRequest<IMessagesResponse>(undefined, Method.Get, messagesPath, params);
+        const event = res.chunk?.[0];
+        if (!event) {
+            throw new Error("No message returned from /messages when trying to construct getLatestTimeline");
+        }
+
+        return this.getEventTimeline(timelineSet, event.event_id);
+    }
+
     /**
      * Makes a request to /messages with the appropriate lazy loading filter set.
      * XXX: if we do get rid of scrollback (as it's not used at the moment),
@@ -5850,6 +5885,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         // There can be only room-kind push rule per room
         // and its id is the room id.
         if (this.pushRules) {
+            if (!this.pushRules[scope] || !this.pushRules[scope].room) {
+                return;
+            }
             for (let i = 0; i < this.pushRules[scope].room.length; i++) {
                 const rule = this.pushRules[scope].room[i];
                 if (rule.rule_id === roomId) {
@@ -6254,9 +6292,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     }
 
     /**
-     * Get the unix timestamp (in seconds) at which the current
+     * Get the unix timestamp (in milliseconds) at which the current
      * TURN credentials (from getTurnServers) expire
-     * @return {number} The expiry timestamp, in seconds, or null if no credentials
+     * @return {number} The expiry timestamp, in milliseconds, or null if no credentials
      */
     public getTurnServersExpiry(): number | null {
         return this.turnServersExpiry;
@@ -6776,7 +6814,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * Get the access token associated with this account.
      * @return {?String} The access_token or null
      */
-    public getAccessToken(): string {
+    public getAccessToken(): string | null {
         return this.http.opts.accessToken || null;
     }
 
@@ -6846,7 +6884,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         guestAccessToken?: string,
         inhibitLogin?: boolean,
         callback?: Callback,
-    ): Promise<any> { // TODO: Types (many)
+    ): Promise<IAuthData> {
         // backwards compat
         if (bindThreepids === true) {
             bindThreepids = { email: true };
@@ -6862,7 +6900,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             auth.session = sessionId;
         }
 
-        const params: any = {
+        const params: IRegisterRequestParams = {
             auth: auth,
             refresh_token: true, // always ask for a refresh token - does nothing if unsupported
         };
@@ -6933,8 +6971,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * @return {Promise} Resolves: to the /register response
      * @return {module:http-api.MatrixError} Rejects: with an error response.
      */
-    public registerRequest(data: any, kind?: string, callback?: Callback): Promise<any> { // TODO: Types
-        const params: any = {};
+    public registerRequest(data: IRegisterRequestParams, kind?: string, callback?: Callback): Promise<IAuthData> {
+        const params: { kind?: string } = {};
         if (kind) {
             params.kind = kind;
         }
@@ -7172,8 +7210,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             .filter(i => !i.id_access_token);
         if (
             invitesNeedingToken.length > 0 &&
-            this.identityServer &&
-            this.identityServer.getAccessToken &&
+            this.identityServer?.getAccessToken &&
             await this.doesServerAcceptIdentityAccessToken()
         ) {
             const identityAccessToken = await this.identityServer.getAccessToken();
@@ -7518,16 +7555,16 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     }
 
     /**
-     * @param {string} roomId
-     * @param {module:client.callback} callback Optional.
+     * Gets the local aliases for the room. Note: this includes all local aliases, unlike the
+     * curated list from the m.room.canonical_alias state event.
+     * @param {string} roomId The room ID to get local aliases for.
      * @return {Promise} Resolves: an object with an `aliases` property, containing an array of local aliases
      * @return {module:http-api.MatrixError} Rejects: with an error response.
      */
-    public unstableGetLocalAliases(roomId: string, callback?: Callback): Promise<{ aliases: string[] }> {
-        const path = utils.encodeUri("/rooms/$roomId/aliases",
-            { $roomId: roomId });
-        const prefix = PREFIX_UNSTABLE + "/org.matrix.msc2432";
-        return this.http.authedRequest(callback, Method.Get, path, null, null, { prefix });
+    public getLocalAliases(roomId: string): Promise<{ aliases: string[] }> {
+        const path = utils.encodeUri("/rooms/$roomId/aliases", { $roomId: roomId });
+        const prefix = PREFIX_V3;
+        return this.http.authedRequest(undefined, Method.Get, path, null, null, { prefix });
     }
 
     /**
@@ -8868,7 +8905,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * @param {string} roomId The room ID to get a tree space reference for.
      * @returns {MSC3089TreeSpace} The tree space, or null if not a tree space.
      */
-    public unstableGetFileTreeSpace(roomId: string): MSC3089TreeSpace {
+    public unstableGetFileTreeSpace(roomId: string): MSC3089TreeSpace | null {
         const room = this.getRoom(roomId);
         if (room?.getMyMembership() !== 'join') return null;
 
@@ -8885,6 +8922,41 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         return new MSC3089TreeSpace(this, roomId);
     }
 
+    /**
+     * Perform a single MSC3575 sliding sync request.
+     * @param {MSC3575SlidingSyncRequest} req The request to make.
+     * @param {string} proxyBaseUrl The base URL for the sliding sync proxy.
+     * @returns {MSC3575SlidingSyncResponse} The sliding sync response, or a standard error.
+     * @throws on non 2xx status codes with an object with a field "httpStatus":number.
+     */
+    public slidingSync(
+        req: MSC3575SlidingSyncRequest, proxyBaseUrl?: string,
+    ): IAbortablePromise<MSC3575SlidingSyncResponse> {
+        const qps: Record<string, any> = {};
+        if (req.pos) {
+            qps.pos = req.pos;
+            delete req.pos;
+        }
+        if (req.timeout) {
+            qps.timeout = req.timeout;
+            delete req.timeout;
+        }
+        const clientTimeout = req.clientTimeout;
+        delete req.clientTimeout;
+        return this.http.authedRequest<MSC3575SlidingSyncResponse>(
+            undefined,
+            Method.Post,
+            "/sync",
+            qps,
+            req,
+            {
+                prefix: "/_matrix/client/unstable/org.matrix.msc3575",
+                baseUrl: proxyBaseUrl,
+                localTimeoutMs: clientTimeout,
+            },
+        );
+    }
+
     /**
      * @experimental
      */
@@ -8914,12 +8986,12 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     }
 
     public processBeaconEvents(
-        room: Room,
+        room?: Room,
         events?: MatrixEvent[],
     ): void {
-        if (!events?.length) {
-            return;
-        }
+        if (!events?.length) return;
+        if (!room) return;
+
         room.currentState.processBeaconEvents(events, this);
     }
 

src/content-helpers.ts

@@ -139,10 +139,10 @@ export const getTextForLocationEvent = (
 /**
  * Generates the content for a Location event
  * @param uri a geo:// uri for the location
- * @param ts the timestamp when the location was correct (milliseconds since
+ * @param timestamp the timestamp when the location was correct (milliseconds since
  *           the UNIX epoch)
  * @param description the (optional) label for this location on the map
- * @param asset_type the (optional) asset type of this location e.g. "m.self"
+ * @param assetType the (optional) asset type of this location e.g. "m.self"
  * @param text optional. A text for the location
  */
 export const makeLocationContent = (

src/crypto/OlmDevice.ts

@@ -92,7 +92,7 @@ export interface InboundGroupSessionData {
     sharedHistory?: boolean;
 }
 
-interface IDecryptedGroupMessage {
+export interface IDecryptedGroupMessage {
     result: string;
     keysClaimed: Record<string, string>;
     senderKey: string;
@@ -100,6 +100,11 @@ interface IDecryptedGroupMessage {
     untrusted: boolean;
 }
 
+export interface IInboundSession {
+    payload: string;
+    session_id: string;
+}
+
 export interface IExportedDevice {
     pickleKey: string;
     pickledAccount: string;
@@ -620,7 +625,7 @@ export class OlmDevice {
         theirDeviceIdentityKey: string,
         messageType: number,
         ciphertext: string,
-    ): Promise<{ payload: string, session_id: string }> { // eslint-disable-line camelcase
+    ): Promise<IInboundSession> {
         if (messageType !== 0) {
             throw new Error("Need messageType == 0 to create inbound session");
         }

src/crypto/algorithms/base.ts

@@ -179,7 +179,7 @@ export abstract class DecryptionAlgorithm {
      *
      * @param {module:models/event.MatrixEvent} params event key event
      */
-    public onRoomKeyEvent(params: MatrixEvent): void {
+    public async onRoomKeyEvent(params: MatrixEvent): Promise<void> {
         // ignore by default
     }
 

src/crypto/algorithms/megolm.ts

@@ -30,7 +30,7 @@ import {
     registerAlgorithm,
     UnknownDeviceError,
 } from "./base";
-import { WITHHELD_MESSAGES } from '../OlmDevice';
+import { IDecryptedGroupMessage, WITHHELD_MESSAGES } from '../OlmDevice';
 import { Room } from '../../models/room';
 import { DeviceInfo } from "../deviceinfo";
 import { IOlmSessionResult } from "../olmlib";
@@ -213,6 +213,8 @@ class OutboundSessionInfo {
                 }
             }
         }
+
+        return false;
     }
 }
 
@@ -231,7 +233,7 @@ class MegolmEncryption extends EncryptionAlgorithm {
     // are using, and which devices we have shared the keys with. It resolves
     // with an OutboundSessionInfo (or undefined, for the first message in the
     // room).
-    private setupPromise = Promise.resolve<OutboundSessionInfo>(undefined);
+    private setupPromise = Promise.resolve<OutboundSessionInfo | null>(null);
 
     // Map of outbound sessions by sessions ID. Used if we need a particular
     // session (the session we're currently using to send is always obtained
@@ -240,8 +242,8 @@ class MegolmEncryption extends EncryptionAlgorithm {
 
     private readonly sessionRotationPeriodMsgs: number;
     private readonly sessionRotationPeriodMs: number;
-    private encryptionPreparation: Promise<void>;
-    private encryptionPreparationMetadata: {
+    private encryptionPreparation?: {
+        promise: Promise<void>;
         startTime: number;
     };
 
@@ -270,193 +272,209 @@ class MegolmEncryption extends EncryptionAlgorithm {
         blocked: IBlockedMap,
         singleOlmCreationPhase = false,
     ): Promise<OutboundSessionInfo> {
-        let session: OutboundSessionInfo;
-
         // takes the previous OutboundSessionInfo, and considers whether to create
         // a new one. Also shares the key with any (new) devices in the room.
-        // Updates `session` to hold the final OutboundSessionInfo.
+        //
+        // Returns the successful session whether keyshare succeeds or not.
         //
         // returns a promise which resolves once the keyshare is successful.
-        const prepareSession = async (oldSession: OutboundSessionInfo) => {
-            session = oldSession;
-
+        const setup = async (oldSession: OutboundSessionInfo | null): Promise<OutboundSessionInfo> => {
             const sharedHistory = isRoomSharedHistory(room);
 
-            // history visibility changed
-            if (session && sharedHistory !== session.sharedHistory) {
-                session = null;
+            const session = await this.prepareSession(devicesInRoom, sharedHistory, oldSession);
+
+            try {
+                await this.shareSession(devicesInRoom, sharedHistory, singleOlmCreationPhase, blocked, session);
+            } catch (e) {
+                logger.error(`Failed to ensure outbound session in ${this.roomId}`, e);
             }
 
-            // need to make a brand new session?
-            if (session && session.needsRotation(this.sessionRotationPeriodMsgs,
-                this.sessionRotationPeriodMs)
-            ) {
-                logger.log("Starting new megolm session because we need to rotate.");
-                session = null;
-            }
-
-            // determine if we have shared with anyone we shouldn't have
-            if (session && session.sharedWithTooManyDevices(devicesInRoom)) {
-                session = null;
-            }
-
-            if (!session) {
-                logger.log(`Starting new megolm session for room ${this.roomId}`);
-                session = await this.prepareNewSession(sharedHistory);
-                logger.log(`Started new megolm session ${session.sessionId} ` +
-                    `for room ${this.roomId}`);
-                this.outboundSessions[session.sessionId] = session;
-            }
-
-            // now check if we need to share with any devices
-            const shareMap: Record<string, DeviceInfo[]> = {};
-
-            for (const [userId, userDevices] of Object.entries(devicesInRoom)) {
-                for (const [deviceId, deviceInfo] of Object.entries(userDevices)) {
-                    const key = deviceInfo.getIdentityKey();
-                    if (key == this.olmDevice.deviceCurve25519Key) {
-                        // don't bother sending to ourself
-                        continue;
-                    }
-
-                    if (
-                        !session.sharedWithDevices[userId] ||
-                        session.sharedWithDevices[userId][deviceId] === undefined
-                    ) {
-                        shareMap[userId] = shareMap[userId] || [];
-                        shareMap[userId].push(deviceInfo);
-                    }
-                }
-            }
-
-            const key = this.olmDevice.getOutboundGroupSessionKey(session.sessionId);
-            const payload: IPayload = {
-                type: "m.room_key",
-                content: {
-                    "algorithm": olmlib.MEGOLM_ALGORITHM,
-                    "room_id": this.roomId,
-                    "session_id": session.sessionId,
-                    "session_key": key.key,
-                    "chain_index": key.chain_index,
-                    "org.matrix.msc3061.shared_history": sharedHistory,
-                },
-            };
-            const [devicesWithoutSession, olmSessions] = await olmlib.getExistingOlmSessions(
-                this.olmDevice, this.baseApis, shareMap,
-            );
-
-            await Promise.all([
-                (async () => {
-                    // share keys with devices that we already have a session for
-                    logger.debug(`Sharing keys with existing Olm sessions in ${this.roomId}`, olmSessions);
-                    await this.shareKeyWithOlmSessions(session, key, payload, olmSessions);
-                    logger.debug(`Shared keys with existing Olm sessions in ${this.roomId}`);
-                })(),
-                (async () => {
-                    logger.debug(
-                        `Sharing keys (start phase 1) with new Olm sessions in ${this.roomId}`,
-                        devicesWithoutSession,
-                    );
-                    const errorDevices: IOlmDevice[] = [];
-
-                    // meanwhile, establish olm sessions for devices that we don't
-                    // already have a session for, and share keys with them.  If
-                    // we're doing two phases of olm session creation, use a
-                    // shorter timeout when fetching one-time keys for the first
-                    // phase.
-                    const start = Date.now();
-                    const failedServers: string[] = [];
-                    await this.shareKeyWithDevices(
-                        session, key, payload, devicesWithoutSession, errorDevices,
-                        singleOlmCreationPhase ? 10000 : 2000, failedServers,
-                    );
-                    logger.debug(`Shared keys (end phase 1) with new Olm sessions in ${this.roomId}`);
-
-                    if (!singleOlmCreationPhase && (Date.now() - start < 10000)) {
-                        // perform the second phase of olm session creation if requested,
-                        // and if the first phase didn't take too long
-                        (async () => {
-                            // Retry sending keys to devices that we were unable to establish
-                            // an olm session for.  This time, we use a longer timeout, but we
-                            // do this in the background and don't block anything else while we
-                            // do this.  We only need to retry users from servers that didn't
-                            // respond the first time.
-                            const retryDevices: Record<string, DeviceInfo[]> = {};
-                            const failedServerMap = new Set;
-                            for (const server of failedServers) {
-                                failedServerMap.add(server);
-                            }
-                            const failedDevices = [];
-                            for (const { userId, deviceInfo } of errorDevices) {
-                                const userHS = userId.slice(userId.indexOf(":") + 1);
-                                if (failedServerMap.has(userHS)) {
-                                    retryDevices[userId] = retryDevices[userId] || [];
-                                    retryDevices[userId].push(deviceInfo);
-                                } else {
-                                    // if we aren't going to retry, then handle it
-                                    // as a failed device
-                                    failedDevices.push({ userId, deviceInfo });
-                                }
-                            }
-
-                            logger.debug(`Sharing keys (start phase 2) with new Olm sessions in ${this.roomId}`);
-                            await this.shareKeyWithDevices(
-                                session, key, payload, retryDevices, failedDevices, 30000,
-                            );
-                            logger.debug(`Shared keys (end phase 2) with new Olm sessions in ${this.roomId}`);
-
-                            await this.notifyFailedOlmDevices(session, key, failedDevices);
-                        })();
-                    } else {
-                        await this.notifyFailedOlmDevices(session, key, errorDevices);
-                    }
-                    logger.debug(`Shared keys (all phases done) with new Olm sessions in ${this.roomId}`);
-                })(),
-                (async () => {
-                    logger.debug(`There are ${Object.entries(blocked).length} blocked devices in ${this.roomId}`,
-                        Object.entries(blocked));
-
-                    // also, notify newly blocked devices that they're blocked
-                    logger.debug(`Notifying newly blocked devices in ${this.roomId}`);
-                    const blockedMap: Record<string, Record<string, { device: IBlockedDevice }>> = {};
-                    let blockedCount = 0;
-                    for (const [userId, userBlockedDevices] of Object.entries(blocked)) {
-                        for (const [deviceId, device] of Object.entries(userBlockedDevices)) {
-                            if (
-                                !session.blockedDevicesNotified[userId] ||
-                                session.blockedDevicesNotified[userId][deviceId] === undefined
-                            ) {
-                                blockedMap[userId] = blockedMap[userId] || {};
-                                blockedMap[userId][deviceId] = { device };
-                                blockedCount++;
-                            }
-                        }
-                    }
-
-                    await this.notifyBlockedDevices(session, blockedMap);
-                    logger.debug(`Notified ${blockedCount} newly blocked devices in ${this.roomId}`, blockedMap);
-                })(),
-            ]);
+            return session;
         };
 
-        // helper which returns the session prepared by prepareSession
-        function returnSession() {
-            return session;
-        }
-
         // first wait for the previous share to complete
-        const prom = this.setupPromise.then(prepareSession);
+        const prom = this.setupPromise.then(setup);
 
         // Ensure any failures are logged for debugging
         prom.catch(e => {
-            logger.error(`Failed to ensure outbound session in ${this.roomId}`, e);
+            logger.error(`Failed to setup outbound session in ${this.roomId}`, e);
         });
 
         // setupPromise resolves to `session` whether or not the share succeeds
-        this.setupPromise = prom.then(returnSession, returnSession);
+        this.setupPromise = prom;
 
         // but we return a promise which only resolves if the share was successful.
-        return prom.then(returnSession);
+        return prom;
+    }
+
+    private async prepareSession(
+        devicesInRoom: DeviceInfoMap,
+        sharedHistory: boolean,
+        session: OutboundSessionInfo | null,
+    ): Promise<OutboundSessionInfo> {
+        // history visibility changed
+        if (session && sharedHistory !== session.sharedHistory) {
+            session = null;
+        }
+
+        // need to make a brand new session?
+        if (session?.needsRotation(this.sessionRotationPeriodMsgs, this.sessionRotationPeriodMs)) {
+            logger.log("Starting new megolm session because we need to rotate.");
+            session = null;
+        }
+
+        // determine if we have shared with anyone we shouldn't have
+        if (session?.sharedWithTooManyDevices(devicesInRoom)) {
+            session = null;
+        }
+
+        if (!session) {
+            logger.log(`Starting new megolm session for room ${this.roomId}`);
+            session = await this.prepareNewSession(sharedHistory);
+            logger.log(`Started new megolm session ${session.sessionId} ` +
+                `for room ${this.roomId}`);
+            this.outboundSessions[session.sessionId] = session;
+        }
+
+        return session;
+    }
+
+    private async shareSession(
+        devicesInRoom: DeviceInfoMap,
+        sharedHistory: boolean,
+        singleOlmCreationPhase: boolean,
+        blocked: IBlockedMap,
+        session: OutboundSessionInfo,
+    ) {
+        // now check if we need to share with any devices
+        const shareMap: Record<string, DeviceInfo[]> = {};
+
+        for (const [userId, userDevices] of Object.entries(devicesInRoom)) {
+            for (const [deviceId, deviceInfo] of Object.entries(userDevices)) {
+                const key = deviceInfo.getIdentityKey();
+                if (key == this.olmDevice.deviceCurve25519Key) {
+                    // don't bother sending to ourself
+                    continue;
+                }
+
+                if (
+                    !session.sharedWithDevices[userId] ||
+                    session.sharedWithDevices[userId][deviceId] === undefined
+                ) {
+                    shareMap[userId] = shareMap[userId] || [];
+                    shareMap[userId].push(deviceInfo);
+                }
+            }
+        }
+
+        const key = this.olmDevice.getOutboundGroupSessionKey(session.sessionId);
+        const payload: IPayload = {
+            type: "m.room_key",
+            content: {
+                "algorithm": olmlib.MEGOLM_ALGORITHM,
+                "room_id": this.roomId,
+                "session_id": session.sessionId,
+                "session_key": key.key,
+                "chain_index": key.chain_index,
+                "org.matrix.msc3061.shared_history": sharedHistory,
+            },
+        };
+        const [devicesWithoutSession, olmSessions] = await olmlib.getExistingOlmSessions(
+            this.olmDevice, this.baseApis, shareMap,
+        );
+
+        await Promise.all([
+            (async () => {
+                // share keys with devices that we already have a session for
+                logger.debug(`Sharing keys with existing Olm sessions in ${this.roomId}`, olmSessions);
+                await this.shareKeyWithOlmSessions(session, key, payload, olmSessions);
+                logger.debug(`Shared keys with existing Olm sessions in ${this.roomId}`);
+            })(),
+            (async () => {
+                logger.debug(
+                    `Sharing keys (start phase 1) with new Olm sessions in ${this.roomId}`,
+                    devicesWithoutSession,
+                );
+                const errorDevices: IOlmDevice[] = [];
+
+                // meanwhile, establish olm sessions for devices that we don't
+                // already have a session for, and share keys with them.  If
+                // we're doing two phases of olm session creation, use a
+                // shorter timeout when fetching one-time keys for the first
+                // phase.
+                const start = Date.now();
+                const failedServers: string[] = [];
+                await this.shareKeyWithDevices(
+                    session, key, payload, devicesWithoutSession, errorDevices,
+                    singleOlmCreationPhase ? 10000 : 2000, failedServers,
+                );
+                logger.debug(`Shared keys (end phase 1) with new Olm sessions in ${this.roomId}`);
+
+                if (!singleOlmCreationPhase && (Date.now() - start < 10000)) {
+                    // perform the second phase of olm session creation if requested,
+                    // and if the first phase didn't take too long
+                    (async () => {
+                        // Retry sending keys to devices that we were unable to establish
+                        // an olm session for.  This time, we use a longer timeout, but we
+                        // do this in the background and don't block anything else while we
+                        // do this.  We only need to retry users from servers that didn't
+                        // respond the first time.
+                        const retryDevices: Record<string, DeviceInfo[]> = {};
+                        const failedServerMap = new Set;
+                        for (const server of failedServers) {
+                            failedServerMap.add(server);
+                        }
+                        const failedDevices: IOlmDevice[] = [];
+                        for (const { userId, deviceInfo } of errorDevices) {
+                            const userHS = userId.slice(userId.indexOf(":") + 1);
+                            if (failedServerMap.has(userHS)) {
+                                retryDevices[userId] = retryDevices[userId] || [];
+                                retryDevices[userId].push(deviceInfo);
+                            } else {
+                                // if we aren't going to retry, then handle it
+                                // as a failed device
+                                failedDevices.push({ userId, deviceInfo });
+                            }
+                        }
+
+                        logger.debug(`Sharing keys (start phase 2) with new Olm sessions in ${this.roomId}`);
+                        await this.shareKeyWithDevices(
+                            session, key, payload, retryDevices, failedDevices, 30000,
+                        );
+                        logger.debug(`Shared keys (end phase 2) with new Olm sessions in ${this.roomId}`);
+
+                        await this.notifyFailedOlmDevices(session, key, failedDevices);
+                    })();
+                } else {
+                    await this.notifyFailedOlmDevices(session, key, errorDevices);
+                }
+                logger.debug(`Shared keys (all phases done) with new Olm sessions in ${this.roomId}`);
+            })(),
+            (async () => {
+                logger.debug(`There are ${Object.entries(blocked).length} blocked devices in ${this.roomId}`,
+                    Object.entries(blocked));
+
+                // also, notify newly blocked devices that they're blocked
+                logger.debug(`Notifying newly blocked devices in ${this.roomId}`);
+                const blockedMap: Record<string, Record<string, { device: IBlockedDevice }>> = {};
+                let blockedCount = 0;
+                for (const [userId, userBlockedDevices] of Object.entries(blocked)) {
+                    for (const [deviceId, device] of Object.entries(userBlockedDevices)) {
+                        if (
+                            !session.blockedDevicesNotified[userId] ||
+                            session.blockedDevicesNotified[userId][deviceId] === undefined
+                        ) {
+                            blockedMap[userId] = blockedMap[userId] || {};
+                            blockedMap[userId][deviceId] = { device };
+                            blockedCount++;
+                        }
+                    }
+                }
+
+                await this.notifyBlockedDevices(session, blockedMap);
+                logger.debug(`Notified ${blockedCount} newly blocked devices in ${this.roomId}`, blockedMap);
+            })(),
+        ]);
     }
 
     /**
@@ -592,7 +610,8 @@ class MegolmEncryption extends EncryptionAlgorithm {
         payload: IPayload,
     ): Promise<void> {
         const contentMap: Record<string, Record<string, IEncryptedContent>> = {};
-        const deviceInfoByDeviceId = new Map<string, DeviceInfo>();
+        // Map from userId to a map of deviceId to deviceInfo
+        const deviceInfoByUserIdAndDeviceId = new Map<string, Map<string, DeviceInfo>>();
 
         const promises: Promise<unknown>[] = [];
         for (let i = 0; i < userDeviceMap.length; i++) {
@@ -605,7 +624,18 @@ class MegolmEncryption extends EncryptionAlgorithm {
             const userId = val.userId;
             const deviceInfo = val.deviceInfo;
             const deviceId = deviceInfo.deviceId;
-            deviceInfoByDeviceId.set(deviceId, deviceInfo);
+
+            // Assign to temp value to make type-checking happy
+            let userIdDeviceInfo = deviceInfoByUserIdAndDeviceId.get(userId);
+
+            if (userIdDeviceInfo === undefined) {
+                userIdDeviceInfo = new Map<string, DeviceInfo>();
+
+                deviceInfoByUserIdAndDeviceId.set(userId, userIdDeviceInfo);
+            }
+
+            // We hold by reference, this updates deviceInfoByUserIdAndDeviceId[userId]
+            userIdDeviceInfo.set(deviceId, deviceInfo);
 
             if (!contentMap[userId]) {
                 contentMap[userId] = {};
@@ -660,7 +690,7 @@ class MegolmEncryption extends EncryptionAlgorithm {
                         session.markSharedWithDevice(
                             userId,
                             deviceId,
-                            deviceInfoByDeviceId.get(deviceId).getIdentityKey(),
+                            deviceInfoByUserIdAndDeviceId.get(userId).get(deviceId).getIdentityKey(),
                             chainIndex,
                         );
                     }
@@ -708,7 +738,6 @@ class MegolmEncryption extends EncryptionAlgorithm {
             contentMap[userId][deviceId] = message;
         }
 
-        await this.baseApis.sendToDevice("org.matrix.room_key.withheld", contentMap);
         await this.baseApis.sendToDevice("m.room_key.withheld", contentMap);
 
         // record the fact that we notified these blocked devices
@@ -854,7 +883,7 @@ class MegolmEncryption extends EncryptionAlgorithm {
         logger.debug(`Ensuring Olm sessions for devices in ${this.roomId}`);
         const devicemap = await olmlib.ensureOlmSessionsForDevices(
             this.olmDevice, this.baseApis, devicesByUser, false, otkTimeout, failedServers,
-            logger.withPrefix(`[${this.roomId}]`),
+            logger.withPrefix?.(`[${this.roomId}]`),
         );
         logger.debug(`Ensured Olm sessions for devices in ${this.roomId}`);
 
@@ -994,11 +1023,11 @@ class MegolmEncryption extends EncryptionAlgorithm {
      * @param {module:models/room} room the room the event is in
      */
     public prepareToEncrypt(room: Room): void {
-        if (this.encryptionPreparation) {
+        if (this.encryptionPreparation != null) {
             // We're already preparing something, so don't do anything else.
             // FIXME: check if we need to restart
             // (https://github.com/matrix-org/matrix-js-sdk/issues/1255)
-            const elapsedTime = Date.now() - this.encryptionPreparationMetadata.startTime;
+            const elapsedTime = Date.now() - this.encryptionPreparation.startTime;
             logger.debug(
                 `Already started preparing to encrypt for ${this.roomId} ` +
                 `${elapsedTime} ms ago, skipping`,
@@ -1008,32 +1037,31 @@ class MegolmEncryption extends EncryptionAlgorithm {
 
         logger.debug(`Preparing to encrypt events for ${this.roomId}`);
 
-        this.encryptionPreparationMetadata = {
+        this.encryptionPreparation = {
             startTime: Date.now(),
-        };
-        this.encryptionPreparation = (async () => {
-            try {
-                logger.debug(`Getting devices in ${this.roomId}`);
-                const [devicesInRoom, blocked] = await this.getDevicesInRoom(room);
+            promise: (async () => {
+                try {
+                    logger.debug(`Getting devices in ${this.roomId}`);
+                    const [devicesInRoom, blocked] = await this.getDevicesInRoom(room);
 
-                if (this.crypto.getGlobalErrorOnUnknownDevices()) {
-                    // Drop unknown devices for now.  When the message gets sent, we'll
-                    // throw an error, but we'll still be prepared to send to the known
-                    // devices.
-                    this.removeUnknownDevices(devicesInRoom);
+                    if (this.crypto.getGlobalErrorOnUnknownDevices()) {
+                        // Drop unknown devices for now.  When the message gets sent, we'll
+                        // throw an error, but we'll still be prepared to send to the known
+                        // devices.
+                        this.removeUnknownDevices(devicesInRoom);
+                    }
+
+                    logger.debug(`Ensuring outbound session in ${this.roomId}`);
+                    await this.ensureOutboundSession(room, devicesInRoom, blocked, true);
+
+                    logger.debug(`Ready to encrypt events for ${this.roomId}`);
+                } catch (e) {
+                    logger.error(`Failed to prepare to encrypt events for ${this.roomId}`, e);
+                } finally {
+                    delete this.encryptionPreparation;
                 }
-
-                logger.debug(`Ensuring outbound session in ${this.roomId}`);
-                await this.ensureOutboundSession(room, devicesInRoom, blocked, true);
-
-                logger.debug(`Ready to encrypt events for ${this.roomId}`);
-            } catch (e) {
-                logger.error(`Failed to prepare to encrypt events for ${this.roomId}`, e);
-            } finally {
-                delete this.encryptionPreparationMetadata;
-                delete this.encryptionPreparation;
-            }
-        })();
+            })(),
+        };
     }
 
     /**
@@ -1048,12 +1076,12 @@ class MegolmEncryption extends EncryptionAlgorithm {
     public async encryptMessage(room: Room, eventType: string, content: object): Promise<object> {
         logger.log(`Starting to encrypt event for ${this.roomId}`);
 
-        if (this.encryptionPreparation) {
+        if (this.encryptionPreparation != null) {
             // If we started sending keys, wait for it to be done.
             // FIXME: check if we need to cancel
             // (https://github.com/matrix-org/matrix-js-sdk/issues/1255)
             try {
-                await this.encryptionPreparation;
+                await this.encryptionPreparation.promise;
             } catch (e) {
                 // ignore any errors -- if the preparation failed, we'll just
                 // restart everything here
@@ -1268,7 +1296,7 @@ class MegolmDecryption extends DecryptionAlgorithm {
         // (fixes https://github.com/vector-im/element-web/issues/5001)
         this.addEventToPendingList(event);
 
-        let res;
+        let res: IDecryptedGroupMessage;
         try {
             res = await this.olmDevice.decryptGroupMessage(
                 event.getRoomId(), content.sender_key, content.session_id, content.ciphertext,
@@ -1298,7 +1326,9 @@ class MegolmDecryption extends DecryptionAlgorithm {
 
         if (res === null) {
             // We've got a message for a session we don't have.
-            //
+            // try and get the missing key from the backup first
+            this.crypto.backupManager.queryKeyBackupRateLimited(event.getRoomId(), content.session_id).catch(() => {});
+
             // (XXX: We might actually have received this key since we started
             // decrypting, in which case we'll have scheduled a retry, and this
             // request will be redundant. We could probably check to see if the
@@ -1391,7 +1421,7 @@ class MegolmDecryption extends DecryptionAlgorithm {
         if (!senderPendingEvents.has(sessionId)) {
             senderPendingEvents.set(sessionId, new Set());
         }
-        senderPendingEvents.get(sessionId).add(event);
+        senderPendingEvents.get(sessionId)?.add(event);
     }
 
     /**
@@ -1425,17 +1455,17 @@ class MegolmDecryption extends DecryptionAlgorithm {
      *
      * @param {module:models/event.MatrixEvent} event key event
      */
-    public onRoomKeyEvent(event: MatrixEvent): Promise<void> {
-        const content = event.getContent();
-        const sessionId = content.session_id;
+    public async onRoomKeyEvent(event: MatrixEvent): Promise<void> {
+        const content = event.getContent<Partial<IMessage["content"]>>();
         let senderKey = event.getSenderKey();
-        let forwardingKeyChain = [];
+        let forwardingKeyChain: string[] = [];
         let exportFormat = false;
-        let keysClaimed;
+        let keysClaimed: ReturnType<MatrixEvent["getKeysClaimed"]>;
 
         if (!content.room_id ||
-            !sessionId ||
-            !content.session_key
+            !content.session_key ||
+            !content.session_id ||
+            !content.algorithm
         ) {
             logger.error("key event is missing fields");
             return;
@@ -1448,20 +1478,18 @@ class MegolmDecryption extends DecryptionAlgorithm {
 
         if (event.getType() == "m.forwarded_room_key") {
             exportFormat = true;
-            forwardingKeyChain = content.forwarding_curve25519_key_chain;
-            if (!Array.isArray(forwardingKeyChain)) {
-                forwardingKeyChain = [];
-            }
+            forwardingKeyChain = Array.isArray(content.forwarding_curve25519_key_chain) ?
+                content.forwarding_curve25519_key_chain : [];
 
             // copy content before we modify it
             forwardingKeyChain = forwardingKeyChain.slice();
             forwardingKeyChain.push(senderKey);
 
-            senderKey = content.sender_key;
-            if (!senderKey) {
+            if (!content.sender_key) {
                 logger.error("forwarded_room_key event is missing sender_key field");
                 return;
             }
+            senderKey = content.sender_key;
 
             const ed25519Key = content.sender_claimed_ed25519_key;
             if (!ed25519Key) {
@@ -1482,34 +1510,39 @@ class MegolmDecryption extends DecryptionAlgorithm {
         if (content["org.matrix.msc3061.shared_history"]) {
             extraSessionData.sharedHistory = true;
         }
-        return this.olmDevice.addInboundGroupSession(
-            content.room_id, senderKey, forwardingKeyChain, sessionId,
-            content.session_key, keysClaimed,
-            exportFormat, extraSessionData,
-        ).then(() => {
+
+        try {
+            await this.olmDevice.addInboundGroupSession(
+                content.room_id,
+                senderKey,
+                forwardingKeyChain,
+                content.session_id,
+                content.session_key,
+                keysClaimed,
+                exportFormat,
+                extraSessionData,
+            );
+
             // have another go at decrypting events sent with this session.
-            this.retryDecryption(senderKey, sessionId)
-                .then((success) => {
-                    // cancel any outstanding room key requests for this session.
-                    // Only do this if we managed to decrypt every message in the
-                    // session, because if we didn't, we leave the other key
-                    // requests in the hopes that someone sends us a key that
-                    // includes an earlier index.
-                    if (success) {
-                        this.crypto.cancelRoomKeyRequest({
-                            algorithm: content.algorithm,
-                            room_id: content.room_id,
-                            session_id: content.session_id,
-                            sender_key: senderKey,
-                        });
-                    }
+            if (await this.retryDecryption(senderKey, content.session_id)) {
+                // cancel any outstanding room key requests for this session.
+                // Only do this if we managed to decrypt every message in the
+                // session, because if we didn't, we leave the other key
+                // requests in the hopes that someone sends us a key that
+                // includes an earlier index.
+                this.crypto.cancelRoomKeyRequest({
+                    algorithm: content.algorithm,
+                    room_id: content.room_id,
+                    session_id: content.session_id,
+                    sender_key: senderKey,
                 });
-        }).then(() => {
+            }
+
             // don't wait for the keys to be backed up for the server
-            this.crypto.backupManager.backupGroupSession(senderKey, content.session_id);
-        }).catch((e) => {
+            await this.crypto.backupManager.backupGroupSession(senderKey, content.session_id);
+        } catch (e) {
             logger.error(`Error handling m.room_key_event: ${e}`);
-        });
+        }
     }
 
     /**
@@ -1702,7 +1735,10 @@ class MegolmDecryption extends DecryptionAlgorithm {
      * @param {boolean} [opts.untrusted] whether the key should be considered as untrusted
      * @param {string} [opts.source] where the key came from
      */
-    public importRoomKey(session: IMegolmSessionData, opts: any = {}): Promise<void> {
+    public importRoomKey(
+        session: IMegolmSessionData,
+        opts: { untrusted?: boolean, source?: string } = {},
+    ): Promise<void> {
         const extraSessionData: any = {};
         if (opts.untrusted || session.untrusted) {
             extraSessionData.untrusted = true;

src/crypto/algorithms/olm.ts

@@ -32,6 +32,7 @@ import {
 import { Room } from '../../models/room';
 import { MatrixEvent } from "../..";
 import { IEventDecryptionResult } from "../index";
+import { IInboundSession } from "../OlmDevice";
 
 const DeviceVerification = DeviceInfo.DeviceVerification;
 
@@ -51,7 +52,7 @@ interface IMessage {
  */
 class OlmEncryption extends EncryptionAlgorithm {
     private sessionPrepared = false;
-    private prepPromise: Promise<void> = null;
+    private prepPromise: Promise<void> | null = null;
 
     /**
      * @private
@@ -116,11 +117,11 @@ class OlmEncryption extends EncryptionAlgorithm {
             ciphertext: {},
         };
 
-        const promises = [];
+        const promises: Promise<void>[] = [];
 
         for (let i = 0; i < users.length; ++i) {
             const userId = users[i];
-            const devices = this.crypto.getStoredDevicesForUser(userId);
+            const devices = this.crypto.getStoredDevicesForUser(userId) || [];
 
             for (let j = 0; j < devices.length; ++j) {
                 const deviceInfo = devices[j];
@@ -239,7 +240,7 @@ class OlmDecryption extends DecryptionAlgorithm {
             throw new DecryptionError(
                 "OLM_BAD_ROOM",
                 "Message intended for room " + payload.room_id, {
-                    reported_room: event.getRoomId(),
+                    reported_room: event.getRoomId() || "ROOM_ID_UNDEFINED",
                 },
             );
         }
@@ -331,7 +332,7 @@ class OlmDecryption extends DecryptionAlgorithm {
         // prekey message which doesn't match any existing sessions: make a new
         // session.
 
-        let res;
+        let res: IInboundSession;
         try {
             res = await this.olmDevice.createInboundSession(
                 theirDeviceIdentityKey, message.type, message.body,

src/crypto/api.ts

@@ -109,11 +109,6 @@ export interface ISecretStorageKeyInfo {
     passphrase: IPassphraseInfo;
 }
 
-export interface ISecretStorageKey {
-    keyId: string;
-    keyInfo: ISecretStorageKeyInfo;
-}
-
 export interface IPassphraseInfo {
     algorithm: "m.pbkdf2";
     iterations: number;

src/crypto/backup.ts

@@ -35,6 +35,7 @@ import { UnstableValue } from "../NamespacedValue";
 import { CryptoEvent, IMegolmSessionData } from "./index";
 
 const KEY_BACKUP_KEYS_PER_REQUEST = 200;
+const KEY_BACKUP_CHECK_RATE_LIMIT = 5000; // ms
 
 type AuthData = IKeyBackupInfo["auth_data"];
 
@@ -111,6 +112,8 @@ export class BackupManager {
     public backupInfo: IKeyBackupInfo | undefined; // The info dict from /room_keys/version
     public checkedForBackup: boolean; // Have we checked the server for a backup we can use?
     private sendingBackups: boolean; // Are we currently sending backups?
+    private sessionLastCheckAttemptedTime: Record<string, number> = {}; // When did we last try to check the server for a given session id?
+
     constructor(private readonly baseApis: MatrixClient, public readonly getKey: GetKey) {
         this.checkedForBackup = false;
         this.sendingBackups = false;
@@ -282,6 +285,26 @@ export class BackupManager {
         return this.checkAndStart();
     }
 
+    /**
+     * Attempts to retrieve a session from a key backup, if enough time
+     * has elapsed since the last check for this session id.
+     */
+    public async queryKeyBackupRateLimited(
+        targetRoomId: string | undefined,
+        targetSessionId: string | undefined,
+    ): Promise<void> {
+        if (!this.backupInfo) { return; }
+
+        const now = new Date().getTime();
+        if (
+            !this.sessionLastCheckAttemptedTime[targetSessionId]
+                || now - this.sessionLastCheckAttemptedTime[targetSessionId] > KEY_BACKUP_CHECK_RATE_LIMIT
+        ) {
+            this.sessionLastCheckAttemptedTime[targetSessionId] = now;
+            await this.baseApis.restoreKeyBackupWithCache(targetRoomId, targetSessionId, this.backupInfo, {});
+        }
+    }
+
     /**
      * Check if the given backup info is trusted.
      *

src/crypto/index.ts

@@ -75,7 +75,6 @@ import {
     ISignedKey,
     IUploadKeySignaturesResponse,
     MatrixClient,
-    SessionStore,
 } from "../client";
 import type { IRoomEncryption, RoomList } from "./RoomList";
 import { IKeyBackupInfo } from "./keybackup";
@@ -121,7 +120,7 @@ interface IInitOpts {
 
 export interface IBootstrapCrossSigningOpts {
     setupNewCrossSigning?: boolean;
-    authUploadDeviceSigningKeys?(makeRequest: (authData: any) => {}): Promise<void>;
+    authUploadDeviceSigningKeys?(makeRequest: (authData: any) => Promise<{}>): Promise<void>;
 }
 
 /* eslint-disable camelcase */
@@ -323,9 +322,6 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
      *
      * @param {MatrixClient} baseApis base matrix api interface
      *
-     * @param {module:store/session/webstorage~WebStorageSessionStore} sessionStore
-     *    Store to be used for end-to-end crypto session data
-     *
      * @param {string} userId The user ID for the local user
      *
      * @param {string} deviceId The identifier for this device.
@@ -343,7 +339,6 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
      */
     constructor(
         public readonly baseApis: MatrixClient,
-        public readonly sessionStore: SessionStore,
         public readonly userId: string,
         private readonly deviceId: string,
         private readonly clientStore: IStore,
@@ -1725,13 +1720,6 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
         logger.info(`Finished device verification upgrade for ${userId}`);
     }
 
-    public async setTrustedBackupPubKey(trustedPubKey: string): Promise<void> {
-        // This should be redundant post cross-signing is a thing, so just
-        // plonk it in localStorage for now.
-        this.sessionStore.setLocalTrustedBackupPubKey(trustedPubKey);
-        await this.backupManager.checkKeyBackup();
-    }
-
     /**
      */
     public enableLazyLoading(): void {
@@ -2590,7 +2578,7 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
         // because it first stores in memory. We should await the promise only
         // after all the in-memory state (roomEncryptors and _roomList) has been updated
         // to avoid races when calling this method multiple times. Hence keep a hold of the promise.
-        let storeConfigPromise = null;
+        let storeConfigPromise: Promise<void> = null;
         if (!existingConfig) {
             storeConfigPromise = this.roomList.setRoomEncryption(roomId, config);
         }
@@ -2871,7 +2859,10 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
      */
     public async decryptEvent(event: MatrixEvent): Promise<IEventDecryptionResult> {
         if (event.isRedacted()) {
-            const redactionEvent = new MatrixEvent(event.getUnsigned().redacted_because);
+            const redactionEvent = new MatrixEvent({
+                room_id: event.getRoomId(),
+                ...event.getUnsigned().redacted_because,
+            });
             const decryptedEvent = await this.decryptEvent(redactionEvent);
 
             return {
@@ -3131,8 +3122,7 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
                 this.secretStorage.onRequestReceived(event);
             } else if (event.getType() === "m.secret.send") {
                 this.secretStorage.onSecretReceived(event);
-            } else if (event.getType() === "m.room_key.withheld"
-                || event.getType() === "org.matrix.room_key.withheld") {
+            } else if (event.getType() === "m.room_key.withheld") {
                 this.onRoomKeyWithheldEvent(event);
             } else if (event.getContent().transaction_id) {
                 this.onKeyVerificationMessage(event);
@@ -3296,7 +3286,7 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
                     event.on(MatrixEventEvent.Status, statusListener);
                 });
             } catch (err) {
-                logger.error("error while waiting for the verification event to be sent: " + err.message);
+                logger.error("error while waiting for the verification event to be sent: ", err);
                 return;
             } finally {
                 event.removeListener(MatrixEventEvent.LocalEventIdReplaced, eventIdListener);
@@ -3320,7 +3310,7 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
         try {
             await request.channel.handleEvent(event, request, isLiveEvent);
         } catch (err) {
-            logger.error("error while handling verification event: " + err.message);
+            logger.error("error while handling verification event", err);
         }
         const shouldEmit = isNewRequest &&
             !request.initiatedByMe &&

src/crypto/keybackup.ts

@@ -15,17 +15,19 @@ limitations under the License.
 */
 
 import { ISigned } from "../@types/signed";
+import { IEncryptedPayload } from "./aes";
+
+export interface Curve25519SessionData {
+    ciphertext: string;
+    ephemeral: string;
+    mac: string;
+}
 
 export interface IKeyBackupSession {
     first_message_index: number; // eslint-disable-line camelcase
     forwarded_count: number; // eslint-disable-line camelcase
     is_verified: boolean; // eslint-disable-line camelcase
-    session_data: { // eslint-disable-line camelcase
-        ciphertext: string;
-        ephemeral: string;
-        mac: string;
-        iv: string;
-    };
+    session_data: Curve25519SessionData | IEncryptedPayload; // eslint-disable-line camelcase
 }
 
 export interface IKeyBackupRoomSessions {

src/crypto/olmlib.ts

@@ -76,7 +76,7 @@ export interface IOlmSessionResult {
 export async function encryptMessageForDevice(
     resultsObject: Record<string, string>,
     ourUserId: string,
-    ourDeviceId: string,
+    ourDeviceId: string | undefined,
     olmDevice: OlmDevice,
     recipientUserId: string,
     recipientDevice: DeviceInfo,
@@ -323,7 +323,7 @@ export async function ensureOlmSessionsForDevices(
     }
 
     const oneTimeKeyAlgorithm = "signed_curve25519";
-    let res;
+    let res: IClaimOTKsResult;
     let taskDetail = `one-time keys for ${devicesWithoutSession.length} devices`;
     try {
         log.debug(`Claiming ${taskDetail}`);

src/crypto/recoverykey.ts

@@ -21,7 +21,7 @@ import * as bs58 from 'bs58';
 const OLM_RECOVERY_KEY_PREFIX = [0x8B, 0x01];
 
 export function encodeRecoveryKey(key: ArrayLike<number>): string {
-    const buf = new Buffer(OLM_RECOVERY_KEY_PREFIX.length + key.length + 1);
+    const buf = Buffer.alloc(OLM_RECOVERY_KEY_PREFIX.length + key.length + 1);
     buf.set(OLM_RECOVERY_KEY_PREFIX, 0);
     buf.set(key, OLM_RECOVERY_KEY_PREFIX.length);
 

src/crypto/store/indexeddb-crypto-store.ts

@@ -157,7 +157,7 @@ export class IndexedDBCryptoStore implements CryptoStore {
             }
         }).then(backend => {
             this.backend = backend;
-            return backend as CryptoStore;
+            return backend;
         });
 
         return this.backendPromise;

src/crypto/verification/Error.ts

@@ -46,13 +46,6 @@ export const newUserCancelledError = errorFactory("m.user", "Cancelled by user")
  */
 export const newTimeoutError = errorFactory("m.timeout", "Timed out");
 
-/**
- * The transaction is unknown.
- */
-export const newUnknownTransactionError = errorFactory(
-    "m.unknown_transaction", "Unknown transaction",
-);
-
 /**
  * An unknown method was selected.
  */
@@ -72,11 +65,6 @@ export const newKeyMismatchError = errorFactory(
     "m.key_mismatch", "Key mismatch",
 );
 
-/**
- * The user does not match.
- */
-export const newUserMismatchError = errorFactory("m.user_error", "User mismatch");
-
 /**
  * An invalid message was sent.
  */

src/crypto/verification/SAS.ts

@@ -180,6 +180,7 @@ function generateSas(sasBytes: number[], methods: string[]): IGeneratedSas {
 
 const macMethods = {
     "hkdf-hmac-sha256": "calculate_mac",
+    "org.matrix.msc3783.hkdf-hmac-sha256": "calculate_mac_fixed_base64",
     "hmac-sha256": "calculate_mac_long_kdf",
 };
 
@@ -221,7 +222,7 @@ const calculateKeyAgreement = {
  */
 const KEY_AGREEMENT_LIST = ["curve25519-hkdf-sha256", "curve25519"];
 const HASHES_LIST = ["sha256"];
-const MAC_LIST = ["hkdf-hmac-sha256", "hmac-sha256"];
+const MAC_LIST = ["org.matrix.msc3783.hkdf-hmac-sha256", "hkdf-hmac-sha256", "hmac-sha256"];
 const SAS_LIST = Object.keys(sasGenerators);
 
 const KEY_AGREEMENT_SET = new Set(KEY_AGREEMENT_LIST);

src/crypto/verification/request/VerificationRequest.ts

@@ -841,11 +841,11 @@ export class VerificationRequest<
         }
 
         const isUnexpectedRequest = type === REQUEST_TYPE && this.phase !== PHASE_UNSENT;
-        const isUnexpectedReady = type === READY_TYPE && this.phase !== PHASE_REQUESTED;
+        const isUnexpectedReady = type === READY_TYPE && this.phase !== PHASE_REQUESTED && this.phase !== PHASE_STARTED;
         // only if phase has passed from PHASE_UNSENT should we cancel, because events
         // are allowed to come in in any order (at least with InRoomChannel). So we only know
-        // we're dealing with a valid request we should participate in once we've moved to PHASE_REQUESTED
-        // before that, we could be looking at somebody elses verification request and we just
+        // we're dealing with a valid request we should participate in once we've moved to PHASE_REQUESTED.
+        // Before that, we could be looking at somebody else's verification request and we just
         // happen to be in the room
         if (this.phase !== PHASE_UNSENT && (isUnexpectedRequest || isUnexpectedReady)) {
             logger.warn(`Cancelling, unexpected ${type} verification ` +

src/http-api.ts

@@ -48,10 +48,15 @@ TODO:
 export const PREFIX_R0 = "/_matrix/client/r0";
 
 /**
- * A constant representing the URI path for release v1 of the Client-Server HTTP API.
+ * A constant representing the URI path for the legacy release v1 of the Client-Server HTTP API.
  */
 export const PREFIX_V1 = "/_matrix/client/v1";
 
+/**
+ * A constant representing the URI path for Client-Server API endpoints versioned at v3.
+ */
+export const PREFIX_V3 = "/_matrix/client/v3";
+
 /**
  * A constant representing the URI path for as-yet unspecified Client-Server HTTP APIs.
  */
@@ -106,6 +111,7 @@ interface IRequest extends _Request {
 
 interface IRequestOpts<T> {
     prefix?: string;
+    baseUrl?: string;
     localTimeoutMs?: number;
     headers?: Record<string, string>;
     json?: boolean; // defaults to true
@@ -403,7 +409,7 @@ export class MatrixHttpApi {
                                 resp = bodyParser(resp);
                             }
                         } catch (err) {
-                            err.http_status = xhr.status;
+                            err.httpStatus = xhr.status;
                             cb(err);
                             return;
                         }
@@ -571,6 +577,9 @@ export class MatrixHttpApi {
      * @param {string=} opts.prefix The full prefix to use e.g.
      * "/_matrix/client/v2_alpha". If not specified, uses this.opts.prefix.
      *
+     * @param {string=} opts.baseUrl The alternative base url to use.
+     * If not specified, uses this.opts.baseUrl
+     *
      * @param {Object=} opts.headers map of additional request headers
      *
      * @return {Promise} Resolves to <code>{data: {Object},
@@ -666,7 +675,8 @@ export class MatrixHttpApi {
         opts?: O,
     ): IAbortablePromise<ResponseType<T, O>> {
         const prefix = opts?.prefix ?? this.opts.prefix;
-        const fullUri = this.opts.baseUrl + prefix + path;
+        const baseUrl = opts?.baseUrl ?? this.opts.baseUrl;
+        const fullUri = baseUrl + prefix + path;
 
         return this.requestOtherUrl<T, O>(callback, method, fullUri, queryParams, data, opts);
     }

src/interactive-auth.ts

@@ -44,11 +44,19 @@ export interface IStageStatus {
 
 export interface IAuthData {
     session?: string;
+    type?: string;
     completed?: string[];
     flows?: IFlow[];
+    available_flows?: IFlow[];
+    stages?: string[];
+    required_stages?: AuthType[];
     params?: Record<string, Record<string, any>>;
+    data?: Record<string, string>;
     errcode?: string;
     error?: string;
+    user_id?: string;
+    device_id?: string;
+    access_token?: string;
 }
 
 export enum AuthType {

src/matrix.ts

@@ -41,7 +41,6 @@ export * from "./interactive-auth";
 export * from "./service-types";
 export * from "./store/memory";
 export * from "./store/indexeddb";
-export * from "./store/session/webstorage";
 export * from "./crypto/store/memory-crypto-store";
 export * from "./crypto/store/indexeddb-crypto-store";
 export * from "./content-repo";

src/models/beacon.ts

@@ -54,8 +54,8 @@ export class Beacon extends TypedEventEmitter<Exclude<BeaconEvent, BeaconEvent.N
     public readonly roomId: string;
     private _beaconInfo: BeaconInfoState;
     private _isLive: boolean;
-    private livenessWatchInterval: ReturnType<typeof setInterval>;
-    private _latestLocationState: BeaconLocationState | undefined;
+    private livenessWatchTimeout: ReturnType<typeof setTimeout>;
+    private _latestLocationEvent: MatrixEvent | undefined;
 
     constructor(
         private rootEvent: MatrixEvent,
@@ -90,7 +90,11 @@ export class Beacon extends TypedEventEmitter<Exclude<BeaconEvent, BeaconEvent.N
     }
 
     public get latestLocationState(): BeaconLocationState | undefined {
-        return this._latestLocationState;
+        return this._latestLocationEvent && parseBeaconContent(this._latestLocationEvent.getContent());
+    }
+
+    public get latestLocationEvent(): MatrixEvent | undefined {
+        return this._latestLocationEvent;
     }
 
     public update(beaconInfoEvent: MatrixEvent): void {
@@ -109,8 +113,8 @@ export class Beacon extends TypedEventEmitter<Exclude<BeaconEvent, BeaconEvent.N
     }
 
     public destroy(): void {
-        if (this.livenessWatchInterval) {
-            clearInterval(this.livenessWatchInterval);
+        if (this.livenessWatchTimeout) {
+            clearTimeout(this.livenessWatchTimeout);
         }
 
         this._isLive = false;
@@ -122,19 +126,26 @@ export class Beacon extends TypedEventEmitter<Exclude<BeaconEvent, BeaconEvent.N
      * Emits BeaconEvent.LivenessChange when beacon expires
      */
     public monitorLiveness(): void {
-        if (this.livenessWatchInterval) {
-            clearInterval(this.livenessWatchInterval);
+        if (this.livenessWatchTimeout) {
+            clearTimeout(this.livenessWatchTimeout);
         }
 
         this.checkLiveness();
         if (this.isLive) {
             const expiryInMs = (this._beaconInfo?.timestamp + this._beaconInfo?.timeout) - Date.now();
             if (expiryInMs > 1) {
-                this.livenessWatchInterval = setInterval(
+                this.livenessWatchTimeout = setTimeout(
                     () => { this.monitorLiveness(); },
                     expiryInMs,
                 );
             }
+        } else if (this._beaconInfo?.timestamp > Date.now()) {
+            // beacon start timestamp is in the future
+            // check liveness again then
+            this.livenessWatchTimeout = setTimeout(
+                () => { this.monitorLiveness(); },
+                this.beaconInfo?.timestamp - Date.now(),
+            );
         }
     }
 
@@ -161,13 +172,13 @@ export class Beacon extends TypedEventEmitter<Exclude<BeaconEvent, BeaconEvent.N
         const latestLocationEvent = validLocationEvents.sort(sortEventsByLatestContentTimestamp)?.[0];
 
         if (latestLocationEvent) {
-            this._latestLocationState = parseBeaconContent(latestLocationEvent.getContent());
+            this._latestLocationEvent = latestLocationEvent;
             this.emit(BeaconEvent.LocationUpdate, this.latestLocationState);
         }
     }
 
     private clearLatestLocation = () => {
-        this._latestLocationState = undefined;
+        this._latestLocationEvent = undefined;
         this.emit(BeaconEvent.LocationUpdate, this.latestLocationState);
     };
 
@@ -178,8 +189,16 @@ export class Beacon extends TypedEventEmitter<Exclude<BeaconEvent, BeaconEvent.N
 
     private checkLiveness(): void {
         const prevLiveness = this.isLive;
+
+        // element web sets a beacon's start timestamp to the senders local current time
+        // when Alice's system clock deviates slightly from Bob's a beacon Alice intended to be live
+        // may have a start timestamp in the future from Bob's POV
+        // handle this by adding 6min of leniency to the start timestamp when it is in the future
+        const startTimestamp = this._beaconInfo?.timestamp > Date.now() ?
+            this._beaconInfo?.timestamp - 360000 /* 6min */ :
+            this._beaconInfo?.timestamp;
         this._isLive = this._beaconInfo?.live &&
-            isTimestampInDuration(this._beaconInfo?.timestamp, this._beaconInfo?.timeout, Date.now());
+            isTimestampInDuration(startTimestamp, this._beaconInfo?.timeout, Date.now());
 
         if (prevLiveness !== this.isLive) {
             this.emit(BeaconEvent.LivenessChange, this.isLive, this);

src/models/event-timeline-set.ts

@@ -18,15 +18,16 @@ limitations under the License.
  * @module models/event-timeline-set
  */
 
-import { EventTimeline } from "./event-timeline";
-import { EventStatus, MatrixEvent, MatrixEventEvent } from "./event";
+import { EventTimeline, IAddEventOptions } from "./event-timeline";
+import { MatrixEvent } from "./event";
 import { logger } from '../logger';
-import { Relations } from './relations';
 import { Room, RoomEvent } from "./room";
 import { Filter } from "../filter";
-import { EventType, RelationType } from "../@types/event";
 import { RoomState } from "./room-state";
 import { TypedEventEmitter } from "./typed-event-emitter";
+import { RelationsContainer } from "./relations-container";
+import { MatrixClient } from "../client";
+import { Thread } from "./thread";
 
 const DEBUG = true;
 
@@ -41,7 +42,6 @@ if (DEBUG) {
 interface IOpts {
     timelineSupport?: boolean;
     filter?: Filter;
-    unstableClientRelationAggregation?: boolean;
     pendingEvents?: boolean;
 }
 
@@ -55,6 +55,23 @@ export interface IRoomTimelineData {
     liveEvent?: boolean;
 }
 
+export interface IAddEventToTimelineOptions
+    extends Pick<IAddEventOptions, 'toStartOfTimeline' | 'roomState' | 'timelineWasEmpty'> {
+    /** Whether the sync response came from cache */
+    fromCache?: boolean;
+}
+
+export interface IAddLiveEventOptions
+    extends Pick<IAddEventToTimelineOptions, 'fromCache' | 'roomState' | 'timelineWasEmpty'> {
+    /** Applies to events in the timeline only. If this is 'replace' then if a
+     * duplicate is encountered, the event passed to this function will replace
+     * the existing event in the timeline. If this is not specified, or is
+     * 'ignore', then the event passed to this function will be ignored
+     * entirely, preserving the existing event in the timeline. Events are
+     * identical based on their event ID <b>only</b>. */
+    duplicateStrategy?: DuplicateStrategy;
+}
+
 type EmittedEvents = RoomEvent.Timeline | RoomEvent.TimelineReset;
 
 export type EventTimelineSetHandlerMap = {
@@ -64,14 +81,13 @@ export type EventTimelineSetHandlerMap = {
 };
 
 export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTimelineSetHandlerMap> {
+    public readonly relations?: RelationsContainer;
     private readonly timelineSupport: boolean;
-    private unstableClientRelationAggregation: boolean;
-    private displayPendingEvents: boolean;
+    private readonly displayPendingEvents: boolean;
     private liveTimeline: EventTimeline;
     private timelines: EventTimeline[];
     private _eventIdToTimeline: Record<string, EventTimeline>;
     private filter?: Filter;
-    private relations: Record<string, Record<string, Record<RelationType, Relations>>>;
 
     /**
      * Construct a set of EventTimeline objects, typically on behalf of a given
@@ -95,7 +111,7 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
      * map from event_id to timeline and index.
      *
      * @constructor
-     * @param {?Room} room
+     * @param {Room=} room
      * Room for this timelineSet. May be null for non-room cases, such as the
      * notification timeline.
      * @param {Object} opts Options inherited from Room.
@@ -104,17 +120,20 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
      * Set to true to enable improved timeline support.
      * @param {Object} [opts.filter = null]
      * The filter object, if any, for this timelineSet.
-     * @param {boolean} [opts.unstableClientRelationAggregation = false]
-     * Optional. Set to true to enable client-side aggregation of event relations
-     * via `getRelationsForEvent`.
-     * This feature is currently unstable and the API may change without notice.
+     * @param {MatrixClient=} client the Matrix client which owns this EventTimelineSet,
+     * can be omitted if room is specified.
+     * @param {Thread=} thread the thread to which this timeline set relates.
      */
-    constructor(public readonly room: Room, opts: IOpts) {
+    constructor(
+        public readonly room: Room | undefined,
+        opts: IOpts = {},
+        client?: MatrixClient,
+        public readonly thread?: Thread,
+    ) {
         super();
 
         this.timelineSupport = Boolean(opts.timelineSupport);
         this.liveTimeline = new EventTimeline(this);
-        this.unstableClientRelationAggregation = !!opts.unstableClientRelationAggregation;
         this.displayPendingEvents = opts.pendingEvents !== false;
 
         // just a list - *not* ordered.
@@ -123,11 +142,7 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
 
         this.filter = opts.filter;
 
-        if (this.unstableClientRelationAggregation) {
-            // A tree of objects to access a set of relations for an event, as in:
-            // this.relations[relatesToEventId][relationType][relationEventType]
-            this.relations = {};
-        }
+        this.relations = this.room?.relations ?? new RelationsContainer(room?.client ?? client);
     }
 
     /**
@@ -180,6 +195,15 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
         return this.liveTimeline;
     }
 
+    /**
+     * Set the live timeline for this room.
+     *
+     * @return {module:models/event-timeline~EventTimeline} live timeline
+     */
+    public setLiveTimeline(timeline: EventTimeline): void {
+        this.liveTimeline = timeline;
+    }
+
     /**
      * Return the timeline (if any) this event is in.
      * @param {String} eventId the eventId being sought
@@ -430,7 +454,9 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
 
             if (!existingTimeline) {
                 // we don't know about this event yet. Just add it to the timeline.
-                this.addEventToTimeline(event, timeline, toStartOfTimeline);
+                this.addEventToTimeline(event, timeline, {
+                    toStartOfTimeline,
+                });
                 lastEventWasNew = true;
                 didUpdate = true;
                 continue;
@@ -522,16 +548,52 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
      * Add an event to the end of this live timeline.
      *
      * @param {MatrixEvent} event Event to be added
-     * @param {string?} duplicateStrategy 'ignore' or 'replace'
-     * @param {boolean} fromCache whether the sync response came from cache
-     * @param roomState the state events to reconcile metadata from
+     * @param {IAddLiveEventOptions} options addLiveEvent options
      */
     public addLiveEvent(
         event: MatrixEvent,
-        duplicateStrategy: DuplicateStrategy = DuplicateStrategy.Ignore,
+        {
+            duplicateStrategy,
+            fromCache,
+            roomState,
+            timelineWasEmpty,
+        }: IAddLiveEventOptions,
+    ): void;
+    /**
+     * @deprecated In favor of the overload with `IAddLiveEventOptions`
+     */
+    public addLiveEvent(
+        event: MatrixEvent,
+        duplicateStrategy?: DuplicateStrategy,
+        fromCache?: boolean,
+        roomState?: RoomState,
+    ): void;
+    public addLiveEvent(
+        event: MatrixEvent,
+        duplicateStrategyOrOpts?: DuplicateStrategy | IAddLiveEventOptions,
         fromCache = false,
         roomState?: RoomState,
     ): void {
+        let duplicateStrategy = duplicateStrategyOrOpts as DuplicateStrategy || DuplicateStrategy.Ignore;
+        let timelineWasEmpty: boolean;
+        if (typeof (duplicateStrategyOrOpts) === 'object') {
+            ({
+                duplicateStrategy = DuplicateStrategy.Ignore,
+                fromCache = false,
+                roomState,
+                timelineWasEmpty,
+            } = duplicateStrategyOrOpts);
+        } else if (duplicateStrategyOrOpts !== undefined) {
+            // Deprecation warning
+            // FIXME: Remove after 2023-06-01 (technical debt)
+            logger.warn(
+                'Overload deprecated: ' +
+                '`EventTimelineSet.addLiveEvent(event, duplicateStrategy?, fromCache?, roomState?)` ' +
+                'is deprecated in favor of the overload with ' +
+                '`EventTimelineSet.addLiveEvent(event, IAddLiveEventOptions)`',
+            );
+        }
+
         if (this.filter) {
             const events = this.filter.filterRoomTimeline([event]);
             if (!events.length) {
@@ -542,8 +604,7 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
         const timeline = this._eventIdToTimeline[event.getId()];
         if (timeline) {
             if (duplicateStrategy === DuplicateStrategy.Replace) {
-                debuglog("EventTimelineSet.addLiveEvent: replacing duplicate event " +
-                    event.getId());
+                debuglog("EventTimelineSet.addLiveEvent: replacing duplicate event " + event.getId());
                 const tlEvents = timeline.getEvents();
                 for (let j = 0; j < tlEvents.length; j++) {
                     if (tlEvents[j].getId() === event.getId()) {
@@ -563,13 +624,17 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
                     }
                 }
             } else {
-                debuglog("EventTimelineSet.addLiveEvent: ignoring duplicate event " +
-                    event.getId());
+                debuglog("EventTimelineSet.addLiveEvent: ignoring duplicate event " + event.getId());
             }
             return;
         }
 
-        this.addEventToTimeline(event, this.liveTimeline, false, fromCache, roomState);
+        this.addEventToTimeline(event, this.liveTimeline, {
+            toStartOfTimeline: false,
+            fromCache,
+            roomState,
+            timelineWasEmpty,
+        });
     }
 
     /**
@@ -580,24 +645,62 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
      *
      * @param {MatrixEvent} event
      * @param {EventTimeline} timeline
-     * @param {boolean} toStartOfTimeline
-     * @param {boolean} fromCache whether the sync response came from cache
+     * @param {IAddEventToTimelineOptions} options addEventToTimeline options
      *
      * @fires module:client~MatrixClient#event:"Room.timeline"
      */
+    public addEventToTimeline(
+        event: MatrixEvent,
+        timeline: EventTimeline,
+        {
+            toStartOfTimeline,
+            fromCache,
+            roomState,
+            timelineWasEmpty,
+        }: IAddEventToTimelineOptions,
+    ): void;
+    /**
+     * @deprecated In favor of the overload with `IAddEventToTimelineOptions`
+     */
     public addEventToTimeline(
         event: MatrixEvent,
         timeline: EventTimeline,
         toStartOfTimeline: boolean,
+        fromCache?: boolean,
+        roomState?: RoomState,
+    ): void;
+    public addEventToTimeline(
+        event: MatrixEvent,
+        timeline: EventTimeline,
+        toStartOfTimelineOrOpts: boolean | IAddEventToTimelineOptions,
         fromCache = false,
         roomState?: RoomState,
-    ) {
+    ): void {
+        let toStartOfTimeline = !!toStartOfTimelineOrOpts;
+        let timelineWasEmpty: boolean;
+        if (typeof (toStartOfTimelineOrOpts) === 'object') {
+            ({ toStartOfTimeline, fromCache = false, roomState, timelineWasEmpty } = toStartOfTimelineOrOpts);
+        } else if (toStartOfTimelineOrOpts !== undefined) {
+            // Deprecation warning
+            // FIXME: Remove after 2023-06-01 (technical debt)
+            logger.warn(
+                'Overload deprecated: ' +
+                '`EventTimelineSet.addEventToTimeline(event, timeline, toStartOfTimeline, fromCache?, roomState?)` ' +
+                'is deprecated in favor of the overload with ' +
+                '`EventTimelineSet.addEventToTimeline(event, timeline, IAddEventToTimelineOptions)`',
+            );
+        }
+
         const eventId = event.getId();
-        timeline.addEvent(event, toStartOfTimeline, roomState);
+        timeline.addEvent(event, {
+            toStartOfTimeline,
+            roomState,
+            timelineWasEmpty,
+        });
         this._eventIdToTimeline[eventId] = timeline;
 
-        this.setRelationsTarget(event);
-        this.aggregateRelations(event);
+        this.relations.aggregateParentEvent(event);
+        this.relations.aggregateChildEvent(event, this);
 
         const data: IRoomTimelineData = {
             timeline: timeline,
@@ -629,10 +732,14 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
         } else {
             if (this.filter) {
                 if (this.filter.filterRoomTimeline([localEvent]).length) {
-                    this.addEventToTimeline(localEvent, this.liveTimeline, false);
+                    this.addEventToTimeline(localEvent, this.liveTimeline, {
+                        toStartOfTimeline: false,
+                    });
                 }
             } else {
-                this.addEventToTimeline(localEvent, this.liveTimeline, false);
+                this.addEventToTimeline(localEvent, this.liveTimeline, {
+                    toStartOfTimeline: false,
+                });
             }
         }
     }
@@ -693,8 +800,8 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
         if (timeline1 === timeline2) {
             // both events are in the same timeline - figure out their
             // relative indices
-            let idx1;
-            let idx2;
+            let idx1: number;
+            let idx2: number;
             const events = timeline1.getEvents();
             for (let idx = 0; idx < events.length &&
             (idx1 === undefined || idx2 === undefined); idx++) {
@@ -737,133 +844,28 @@ export class EventTimelineSet extends TypedEventEmitter<EmittedEvents, EventTime
     }
 
     /**
-     * Get a collection of relations to a given event in this timeline set.
+     * Determine whether a given event can sanely be added to this event timeline set,
+     * for timeline sets relating to a thread, only return true for events in the same
+     * thread timeline, for timeline sets not relating to a thread only return true
+     * for events which should be shown in the main room timeline.
+     * Requires the `room` property to have been set at EventTimelineSet construction time.
      *
-     * @param {String} eventId
-     * The ID of the event that you'd like to access relation events for.
-     * For example, with annotations, this would be the ID of the event being annotated.
-     * @param {String} relationType
-     * The type of relation involved, such as "m.annotation", "m.reference", "m.replace", etc.
-     * @param {String} eventType
-     * The relation event's type, such as "m.reaction", etc.
-     * @throws If <code>eventId</code>, <code>relationType</code> or <code>eventType</code>
-     * are not valid.
-     *
-     * @returns {?Relations}
-     * A container for relation events or undefined if there are no relation events for
-     * the relationType.
+     * @param event {MatrixEvent} the event to check whether it belongs to this timeline set.
+     * @throws {Error} if `room` was not set when constructing this timeline set.
+     * @return {boolean} whether the event belongs to this timeline set.
      */
-    public getRelationsForEvent(
-        eventId: string,
-        relationType: RelationType | string,
-        eventType: EventType | string,
-    ): Relations | undefined {
-        if (!this.unstableClientRelationAggregation) {
-            throw new Error("Client-side relation aggregation is disabled");
+    public canContain(event: MatrixEvent): boolean {
+        if (!this.room) {
+            throw new Error("Cannot call `EventTimelineSet::canContain without a `room` set. " +
+                "Set the room when creating the EventTimelineSet to call this method.");
         }
 
-        if (!eventId || !relationType || !eventType) {
-            throw new Error("Invalid arguments for `getRelationsForEvent`");
+        const { threadId, shouldLiveInRoom } = this.room.eventShouldLiveIn(event);
+
+        if (this.thread) {
+            return this.thread.id === threadId;
         }
-
-        // debuglog("Getting relations for: ", eventId, relationType, eventType);
-
-        const relationsForEvent = this.relations[eventId] || {};
-        const relationsWithRelType = relationsForEvent[relationType] || {};
-        return relationsWithRelType[eventType];
-    }
-
-    public getAllRelationsEventForEvent(eventId: string): MatrixEvent[] {
-        const relationsForEvent = this.relations?.[eventId] || {};
-        const events = [];
-        for (const relationsRecord of Object.values(relationsForEvent)) {
-            for (const relations of Object.values(relationsRecord)) {
-                events.push(...relations.getRelations());
-            }
-        }
-        return events;
-    }
-
-    /**
-     * Set an event as the target event if any Relations exist for it already
-     *
-     * @param {MatrixEvent} event
-     * The event to check as relation target.
-     */
-    public setRelationsTarget(event: MatrixEvent): void {
-        if (!this.unstableClientRelationAggregation) {
-            return;
-        }
-
-        const relationsForEvent = this.relations[event.getId()];
-        if (!relationsForEvent) {
-            return;
-        }
-
-        for (const relationsWithRelType of Object.values(relationsForEvent)) {
-            for (const relationsWithEventType of Object.values(relationsWithRelType)) {
-                relationsWithEventType.setTargetEvent(event);
-            }
-        }
-    }
-
-    /**
-     * Add relation events to the relevant relation collection.
-     *
-     * @param {MatrixEvent} event
-     * The new relation event to be aggregated.
-     */
-    public aggregateRelations(event: MatrixEvent): void {
-        if (!this.unstableClientRelationAggregation) {
-            return;
-        }
-
-        if (event.isRedacted() || event.status === EventStatus.CANCELLED) {
-            return;
-        }
-
-        // If the event is currently encrypted, wait until it has been decrypted.
-        if (event.isBeingDecrypted() || event.shouldAttemptDecryption()) {
-            event.once(MatrixEventEvent.Decrypted, () => {
-                this.aggregateRelations(event);
-            });
-            return;
-        }
-
-        const relation = event.getRelation();
-        if (!relation) {
-            return;
-        }
-
-        const relatesToEventId = relation.event_id;
-        const relationType = relation.rel_type;
-        const eventType = event.getType();
-
-        // debuglog("Aggregating relation: ", event.getId(), eventType, relation);
-
-        let relationsForEvent: Record<string, Partial<Record<string, Relations>>> = this.relations[relatesToEventId];
-        if (!relationsForEvent) {
-            relationsForEvent = this.relations[relatesToEventId] = {};
-        }
-        let relationsWithRelType = relationsForEvent[relationType];
-        if (!relationsWithRelType) {
-            relationsWithRelType = relationsForEvent[relationType] = {};
-        }
-        let relationsWithEventType = relationsWithRelType[eventType];
-
-        if (!relationsWithEventType) {
-            relationsWithEventType = relationsWithRelType[eventType] = new Relations(
-                relationType,
-                eventType,
-                this.room,
-            );
-            const relatesToEvent = this.findEventById(relatesToEventId) || this.room.getPendingEvent(relatesToEventId);
-            if (relatesToEvent) {
-                relationsWithEventType.setTargetEvent(relatesToEvent);
-            }
-        }
-
-        relationsWithEventType.addEvent(event);
+        return shouldLiveInRoom;
     }
 }
 

src/models/event-timeline.ts

@@ -18,12 +18,30 @@ limitations under the License.
  * @module models/event-timeline
  */
 
-import { RoomState } from "./room-state";
+import { logger } from '../logger';
+import { RoomState, IMarkerFoundOptions } from "./room-state";
 import { EventTimelineSet } from "./event-timeline-set";
 import { MatrixEvent } from "./event";
 import { Filter } from "../filter";
 import { EventType } from "../@types/event";
 
+export interface IInitialiseStateOptions extends Pick<IMarkerFoundOptions, 'timelineWasEmpty'> {
+    // This is a separate interface without any extra stuff currently added on
+    // top of `IMarkerFoundOptions` just because it feels like they have
+    // different concerns. One shouldn't necessarily look to add to
+    // `IMarkerFoundOptions` just because they want to add an extra option to
+    // `initialiseState`.
+}
+
+export interface IAddEventOptions extends Pick<IMarkerFoundOptions, 'timelineWasEmpty'> {
+    /** Whether to insert the new event at the start of the timeline where the
+     * oldest events are (timeline is in chronological order, oldest to most
+     * recent) */
+    toStartOfTimeline: boolean;
+    /** The state events to reconcile metadata from */
+    roomState?: RoomState;
+}
+
 export enum Direction {
     Backward = "b",
     Forward = "f",
@@ -131,7 +149,7 @@ export class EventTimeline {
      * state with.
      * @throws {Error} if an attempt is made to call this after addEvent is called.
      */
-    public initialiseState(stateEvents: MatrixEvent[]): void {
+    public initialiseState(stateEvents: MatrixEvent[], { timelineWasEmpty }: IInitialiseStateOptions = {}): void {
         if (this.events.length > 0) {
             throw new Error("Cannot initialise state after events are added");
         }
@@ -152,8 +170,12 @@ export class EventTimeline {
             Object.freeze(e);
         }
 
-        this.startState.setStateEvents(stateEvents);
-        this.endState.setStateEvents(stateEvents);
+        this.startState.setStateEvents(stateEvents, {
+            timelineWasEmpty,
+        });
+        this.endState.setStateEvents(stateEvents, {
+            timelineWasEmpty,
+        });
     }
 
     /**
@@ -345,24 +367,60 @@ export class EventTimeline {
      * Add a new event to the timeline, and update the state
      *
      * @param {MatrixEvent} event   new event
-     * @param {boolean}  atStart     true to insert new event at the start
+     * @param {IAddEventOptions} options addEvent options
      */
-    public addEvent(event: MatrixEvent, atStart: boolean, stateContext?: RoomState): void {
-        if (!stateContext) {
-            stateContext = atStart ? this.startState : this.endState;
+    public addEvent(
+        event: MatrixEvent,
+        {
+            toStartOfTimeline,
+            roomState,
+            timelineWasEmpty,
+        }: IAddEventOptions,
+    ): void;
+    /**
+     * @deprecated In favor of the overload with `IAddEventOptions`
+     */
+    public addEvent(
+        event: MatrixEvent,
+        toStartOfTimeline: boolean,
+        roomState?: RoomState
+    ): void;
+    public addEvent(
+        event: MatrixEvent,
+        toStartOfTimelineOrOpts: boolean | IAddEventOptions,
+        roomState?: RoomState,
+    ): void {
+        let toStartOfTimeline = !!toStartOfTimelineOrOpts;
+        let timelineWasEmpty: boolean;
+        if (typeof (toStartOfTimelineOrOpts) === 'object') {
+            ({ toStartOfTimeline, roomState, timelineWasEmpty } = toStartOfTimelineOrOpts);
+        } else if (toStartOfTimelineOrOpts !== undefined) {
+            // Deprecation warning
+            // FIXME: Remove after 2023-06-01 (technical debt)
+            logger.warn(
+                'Overload deprecated: ' +
+                '`EventTimeline.addEvent(event, toStartOfTimeline, roomState?)` ' +
+                'is deprecated in favor of the overload with `EventTimeline.addEvent(event, IAddEventOptions)`',
+            );
+        }
+
+        if (!roomState) {
+            roomState = toStartOfTimeline ? this.startState : this.endState;
         }
 
         const timelineSet = this.getTimelineSet();
 
         if (timelineSet.room) {
-            EventTimeline.setEventMetadata(event, stateContext, atStart);
+            EventTimeline.setEventMetadata(event, roomState, toStartOfTimeline);
 
             // modify state but only on unfiltered timelineSets
             if (
                 event.isState() &&
                 timelineSet.room.getUnfilteredTimelineSet() === timelineSet
             ) {
-                stateContext.setStateEvents([event]);
+                roomState.setStateEvents([event], {
+                    timelineWasEmpty,
+                });
                 // it is possible that the act of setting the state event means we
                 // can set more metadata (specifically sender/target props), so try
                 // it again if the prop wasn't previously set. It may also mean that
@@ -373,22 +431,22 @@ export class EventTimeline {
                 // back in time, else we'll set the .sender value for BEFORE the given
                 // member event, whereas we want to set the .sender value for the ACTUAL
                 // member event itself.
-                if (!event.sender || (event.getType() === "m.room.member" && !atStart)) {
-                    EventTimeline.setEventMetadata(event, stateContext, atStart);
+                if (!event.sender || (event.getType() === "m.room.member" && !toStartOfTimeline)) {
+                    EventTimeline.setEventMetadata(event, roomState, toStartOfTimeline);
                 }
             }
         }
 
-        let insertIndex;
+        let insertIndex: number;
 
-        if (atStart) {
+        if (toStartOfTimeline) {
             insertIndex = 0;
         } else {
             insertIndex = this.events.length;
         }
 
         this.events.splice(insertIndex, 0, event); // insert element
-        if (atStart) {
+        if (toStartOfTimeline) {
             this.baseIndex++;
         }
     }

src/models/event.ts

@@ -514,13 +514,6 @@ export class MatrixEvent extends TypedEventEmitter<EmittedEvents, MatrixEventHan
         }
     }
 
-    /**
-     * @experimental
-     */
-    public get isThreadRelation(): boolean {
-        return !!this.threadRootId && this.threadId !== this.getId();
-    }
-
     /**
      * @experimental
      */
@@ -544,7 +537,7 @@ export class MatrixEvent extends TypedEventEmitter<EmittedEvents, MatrixEventHan
         return mRelatesTo?.['m.in_reply_to']?.event_id;
     }
 
-    public get relationEventId(): string {
+    public get relationEventId(): string | undefined {
         return this.getWireContent()
             ?.["m.relates_to"]
             ?.event_id;
@@ -830,18 +823,13 @@ export class MatrixEvent extends TypedEventEmitter<EmittedEvents, MatrixEventHan
                 //
                 if (this.retryDecryption) {
                     // decryption error, but we have a retry queued.
-                    logger.log(
-                        `Got error decrypting event (id=${this.getId()}: ` +
-                        `${e}), but retrying`,
-                    );
+                    logger.log(`Got error decrypting event (id=${this.getId()}: ${e.detailedString}), but retrying`, e);
                     continue;
                 }
 
                 // decryption error, no retries queued. Warn about the error and
                 // set it to m.bad.encrypted.
-                logger.warn(
-                    `Error decrypting event (id=${this.getId()}): ${e.detailedString}`,
-                );
+                logger.warn(`Got error decrypting event (id=${this.getId()}: ${e.detailedString})`, e);
 
                 res = this.badEncryptedMessage(e.message);
             }
@@ -1290,7 +1278,7 @@ export class MatrixEvent extends TypedEventEmitter<EmittedEvents, MatrixEventHan
 
     /**
      * Get whether the event is a relation event, and of a given type if
-     * `relType` is passed in.
+     * `relType` is passed in. State events cannot be relation events
      *
      * @param {string?} relType if given, checks that the relation is of the
      * given type
@@ -1300,8 +1288,11 @@ export class MatrixEvent extends TypedEventEmitter<EmittedEvents, MatrixEventHan
         // Relation info is lifted out of the encrypted content when sent to
         // encrypted rooms, so we have to check `getWireContent` for this.
         const relation = this.getWireContent()?.["m.relates_to"];
-        return relation && relation.rel_type && relation.event_id &&
-            ((relType && relation.rel_type === relType) || !relType);
+        if (this.isState() && relation?.rel_type === RelationType.Replace) {
+            // State events cannot be m.replace relations
+            return false;
+        }
+        return relation?.rel_type && relation.event_id && (relType ? relation.rel_type === relType : true);
     }
 
     /**

src/models/relations-container.ts

@@ -0,0 +1,155 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { Relations } from "./relations";
+import { EventType, RelationType } from "../@types/event";
+import { EventStatus, MatrixEvent, MatrixEventEvent } from "./event";
+import { EventTimelineSet } from "./event-timeline-set";
+import { MatrixClient } from "../client";
+import { Room } from "./room";
+
+export class RelationsContainer {
+    // A tree of objects to access a set of related children for an event, as in:
+    // this.relations[parentEventId][relationType][relationEventType]
+    private relations: {
+        [parentEventId: string]: {
+            [relationType: RelationType | string]: {
+                [eventType: EventType | string]: Relations;
+            };
+        };
+    } = {};
+
+    constructor(private readonly client: MatrixClient, private readonly room?: Room) {
+    }
+
+    /**
+     * Get a collection of child events to a given event in this timeline set.
+     *
+     * @param {String} eventId
+     * The ID of the event that you'd like to access child events for.
+     * For example, with annotations, this would be the ID of the event being annotated.
+     * @param {String} relationType
+     * The type of relationship involved, such as "m.annotation", "m.reference", "m.replace", etc.
+     * @param {String} eventType
+     * The relation event's type, such as "m.reaction", etc.
+     * @throws If <code>eventId</code>, <code>relationType</code> or <code>eventType</code>
+     * are not valid.
+     *
+     * @returns {?Relations}
+     * A container for relation events or undefined if there are no relation events for
+     * the relationType.
+     */
+    public getChildEventsForEvent(
+        eventId: string,
+        relationType: RelationType | string,
+        eventType: EventType | string,
+    ): Relations | undefined {
+        return this.relations[eventId]?.[relationType]?.[eventType];
+    }
+
+    public getAllChildEventsForEvent(parentEventId: string): MatrixEvent[] {
+        const relationsForEvent = this.relations[parentEventId] ?? {};
+        const events: MatrixEvent[] = [];
+        for (const relationsRecord of Object.values(relationsForEvent)) {
+            for (const relations of Object.values(relationsRecord)) {
+                events.push(...relations.getRelations());
+            }
+        }
+        return events;
+    }
+
+    /**
+     * Set an event as the target event if any Relations exist for it already.
+     * Child events can point to other child events as their parent, so this method may be
+     * called for events which are also logically child events.
+     *
+     * @param {MatrixEvent} event The event to check as relation target.
+     */
+    public aggregateParentEvent(event: MatrixEvent): void {
+        const relationsForEvent = this.relations[event.getId()];
+        if (!relationsForEvent) return;
+
+        for (const relationsWithRelType of Object.values(relationsForEvent)) {
+            for (const relationsWithEventType of Object.values(relationsWithRelType)) {
+                relationsWithEventType.setTargetEvent(event);
+            }
+        }
+    }
+
+    /**
+     * Add relation events to the relevant relation collection.
+     *
+     * @param {MatrixEvent} event The new child event to be aggregated.
+     * @param {EventTimelineSet} timelineSet The event timeline set within which to search for the related event if any.
+     */
+    public aggregateChildEvent(event: MatrixEvent, timelineSet?: EventTimelineSet): void {
+        if (event.isRedacted() || event.status === EventStatus.CANCELLED) {
+            return;
+        }
+
+        const relation = event.getRelation();
+        if (!relation) return;
+
+        const onEventDecrypted = () => {
+            if (event.isDecryptionFailure()) {
+                // This could for example happen if the encryption keys are not yet available.
+                // The event may still be decrypted later. Register the listener again.
+                event.once(MatrixEventEvent.Decrypted, onEventDecrypted);
+                return;
+            }
+
+            this.aggregateChildEvent(event, timelineSet);
+        };
+
+        // If the event is currently encrypted, wait until it has been decrypted.
+        if (event.isBeingDecrypted() || event.shouldAttemptDecryption()) {
+            event.once(MatrixEventEvent.Decrypted, onEventDecrypted);
+            return;
+        }
+
+        const { event_id: relatesToEventId, rel_type: relationType } = relation;
+        const eventType = event.getType();
+
+        let relationsForEvent = this.relations[relatesToEventId];
+        if (!relationsForEvent) {
+            relationsForEvent = this.relations[relatesToEventId] = {};
+        }
+
+        let relationsWithRelType = relationsForEvent[relationType];
+        if (!relationsWithRelType) {
+            relationsWithRelType = relationsForEvent[relationType] = {};
+        }
+
+        let relationsWithEventType = relationsWithRelType[eventType];
+        if (!relationsWithEventType) {
+            relationsWithEventType = relationsWithRelType[eventType] = new Relations(
+                relationType,
+                eventType,
+                this.client,
+            );
+
+            const room = this.room ?? timelineSet?.room;
+            const relatesToEvent = timelineSet?.findEventById(relatesToEventId)
+                ?? room?.findEventById(relatesToEventId)
+                ?? room?.getPendingEvent(relatesToEventId);
+            if (relatesToEvent) {
+                relationsWithEventType.setTargetEvent(relatesToEvent);
+            }
+        }
+
+        relationsWithEventType.addEvent(event);
+    }
+}

src/models/relations.ts

@@ -15,10 +15,11 @@ limitations under the License.
 */
 
 import { EventStatus, IAggregatedRelation, MatrixEvent, MatrixEventEvent } from './event';
-import { Room } from './room';
 import { logger } from '../logger';
 import { RelationType } from "../@types/event";
 import { TypedEventEmitter } from "./typed-event-emitter";
+import { MatrixClient } from "../client";
+import { Room } from "./room";
 
 export enum RelationsEvent {
     Add = "Relations.add",
@@ -48,6 +49,7 @@ export class Relations extends TypedEventEmitter<RelationsEvent, EventHandlerMap
     private sortedAnnotationsByKey: [string, Set<MatrixEvent>][] = [];
     private targetEvent: MatrixEvent = null;
     private creationEmitted = false;
+    private readonly client: MatrixClient;
 
     /**
      * @param {RelationType} relationType
@@ -55,16 +57,16 @@ export class Relations extends TypedEventEmitter<RelationsEvent, EventHandlerMap
      * "m.replace", etc.
      * @param {String} eventType
      * The relation event's type, such as "m.reaction", etc.
-     * @param {?Room} room
-     * Room for this container. May be null for non-room cases, such as the
-     * notification timeline.
+     * @param {MatrixClient|Room} client
+     * The client which created this instance. For backwards compatibility also accepts a Room.
      */
     constructor(
         public readonly relationType: RelationType | string,
         public readonly eventType: string,
-        private readonly room: Room,
+        client: MatrixClient | Room,
     ) {
         super();
+        this.client = client instanceof Room ? client.client : client;
     }
 
     /**
@@ -103,7 +105,7 @@ export class Relations extends TypedEventEmitter<RelationsEvent, EventHandlerMap
 
         if (this.relationType === RelationType.Annotation) {
             this.addAnnotationToAggregation(event);
-        } else if (this.relationType === RelationType.Replace && this.targetEvent) {
+        } else if (this.relationType === RelationType.Replace && this.targetEvent && !this.targetEvent.isState()) {
             const lastReplacement = await this.getLastReplacement();
             this.targetEvent.makeReplaced(lastReplacement);
         }
@@ -144,7 +146,7 @@ export class Relations extends TypedEventEmitter<RelationsEvent, EventHandlerMap
 
         if (this.relationType === RelationType.Annotation) {
             this.removeAnnotationFromAggregation(event);
-        } else if (this.relationType === RelationType.Replace && this.targetEvent) {
+        } else if (this.relationType === RelationType.Replace && this.targetEvent && !this.targetEvent.isState()) {
             const lastReplacement = await this.getLastReplacement();
             this.targetEvent.makeReplaced(lastReplacement);
         }
@@ -261,7 +263,7 @@ export class Relations extends TypedEventEmitter<RelationsEvent, EventHandlerMap
         if (this.relationType === RelationType.Annotation) {
             // Remove the redacted annotation from aggregation by key
             this.removeAnnotationFromAggregation(redactedEvent);
-        } else if (this.relationType === RelationType.Replace && this.targetEvent) {
+        } else if (this.relationType === RelationType.Replace && this.targetEvent && !this.targetEvent.isState()) {
             const lastReplacement = await this.getLastReplacement();
             this.targetEvent.makeReplaced(lastReplacement);
         }
@@ -347,7 +349,7 @@ export class Relations extends TypedEventEmitter<RelationsEvent, EventHandlerMap
         }, null);
 
         if (lastReplacement?.shouldAttemptDecryption()) {
-            await lastReplacement.attemptDecryption(this.room.client.crypto);
+            await lastReplacement.attemptDecryption(this.client.crypto);
         } else if (lastReplacement?.isBeingDecrypted()) {
             await lastReplacement.getDecryptionPromise();
         }
@@ -364,7 +366,7 @@ export class Relations extends TypedEventEmitter<RelationsEvent, EventHandlerMap
         }
         this.targetEvent = event;
 
-        if (this.relationType === RelationType.Replace) {
+        if (this.relationType === RelationType.Replace && !this.targetEvent.isState()) {
             const replacement = await this.getLastReplacement();
             // this is the initial update, so only call it if we already have something
             // to not emit Event.replaced needlessly

src/models/room-state.ts

@@ -21,7 +21,7 @@ limitations under the License.
 import { RoomMember } from "./room-member";
 import { logger } from '../logger';
 import * as utils from "../utils";
-import { EventType } from "../@types/event";
+import { EventType, UNSTABLE_MSC2716_MARKER } from "../@types/event";
 import { MatrixEvent, MatrixEventEvent } from "./event";
 import { MatrixClient } from "../client";
 import { GuestAccess, HistoryVisibility, IJoinRuleEventContent, JoinRule } from "../@types/partials";
@@ -30,6 +30,22 @@ import { Beacon, BeaconEvent, BeaconEventHandlerMap, getBeaconInfoIdentifier, Be
 import { TypedReEmitter } from "../ReEmitter";
 import { M_BEACON, M_BEACON_INFO } from "../@types/beacon";
 
+export interface IMarkerFoundOptions {
+    /** Whether the timeline was empty before the marker event arrived in the
+     *  room. This could be happen in a variety of cases:
+     *  1. From the initial sync
+     *  2. It's the first state we're seeing after joining the room
+     *  3. Or whether it's coming from `syncFromCache`
+     *
+     * A marker event refers to `UNSTABLE_MSC2716_MARKER` and indicates that
+     * history was imported somewhere back in time. It specifically points to an
+     * MSC2716 insertion event where the history was imported at. Marker events
+     * are sent as state events so they are easily discoverable by clients and
+     * homeservers and don't get lost in timeline gaps.
+     */
+    timelineWasEmpty?: boolean;
+}
+
 // possible statuses for out-of-band member loading
 enum OobStatus {
     NotStarted,
@@ -43,6 +59,7 @@ export enum RoomStateEvent {
     NewMember = "RoomState.newMember",
     Update = "RoomState.update", // signals batches of updates without specificity
     BeaconLiveness = "RoomState.BeaconLiveness",
+    Marker = "RoomState.Marker",
 }
 
 export type RoomStateEventHandlerMap = {
@@ -51,6 +68,7 @@ export type RoomStateEventHandlerMap = {
     [RoomStateEvent.NewMember]: (event: MatrixEvent, state: RoomState, member: RoomMember) => void;
     [RoomStateEvent.Update]: (state: RoomState) => void;
     [RoomStateEvent.BeaconLiveness]: (state: RoomState, hasLiveBeacons: boolean) => void;
+    [RoomStateEvent.Marker]: (event: MatrixEvent, setStateOptions: IMarkerFoundOptions) => void;
     [BeaconEvent.New]: (event: MatrixEvent, beacon: Beacon) => void;
 };
 
@@ -314,16 +332,19 @@ export class RoomState extends TypedEventEmitter<EmittedEvents, EventHandlerMap>
     }
 
     /**
-     * Add an array of one or more state MatrixEvents, overwriting
-     * any existing state with the same {type, stateKey} tuple. Will fire
-     * "RoomState.events" for every event added. May fire "RoomState.members"
-     * if there are <code>m.room.member</code> events.
+     * Add an array of one or more state MatrixEvents, overwriting any existing
+     * state with the same {type, stateKey} tuple. Will fire "RoomState.events"
+     * for every event added. May fire "RoomState.members" if there are
+     * <code>m.room.member</code> events. May fire "RoomStateEvent.Marker" if there are
+     * <code>UNSTABLE_MSC2716_MARKER</code> events.
      * @param {MatrixEvent[]} stateEvents a list of state events for this room.
+     * @param {IMarkerFoundOptions} markerFoundOptions
      * @fires module:client~MatrixClient#event:"RoomState.members"
      * @fires module:client~MatrixClient#event:"RoomState.newMember"
      * @fires module:client~MatrixClient#event:"RoomState.events"
+     * @fires module:client~MatrixClient#event:"RoomStateEvent.Marker"
      */
-    public setStateEvents(stateEvents: MatrixEvent[]) {
+    public setStateEvents(stateEvents: MatrixEvent[], markerFoundOptions?: IMarkerFoundOptions) {
         this.updateModifiedTime();
 
         // update the core event dict
@@ -349,7 +370,6 @@ export class RoomState extends TypedEventEmitter<EmittedEvents, EventHandlerMap>
         });
 
         this.onBeaconLivenessChange();
-
         // update higher level data structures. This needs to be done AFTER the
         // core event dict as these structures may depend on other state events in
         // the given array (e.g. disambiguating display names in one go to do both
@@ -380,7 +400,6 @@ export class RoomState extends TypedEventEmitter<EmittedEvents, EventHandlerMap>
 
                 const member = this.getOrCreateMember(userId, event);
                 member.setMembershipEvent(event, this);
-
                 this.updateMember(member);
                 this.emit(RoomStateEvent.Members, event, this, member);
             } else if (event.getType() === EventType.RoomPowerLevels) {
@@ -403,6 +422,8 @@ export class RoomState extends TypedEventEmitter<EmittedEvents, EventHandlerMap>
 
                 // assume all our sentinels are now out-of-date
                 this.sentinels = {};
+            } else if (UNSTABLE_MSC2716_MARKER.matches(event.getType())) {
+                this.emit(RoomStateEvent.Marker, event, markerFoundOptions);
             }
         });
 

src/models/room.ts

@@ -18,7 +18,7 @@ limitations under the License.
  * @module models/room
  */
 
-import { EventTimelineSet, DuplicateStrategy } from "./event-timeline-set";
+import { EventTimelineSet, DuplicateStrategy, IAddLiveEventOptions } from "./event-timeline-set";
 import { Direction, EventTimeline } from "./event-timeline";
 import { getHttpUriForMxc } from "../content-repo";
 import * as utils from "../utils";
@@ -49,6 +49,7 @@ import {
 import { TypedEventEmitter } from "./typed-event-emitter";
 import { ReceiptType } from "../@types/read_receipts";
 import { IStateEventWithRoomId } from "../@types/search";
+import { RelationsContainer } from "./relations-container";
 
 // These constants are used as sane defaults when the homeserver doesn't support
 // the m.room_versions capability. In practice, KNOWN_SAFE_ROOM_VERSION should be
@@ -56,7 +57,7 @@ import { IStateEventWithRoomId } from "../@types/search";
 // room versions which are considered okay for people to run without being asked
 // to upgrade (ie: "stable"). Eventually, we should remove these when all homeservers
 // return an m.room_versions capability.
-const KNOWN_SAFE_ROOM_VERSION = '9';
+export const KNOWN_SAFE_ROOM_VERSION = '9';
 const SAFE_ROOM_VERSIONS = ['1', '2', '3', '4', '5', '6', '7', '8', '9'];
 
 function synthesizeReceipt(userId: string, event: MatrixEvent, receiptType: ReceiptType): MatrixEvent {
@@ -80,7 +81,6 @@ interface IOpts {
     storageToken?: string;
     pendingEventOrdering?: PendingEventOrdering;
     timelineSupport?: boolean;
-    unstableClientRelationAggregation?: boolean;
     lazyLoadMembers?: boolean;
 }
 
@@ -165,6 +165,10 @@ export enum RoomEvent {
     LocalEchoUpdated = "Room.localEchoUpdated",
     Timeline = "Room.timeline",
     TimelineReset = "Room.timelineReset",
+    TimelineRefresh = "Room.TimelineRefresh",
+    OldStateUpdated = "Room.OldStateUpdated",
+    CurrentStateUpdated = "Room.CurrentStateUpdated",
+    HistoryImportedWithinTimeline = "Room.historyImportedWithinTimeline",
 }
 
 type EmittedEvents = RoomEvent
@@ -173,6 +177,10 @@ type EmittedEvents = RoomEvent
     | ThreadEvent.NewReply
     | RoomEvent.Timeline
     | RoomEvent.TimelineReset
+    | RoomEvent.TimelineRefresh
+    | RoomEvent.HistoryImportedWithinTimeline
+    | RoomEvent.OldStateUpdated
+    | RoomEvent.CurrentStateUpdated
     | MatrixEventEvent.BeforeRedaction;
 
 export type RoomEventHandlerMap = {
@@ -189,6 +197,13 @@ export type RoomEventHandlerMap = {
         oldEventId?: string,
         oldStatus?: EventStatus,
     ) => void;
+    [RoomEvent.OldStateUpdated]: (room: Room, previousRoomState: RoomState, roomState: RoomState) => void;
+    [RoomEvent.CurrentStateUpdated]: (room: Room, previousRoomState: RoomState, roomState: RoomState) => void;
+    [RoomEvent.HistoryImportedWithinTimeline]: (
+        markerEvent: MatrixEvent,
+        room: Room,
+    ) => void;
+    [RoomEvent.TimelineRefresh]: (room: Room, eventTimelineSet: EventTimelineSet) => void;
     [ThreadEvent.New]: (thread: Thread, toStartOfTimeline: boolean) => void;
 } & ThreadHandlerMap & MatrixEventHandlerMap;
 
@@ -206,6 +221,7 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
     public readonly threadsTimelineSets: EventTimelineSet[] = [];
     // any filtered timeline sets we're maintaining for this room
     private readonly filteredTimelineSets: Record<string, EventTimelineSet> = {}; // filter_id: timelineSet
+    private timelineNeedsRefresh = false;
     private readonly pendingEventList?: MatrixEvent[];
     // read by megolm via getter; boolean value - null indicates "use global value"
     private blacklistUnverifiedDevices: boolean = null;
@@ -261,6 +277,7 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
      * prefer getLiveTimeline().getState(EventTimeline.FORWARDS).
      */
     public currentState: RoomState;
+    public readonly relations = new RelationsContainer(this.client, this);
 
     /**
      * @experimental
@@ -322,10 +339,6 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
      * "chronological".
      * @param {boolean} [opts.timelineSupport = false] Set to true to enable improved
      * timeline support.
-     * @param {boolean} [opts.unstableClientRelationAggregation = false]
-     * Optional. Set to true to enable client-side aggregation of event relations
-     * via `EventTimelineSet#getRelationsForEvent`.
-     * This feature is currently unstable and the API may change without notice.
      */
     constructor(
         public readonly roomId: string,
@@ -355,18 +368,16 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
 
         if (this.opts.pendingEventOrdering === PendingEventOrdering.Detached) {
             this.pendingEventList = [];
-            const serializedPendingEventList = client.sessionStore.store.getItem(pendingEventsKey(this.roomId));
-            if (serializedPendingEventList) {
-                JSON.parse(serializedPendingEventList)
-                    .forEach(async (serializedEvent: Partial<IEvent>) => {
-                        const event = new MatrixEvent(serializedEvent);
-                        if (event.getType() === EventType.RoomMessageEncrypted) {
-                            await event.attemptDecryption(this.client.crypto);
-                        }
-                        event.setStatus(EventStatus.NOT_SENT);
-                        this.addPendingEvent(event, event.getTxnId());
-                    });
-            }
+            this.client.store.getPendingEvents(this.roomId).then(events => {
+                events.forEach(async (serializedEvent: Partial<IEvent>) => {
+                    const event = new MatrixEvent(serializedEvent);
+                    if (event.getType() === EventType.RoomMessageEncrypted) {
+                        await event.attemptDecryption(this.client.crypto);
+                    }
+                    event.setStatus(EventStatus.NOT_SENT);
+                    this.addPendingEvent(event, event.getTxnId());
+                });
+            });
         }
 
         // awaited by getEncryptionTargetMembers while room members are loading
@@ -441,6 +452,15 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
         return Promise.allSettled(decryptionPromises) as unknown as Promise<void>;
     }
 
+    /**
+     * Gets the creator of the room
+     * @returns {string} The creator of the room, or null if it could not be determined
+     */
+    public getCreator(): string | null {
+        const createEvent = this.currentState.getStateEvents(EventType.RoomCreate, "");
+        return createEvent?.getContent()['creator'] ?? null;
+    }
+
     /**
      * Gets the version of the room
      * @returns {string} The version of the room, or null if it could not be determined
@@ -897,6 +917,108 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
         });
     }
 
+    /**
+     * Empty out the current live timeline and re-request it. This is used when
+     * historical messages are imported into the room via MSC2716 `/batch_send
+     * because the client may already have that section of the timeline loaded.
+     * We need to force the client to throw away their current timeline so that
+     * when they back paginate over the area again with the historical messages
+     * in between, it grabs the newly imported messages. We can listen for
+     * `UNSTABLE_MSC2716_MARKER`, in order to tell when historical messages are ready
+     * to be discovered in the room and the timeline needs a refresh. The SDK
+     * emits a `RoomEvent.HistoryImportedWithinTimeline` event when we detect a
+     * valid marker and can check the needs refresh status via
+     * `room.getTimelineNeedsRefresh()`.
+     */
+    public async refreshLiveTimeline(): Promise<void> {
+        const liveTimelineBefore = this.getLiveTimeline();
+        const forwardPaginationToken = liveTimelineBefore.getPaginationToken(EventTimeline.FORWARDS);
+        const backwardPaginationToken = liveTimelineBefore.getPaginationToken(EventTimeline.BACKWARDS);
+        const eventsBefore = liveTimelineBefore.getEvents();
+        const mostRecentEventInTimeline = eventsBefore[eventsBefore.length - 1];
+        logger.log(
+            `[refreshLiveTimeline for ${this.roomId}] at ` +
+            `mostRecentEventInTimeline=${mostRecentEventInTimeline && mostRecentEventInTimeline.getId()} ` +
+            `liveTimelineBefore=${liveTimelineBefore.toString()} ` +
+            `forwardPaginationToken=${forwardPaginationToken} ` +
+            `backwardPaginationToken=${backwardPaginationToken}`,
+        );
+
+        // Get the main TimelineSet
+        const timelineSet = this.getUnfilteredTimelineSet();
+
+        let newTimeline: EventTimeline;
+        // If there isn't any event in the timeline, let's go fetch the latest
+        // event and construct a timeline from it.
+        //
+        // This should only really happen if the user ran into an error
+        // with refreshing the timeline before which left them in a blank
+        // timeline from `resetLiveTimeline`.
+        if (!mostRecentEventInTimeline) {
+            newTimeline = await this.client.getLatestTimeline(timelineSet);
+        } else {
+            // Empty out all of `this.timelineSets`. But we also need to keep the
+            // same `timelineSet` references around so the React code updates
+            // properly and doesn't ignore the room events we emit because it checks
+            // that the `timelineSet` references are the same. We need the
+            // `timelineSet` empty so that the `client.getEventTimeline(...)` call
+            // later, will call `/context` and create a new timeline instead of
+            // returning the same one.
+            this.resetLiveTimeline(null, null);
+
+            // Make the UI timeline show the new blank live timeline we just
+            // reset so that if the network fails below it's showing the
+            // accurate state of what we're working with instead of the
+            // disconnected one in the TimelineWindow which is just hanging
+            // around by reference.
+            this.emit(RoomEvent.TimelineRefresh, this, timelineSet);
+
+            // Use `client.getEventTimeline(...)` to construct a new timeline from a
+            // `/context` response state and events for the most recent event before
+            // we reset everything. The `timelineSet` we pass in needs to be empty
+            // in order for this function to call `/context` and generate a new
+            // timeline.
+            newTimeline = await this.client.getEventTimeline(timelineSet, mostRecentEventInTimeline.getId());
+        }
+
+        // If a racing `/sync` beat us to creating a new timeline, use that
+        // instead because it's the latest in the room and any new messages in
+        // the scrollback will include the history.
+        const liveTimeline = timelineSet.getLiveTimeline();
+        if (!liveTimeline || (
+            liveTimeline.getPaginationToken(Direction.Forward) === null &&
+            liveTimeline.getPaginationToken(Direction.Backward) === null &&
+            liveTimeline.getEvents().length === 0
+        )) {
+            logger.log(`[refreshLiveTimeline for ${this.roomId}] using our new live timeline`);
+            // Set the pagination token back to the live sync token (`null`) instead
+            // of using the `/context` historical token (ex. `t12-13_0_0_0_0_0_0_0_0`)
+            // so that it matches the next response from `/sync` and we can properly
+            // continue the timeline.
+            newTimeline.setPaginationToken(forwardPaginationToken, EventTimeline.FORWARDS);
+
+            // Set our new fresh timeline as the live timeline to continue syncing
+            // forwards and back paginating from.
+            timelineSet.setLiveTimeline(newTimeline);
+            // Fixup `this.oldstate` so that `scrollback` has the pagination tokens
+            // available
+            this.fixUpLegacyTimelineFields();
+        } else {
+            logger.log(
+                `[refreshLiveTimeline for ${this.roomId}] \`/sync\` or some other request beat us to creating a new ` +
+                `live timeline after we reset it. We'll use that instead since any events in the scrollback from ` +
+                `this timeline will include the history.`,
+            );
+        }
+
+        // The timeline has now been refreshed ✅
+        this.setTimelineNeedsRefresh(false);
+
+        // Emit an event which clients can react to and re-load the timeline
+        // from the SDK
+        this.emit(RoomEvent.TimelineRefresh, this, timelineSet);
+    }
+
     /**
      * Reset the live timeline of all timelineSets, and start new ones.
      *
@@ -924,6 +1046,9 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
      * @private
      */
     private fixUpLegacyTimelineFields(): void {
+        const previousOldState = this.oldState;
+        const previousCurrentState = this.currentState;
+
         // maintain this.timeline as a reference to the live timeline,
         // and this.oldState and this.currentState as references to the
         // state at the start and end of that timeline. These are more
@@ -933,6 +1058,17 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
             .getState(EventTimeline.BACKWARDS);
         this.currentState = this.getLiveTimeline()
             .getState(EventTimeline.FORWARDS);
+
+        // Let people know to register new listeners for the new state
+        // references. The reference won't necessarily change every time so only
+        // emit when we see a change.
+        if (previousOldState !== this.oldState) {
+            this.emit(RoomEvent.OldStateUpdated, this, previousOldState, this.oldState);
+        }
+
+        if (previousCurrentState !== this.currentState) {
+            this.emit(RoomEvent.CurrentStateUpdated, this, previousCurrentState, this.currentState);
+        }
     }
 
     /**
@@ -1000,6 +1136,24 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
         return this.getUnfilteredTimelineSet().addTimeline();
     }
 
+    /**
+     * Whether the timeline needs to be refreshed in order to pull in new
+     * historical messages that were imported.
+     * @param {Boolean} value The value to set
+     */
+    public setTimelineNeedsRefresh(value: boolean): void {
+        this.timelineNeedsRefresh = value;
+    }
+
+    /**
+     * Whether the timeline needs to be refreshed in order to pull in new
+     * historical messages that were imported.
+     * @return {Boolean} .
+     */
+    public getTimelineNeedsRefresh(): boolean {
+        return this.timelineNeedsRefresh;
+    }
+
     /**
      * Get an event which is stored in our unfiltered timeline set, or in a thread
      *
@@ -1454,7 +1608,9 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
                         return event.getSender() === this.client.getUserId();
                     });
                     if (filterType !== ThreadFilterType.My || currentUserParticipated) {
-                        timelineSet.getLiveTimeline().addEvent(thread.rootEvent, false);
+                        timelineSet.getLiveTimeline().addEvent(thread.rootEvent, {
+                            toStartOfTimeline: false,
+                        });
                     }
                 });
         }
@@ -1501,22 +1657,20 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
         let latestMyThreadsRootEvent: MatrixEvent;
         const roomState = this.getLiveTimeline().getState(EventTimeline.FORWARDS);
         for (const rootEvent of threadRoots) {
-            this.threadsTimelineSets[0].addLiveEvent(
-                rootEvent,
-                DuplicateStrategy.Ignore,
-                false,
+            this.threadsTimelineSets[0].addLiveEvent(rootEvent, {
+                duplicateStrategy: DuplicateStrategy.Ignore,
+                fromCache: false,
                 roomState,
-            );
+            });
 
             const threadRelationship = rootEvent
                 .getServerAggregatedRelation<IThreadBundledRelationship>(RelationType.Thread);
             if (threadRelationship.current_user_participated) {
-                this.threadsTimelineSets[1].addLiveEvent(
-                    rootEvent,
-                    DuplicateStrategy.Ignore,
-                    false,
+                this.threadsTimelineSets[1].addLiveEvent(rootEvent, {
+                    duplicateStrategy: DuplicateStrategy.Ignore,
+                    fromCache: false,
                     roomState,
-                );
+                });
                 latestMyThreadsRootEvent = rootEvent;
             }
 
@@ -1578,7 +1732,7 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
         }
 
         // A thread relation is always only shown in a thread
-        if (event.isThreadRelation) {
+        if (event.isRelation(THREAD_RELATION_TYPE.name)) {
             return {
                 shouldLiveInRoom: false,
                 shouldLiveInThread: true,
@@ -1657,8 +1811,7 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
         toStartOfTimeline: boolean,
     ): Thread {
         if (rootEvent) {
-            const tl = this.getTimelineForEvent(rootEvent.getId());
-            const relatedEvents = tl?.getTimelineSet().getAllRelationsEventForEvent(rootEvent.getId());
+            const relatedEvents = this.relations.getAllChildEventsForEvent(rootEvent.getId());
             if (relatedEvents?.length) {
                 // Include all relations of the root event, given it'll be visible in both timelines,
                 // except `m.replace` as that will already be applied atop the event using `MatrixEvent::makeReplaced`
@@ -1764,6 +1917,27 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
         // If any pending visibility change is waiting for this (older) event,
         this.applyPendingVisibilityEvents(event);
 
+        // Sliding Sync modifications:
+        // The proxy cannot guarantee every sent event will have a transaction_id field, so we need
+        // to check the event ID against the list of pending events if there is no transaction ID
+        // field. Only do this for events sent by us though as it's potentially expensive to loop
+        // the pending events map.
+        const txnId = event.getUnsigned().transaction_id;
+        if (!txnId && event.getSender() === this.myUserId) {
+            // check the txn map for a matching event ID
+            for (const tid in this.txnToEvent) {
+                const localEvent = this.txnToEvent[tid];
+                if (localEvent.getId() === event.getId()) {
+                    logger.debug("processLiveEvent: found sent event without txn ID: ", tid, event.getId());
+                    // update the unsigned field so we can re-use the same codepaths
+                    const unsigned = event.getUnsigned();
+                    unsigned.transaction_id = tid;
+                    event.setUnsigned(unsigned);
+                    break;
+                }
+            }
+        }
+
         if (event.getUnsigned().transaction_id) {
             const existingEvent = this.txnToEvent[event.getUnsigned().transaction_id];
             if (existingEvent) {
@@ -1778,15 +1952,20 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
      * "Room.timeline".
      *
      * @param {MatrixEvent} event Event to be added
-     * @param {string?} duplicateStrategy 'ignore' or 'replace'
-     * @param {boolean} fromCache whether the sync response came from cache
+     * @param {IAddLiveEventOptions} options addLiveEvent options
      * @fires module:client~MatrixClient#event:"Room.timeline"
      * @private
      */
-    private addLiveEvent(event: MatrixEvent, duplicateStrategy: DuplicateStrategy, fromCache = false): void {
+    private addLiveEvent(event: MatrixEvent, addLiveEventOptions: IAddLiveEventOptions): void {
+        const { duplicateStrategy, timelineWasEmpty, fromCache } = addLiveEventOptions;
+
         // add to our timeline sets
         for (let i = 0; i < this.timelineSets.length; i++) {
-            this.timelineSets[i].addLiveEvent(event, duplicateStrategy, fromCache);
+            this.timelineSets[i].addLiveEvent(event, {
+                duplicateStrategy,
+                fromCache,
+                timelineWasEmpty,
+            });
         }
 
         // synthesize and inject implicit read receipts
@@ -1872,11 +2051,15 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
                 if (timelineSet.getFilter()) {
                     if (timelineSet.getFilter().filterRoomTimeline([event]).length) {
                         timelineSet.addEventToTimeline(event,
-                            timelineSet.getLiveTimeline(), false);
+                            timelineSet.getLiveTimeline(), {
+                                toStartOfTimeline: false,
+                            });
                     }
                 } else {
                     timelineSet.addEventToTimeline(event,
-                        timelineSet.getLiveTimeline(), false);
+                        timelineSet.getLiveTimeline(), {
+                            toStartOfTimeline: false,
+                        });
                 }
             }
         }
@@ -1911,15 +2094,7 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
                 return isEventEncrypted || !isRoomEncrypted;
             });
 
-            const { store } = this.client.sessionStore;
-            if (this.pendingEventList.length > 0) {
-                store.setItem(
-                    pendingEventsKey(this.roomId),
-                    JSON.stringify(pendingEvents),
-                );
-            } else {
-                store.removeItem(pendingEventsKey(this.roomId));
-            }
+            this.client.store.setPendingEvents(this.roomId, pendingEvents);
         }
     }
 
@@ -1934,24 +2109,7 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
      * @param {module:models/event.MatrixEvent} event the relation event that needs to be aggregated.
      */
     private aggregateNonLiveRelation(event: MatrixEvent): void {
-        const { shouldLiveInRoom, threadId } = this.eventShouldLiveIn(event);
-        const thread = this.getThread(threadId);
-        thread?.timelineSet.aggregateRelations(event);
-
-        if (shouldLiveInRoom) {
-            // TODO: We should consider whether this means it would be a better
-            // design to lift the relations handling up to the room instead.
-            for (let i = 0; i < this.timelineSets.length; i++) {
-                const timelineSet = this.timelineSets[i];
-                if (timelineSet.getFilter()) {
-                    if (timelineSet.getFilter().filterRoomTimeline([event]).length) {
-                        timelineSet.aggregateRelations(event);
-                    }
-                } else {
-                    timelineSet.aggregateRelations(event);
-                }
-            }
-        }
+        this.relations.aggregateChildEvent(event);
     }
 
     public getEventForTxnId(txnId: string): MatrixEvent {
@@ -2036,7 +2194,22 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
             const timeline = this.getTimelineForEvent(newEventId);
             if (timeline) {
                 // we've already received the event via the event stream.
-                // nothing more to do here.
+                // nothing more to do here, assuming the transaction ID was correctly matched.
+                // Let's check that.
+                const remoteEvent = this.findEventById(newEventId);
+                const remoteTxnId = remoteEvent.getUnsigned().transaction_id;
+                if (!remoteTxnId) {
+                    // This code path is mostly relevant for the Sliding Sync proxy.
+                    // The remote event did not contain a transaction ID, so we did not handle
+                    // the remote echo yet. Handle it now.
+                    const unsigned = remoteEvent.getUnsigned();
+                    unsigned.transaction_id = event.getTxnId();
+                    remoteEvent.setUnsigned(unsigned);
+                    // the remote event is _already_ in the timeline, so we need to remove it so
+                    // we can convert the local event into the final event.
+                    this.removeEvent(remoteEvent.getId());
+                    this.handleRemoteEcho(remoteEvent, event);
+                }
                 return;
             }
         }
@@ -2113,18 +2286,38 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
      * they will go to the end of the timeline.
      *
      * @param {MatrixEvent[]} events A list of events to add.
-     *
-     * @param {string} duplicateStrategy Optional. Applies to events in the
-     * timeline only. If this is 'replace' then if a duplicate is encountered, the
-     * event passed to this function will replace the existing event in the
-     * timeline. If this is not specified, or is 'ignore', then the event passed to
-     * this function will be ignored entirely, preserving the existing event in the
-     * timeline. Events are identical based on their event ID <b>only</b>.
-     *
-     * @param {boolean} fromCache whether the sync response came from cache
+     * @param {IAddLiveEventOptions} options addLiveEvent options
      * @throws If <code>duplicateStrategy</code> is not falsey, 'replace' or 'ignore'.
      */
-    public addLiveEvents(events: MatrixEvent[], duplicateStrategy?: DuplicateStrategy, fromCache = false): void {
+    public addLiveEvents(events: MatrixEvent[], addLiveEventOptions?: IAddLiveEventOptions): void;
+    /**
+     * @deprecated In favor of the overload with `IAddLiveEventOptions`
+     */
+    public addLiveEvents(events: MatrixEvent[], duplicateStrategy?: DuplicateStrategy, fromCache?: boolean): void;
+    public addLiveEvents(
+        events: MatrixEvent[],
+        duplicateStrategyOrOpts?: DuplicateStrategy | IAddLiveEventOptions,
+        fromCache = false,
+    ): void {
+        let duplicateStrategy = duplicateStrategyOrOpts as DuplicateStrategy;
+        let timelineWasEmpty: boolean;
+        if (typeof (duplicateStrategyOrOpts) === 'object') {
+            ({
+                duplicateStrategy,
+                fromCache = false,
+                /* roomState, (not used here) */
+                timelineWasEmpty,
+            } = duplicateStrategyOrOpts);
+        } else if (duplicateStrategyOrOpts !== undefined) {
+            // Deprecation warning
+            // FIXME: Remove after 2023-06-01 (technical debt)
+            logger.warn(
+                'Overload deprecated: ' +
+                '`Room.addLiveEvents(events, duplicateStrategy?, fromCache?)` ' +
+                'is deprecated in favor of the overload with `Room.addLiveEvents(events, IAddLiveEventOptions)`',
+            );
+        }
+
         if (duplicateStrategy && ["replace", "ignore"].indexOf(duplicateStrategy) === -1) {
             throw new Error("duplicateStrategy MUST be either 'replace' or 'ignore'");
         }
@@ -2162,7 +2355,11 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
             eventsByThread[threadId]?.push(event);
 
             if (shouldLiveInRoom) {
-                this.addLiveEvent(event, duplicateStrategy, fromCache);
+                this.addLiveEvent(event, {
+                    duplicateStrategy,
+                    fromCache,
+                    timelineWasEmpty,
+                });
             }
         }
 
@@ -2213,7 +2410,7 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
     private findThreadRoots(events: MatrixEvent[]): Set<string> {
         const threadRoots = new Set<string>();
         for (const event of events) {
-            if (event.isThreadRelation) {
+            if (event.isRelation(THREAD_RELATION_TYPE.name)) {
                 threadRoots.add(event.relationEventId);
             }
         }
@@ -2941,14 +3138,6 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
     }
 }
 
-/**
- * @param {string} roomId ID of the current room
- * @returns {string} Storage key to retrieve pending events
- */
-function pendingEventsKey(roomId: string): string {
-    return `mx_pending_events_${roomId}`;
-}
-
 // a map from current event status to a list of allowed next statuses
 const ALLOWED_TRANSITIONS: Record<EventStatus, EventStatus[]> = {
     [EventStatus.ENCRYPTING]: [

src/models/thread.ts

@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { Optional } from "matrix-events-sdk";
+
 import { MatrixClient, MatrixEventEvent, RelationType, RoomEvent } from "../matrix";
 import { TypedReEmitter } from "../ReEmitter";
 import { IRelationsRequestOpts } from "../@types/requests";
@@ -88,10 +90,9 @@ export class Thread extends TypedEventEmitter<EmittedEvents, EventHandlerMap> {
         this.room = opts.room;
         this.client = opts.client;
         this.timelineSet = new EventTimelineSet(this.room, {
-            unstableClientRelationAggregation: true,
             timelineSupport: true,
             pendingEvents: true,
-        });
+        }, this.client, this);
         this.reEmitter = new TypedReEmitter(this);
 
         this.reEmitter.reEmit(this.timelineSet, [
@@ -166,6 +167,7 @@ export class Thread extends TypedEventEmitter<EmittedEvents, EventHandlerMap> {
     private onEcho = (event: MatrixEvent) => {
         if (event.threadRootId !== this.id) return; // ignore echoes for other timelines
         if (this.lastEvent === event) return;
+        if (!event.isRelation(THREAD_RELATION_TYPE.name)) return;
 
         // There is a risk that the `localTimestamp` approximation will not be accurate
         // when threads are used over federation. That could result in the reply
@@ -199,9 +201,11 @@ export class Thread extends TypedEventEmitter<EmittedEvents, EventHandlerMap> {
             this.timelineSet.addEventToTimeline(
                 event,
                 this.liveTimeline,
-                toStartOfTimeline,
-                false,
-                this.roomState,
+                {
+                    toStartOfTimeline,
+                    fromCache: false,
+                    roomState: this.roomState,
+                },
             );
         }
     }
@@ -227,13 +231,6 @@ export class Thread extends TypedEventEmitter<EmittedEvents, EventHandlerMap> {
             this._currentUserParticipated = true;
         }
 
-        if ([RelationType.Annotation, RelationType.Replace].includes(event.getRelation()?.rel_type as RelationType)) {
-            // Apply annotations and replace relations to the relations of the timeline only
-            this.timelineSet.setRelationsTarget(event);
-            this.timelineSet.aggregateRelations(event);
-            return;
-        }
-
         // Add all incoming events to the thread's timeline set when there's  no server support
         if (!Thread.hasServerSideSupport) {
             // all the relevant membership info to hydrate events with a sender
@@ -249,6 +246,11 @@ export class Thread extends TypedEventEmitter<EmittedEvents, EventHandlerMap> {
         ) {
             this.fetchEditsWhereNeeded(event);
             this.addEventToTimeline(event, false);
+        } else if (event.isRelation(RelationType.Annotation) || event.isRelation(RelationType.Replace)) {
+            // Apply annotations and replace relations to the relations of the timeline only
+            this.timelineSet.relations.aggregateParentEvent(event);
+            this.timelineSet.relations.aggregateChildEvent(event, this.timelineSet);
+            return;
         }
 
         // If no thread support exists we want to count all thread relation
@@ -277,7 +279,10 @@ export class Thread extends TypedEventEmitter<EmittedEvents, EventHandlerMap> {
             this.replyCount = bundledRelationship.count;
             this._currentUserParticipated = bundledRelationship.current_user_participated;
 
-            const event = new MatrixEvent(bundledRelationship.latest_event);
+            const event = new MatrixEvent({
+                room_id: this.rootEvent.getRoomId(),
+                ...bundledRelationship.latest_event,
+            });
             this.setEventMetadata(event);
             event.setThread(this);
             this.lastEvent = event;
@@ -291,6 +296,7 @@ export class Thread extends TypedEventEmitter<EmittedEvents, EventHandlerMap> {
     // XXX: Workaround for https://github.com/matrix-org/matrix-spec-proposals/pull/2676/files#r827240084
     private async fetchEditsWhereNeeded(...events: MatrixEvent[]): Promise<unknown> {
         return Promise.all(events.filter(e => e.isEncrypted()).map((event: MatrixEvent) => {
+            if (event.isRelation()) return; // skip - relations don't get edits
             return this.client.relations(this.roomId, event.getId(), RelationType.Replace, event.getType(), {
                 limit: 1,
             }).then(relations => {
@@ -327,15 +333,16 @@ export class Thread extends TypedEventEmitter<EmittedEvents, EventHandlerMap> {
     }
 
     /**
-     * Return last reply to the thread
+     * Return last reply to the thread, if known.
      */
-    public lastReply(matches: (ev: MatrixEvent) => boolean = () => true): MatrixEvent {
+    public lastReply(matches: (ev: MatrixEvent) => boolean = () => true): Optional<MatrixEvent> {
         for (let i = this.events.length - 1; i >= 0; i--) {
             const event = this.events[i];
             if (matches(event)) {
                 return event;
             }
         }
+        return null;
     }
 
     public get roomId(): string {
@@ -352,9 +359,9 @@ export class Thread extends TypedEventEmitter<EmittedEvents, EventHandlerMap> {
     }
 
     /**
-     * A getter for the last event added to the thread
+     * A getter for the last event added to the thread, if known.
      */
-    public get replyToEvent(): MatrixEvent {
+    public get replyToEvent(): Optional<MatrixEvent> {
         return this.lastEvent ?? this.lastReply();
     }
 

src/pushprocessor.ts

@@ -83,10 +83,13 @@ const DEFAULT_OVERRIDE_RULES: IPushRule[] = [
                 key: "type",
                 pattern: EventType.RoomServerAcl,
             },
+            {
+                kind: ConditionKind.EventMatch,
+                key: "state_key",
+                pattern: "",
+            },
         ],
-        actions: [
-            PushRuleActionName.DontNotify,
-        ],
+        actions: [],
     },
 ];
 

src/realtime-callbacks.ts

@@ -48,20 +48,6 @@ const callbackList: {
 // var debuglog = logger.log.bind(logger);
 const debuglog = function(...params: any[]) {};
 
-/**
- * Replace the function used by this module to get the current time.
- *
- * Intended for use by the unit tests.
- *
- * @param {function} [f] function which should return a millisecond counter
- *
- * @internal
- */
-export function setNow(f: () => number): void {
-    now = f || Date.now;
-}
-let now = Date.now;
-
 /**
  * reimplementation of window.setTimeout, which will call the callback if
  * the wallclock time goes past the deadline.
@@ -78,7 +64,7 @@ export function setTimeout(func: (...params: any[]) => void, delayMs: number, ..
         delayMs = 0;
     }
 
-    const runAt = now() + delayMs;
+    const runAt = Date.now() + delayMs;
     const key = count++;
     debuglog("setTimeout: scheduling cb", key, "at", runAt,
         "(delay", delayMs, ")");
@@ -141,7 +127,7 @@ function scheduleRealCallback(): void {
         return;
     }
 
-    const timestamp = now();
+    const timestamp = Date.now();
     const delayMs = Math.min(first.runAt - timestamp, TIMER_CHECK_PERIOD_MS);
 
     debuglog("scheduleRealCallback: now:", timestamp, "delay:", delayMs);
@@ -150,7 +136,7 @@ function scheduleRealCallback(): void {
 
 function runCallbacks(): void {
     let cb;
-    const timestamp = now();
+    const timestamp = Date.now();
     debuglog("runCallbacks: now:", timestamp);
 
     // get the list of things to call

src/sliding-sync-sdk.ts

@@ -0,0 +1,835 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { NotificationCountType, Room, RoomEvent } from "./models/room";
+import { logger } from './logger';
+import * as utils from "./utils";
+import { EventTimeline } from "./models/event-timeline";
+import { ClientEvent, IStoredClientOpts, MatrixClient, PendingEventOrdering } from "./client";
+import { ISyncStateData, SyncState } from "./sync";
+import { MatrixEvent } from "./models/event";
+import { Crypto } from "./crypto";
+import { IMinimalEvent, IRoomEvent, IStateEvent, IStrippedState } from "./sync-accumulator";
+import { MatrixError } from "./http-api";
+import { RoomStateEvent } from "./models/room-state";
+import { RoomMemberEvent } from "./models/room-member";
+import {
+    Extension,
+    ExtensionState,
+    MSC3575RoomData,
+    MSC3575SlidingSyncResponse,
+    SlidingSync,
+    SlidingSyncEvent,
+    SlidingSyncState,
+} from "./sliding-sync";
+import { EventType, IPushRules } from "./matrix";
+import { PushProcessor } from "./pushprocessor";
+
+// Number of consecutive failed syncs that will lead to a syncState of ERROR as opposed
+// to RECONNECTING. This is needed to inform the client of server issues when the
+// keepAlive is successful but the server /sync fails.
+const FAILED_SYNC_ERROR_THRESHOLD = 3;
+
+class ExtensionE2EE implements Extension {
+    constructor(private readonly crypto: Crypto) {}
+
+    public name(): string {
+        return "e2ee";
+    }
+
+    public when(): ExtensionState {
+        return ExtensionState.PreProcess;
+    }
+
+    public onRequest(isInitial: boolean): object {
+        if (!isInitial) {
+            return undefined;
+        }
+        return {
+            enabled: true, // this is sticky so only send it on the initial request
+        };
+    }
+
+    public async onResponse(data: object): Promise<void> {
+        // Handle device list updates
+        if (data["device_lists"]) {
+            await this.crypto.handleDeviceListChanges({
+                oldSyncToken: "yep", // XXX need to do this so the device list changes get processed :(
+            }, data["device_lists"]);
+        }
+
+        // Handle one_time_keys_count
+        if (data["device_one_time_keys_count"]) {
+            const currentCount = data["device_one_time_keys_count"].signed_curve25519 || 0;
+            this.crypto.updateOneTimeKeyCount(currentCount);
+        }
+        if (data["device_unused_fallback_key_types"] ||
+                data["org.matrix.msc2732.device_unused_fallback_key_types"]) {
+            // The presence of device_unused_fallback_key_types indicates that the
+            // server supports fallback keys. If there's no unused
+            // signed_curve25519 fallback key we need a new one.
+            const unusedFallbackKeys = data["device_unused_fallback_key_types"] ||
+                data["org.matrix.msc2732.device_unused_fallback_key_types"];
+            this.crypto.setNeedsNewFallback(
+                unusedFallbackKeys instanceof Array &&
+                !unusedFallbackKeys.includes("signed_curve25519"),
+            );
+        }
+    }
+}
+
+class ExtensionToDevice implements Extension {
+    private nextBatch?: string = null;
+
+    constructor(private readonly client: MatrixClient) {}
+
+    public name(): string {
+        return "to_device";
+    }
+
+    public when(): ExtensionState {
+        return ExtensionState.PreProcess;
+    }
+
+    public onRequest(isInitial: boolean): object {
+        const extReq = {
+            since: this.nextBatch !== null ? this.nextBatch : undefined,
+        };
+        if (isInitial) {
+            extReq["limit"] = 100;
+            extReq["enabled"] = true;
+        }
+        return extReq;
+    }
+
+    public async onResponse(data: object): Promise<void> {
+        const cancelledKeyVerificationTxns = [];
+        data["events"] = data["events"] || [];
+        data["events"]
+            .map(this.client.getEventMapper())
+            .map((toDeviceEvent) => { // map is a cheap inline forEach
+                // We want to flag m.key.verification.start events as cancelled
+                // if there's an accompanying m.key.verification.cancel event, so
+                // we pull out the transaction IDs from the cancellation events
+                // so we can flag the verification events as cancelled in the loop
+                // below.
+                if (toDeviceEvent.getType() === "m.key.verification.cancel") {
+                    const txnId = toDeviceEvent.getContent()['transaction_id'];
+                    if (txnId) {
+                        cancelledKeyVerificationTxns.push(txnId);
+                    }
+                }
+
+                // as mentioned above, .map is a cheap inline forEach, so return
+                // the unmodified event.
+                return toDeviceEvent;
+            })
+            .forEach(
+                (toDeviceEvent) => {
+                    const content = toDeviceEvent.getContent();
+                    if (
+                        toDeviceEvent.getType() == "m.room.message" &&
+                        content.msgtype == "m.bad.encrypted"
+                    ) {
+                        // the mapper already logged a warning.
+                        logger.log(
+                            'Ignoring undecryptable to-device event from ' +
+                            toDeviceEvent.getSender(),
+                        );
+                        return;
+                    }
+
+                    if (toDeviceEvent.getType() === "m.key.verification.start"
+                        || toDeviceEvent.getType() === "m.key.verification.request") {
+                        const txnId = content['transaction_id'];
+                        if (cancelledKeyVerificationTxns.includes(txnId)) {
+                            toDeviceEvent.flagCancelled();
+                        }
+                    }
+
+                    this.client.emit(ClientEvent.ToDeviceEvent, toDeviceEvent);
+                },
+            );
+
+        this.nextBatch = data["next_batch"];
+    }
+}
+
+class ExtensionAccountData implements Extension {
+    constructor(private readonly client: MatrixClient) {}
+
+    public name(): string {
+        return "account_data";
+    }
+
+    public when(): ExtensionState {
+        return ExtensionState.PostProcess;
+    }
+
+    public onRequest(isInitial: boolean): object {
+        if (!isInitial) {
+            return undefined;
+        }
+        return {
+            enabled: true,
+        };
+    }
+
+    public onResponse(data: {global: object[], rooms: Record<string, object[]>}): void {
+        if (data.global && data.global.length > 0) {
+            this.processGlobalAccountData(data.global);
+        }
+
+        for (const roomId in data.rooms) {
+            const accountDataEvents = mapEvents(this.client, roomId, data.rooms[roomId]);
+            const room = this.client.getRoom(roomId);
+            if (!room) {
+                logger.warn("got account data for room but room doesn't exist on client:", roomId);
+                continue;
+            }
+            room.addAccountData(accountDataEvents);
+            accountDataEvents.forEach((e) => {
+                this.client.emit(ClientEvent.Event, e);
+            });
+        }
+    }
+
+    private processGlobalAccountData(globalAccountData: object[]): void {
+        const events = mapEvents(this.client, undefined, globalAccountData);
+        const prevEventsMap = events.reduce((m, c) => {
+            m[c.getId()] = this.client.store.getAccountData(c.getType());
+            return m;
+        }, {});
+        this.client.store.storeAccountDataEvents(events);
+        events.forEach(
+            (accountDataEvent) => {
+                // Honour push rules that come down the sync stream but also
+                // honour push rules that were previously cached. Base rules
+                // will be updated when we receive push rules via getPushRules
+                // (see sync) before syncing over the network.
+                if (accountDataEvent.getType() === EventType.PushRules) {
+                    const rules = accountDataEvent.getContent<IPushRules>();
+                    this.client.pushRules = PushProcessor.rewriteDefaultRules(rules);
+                }
+                const prevEvent = prevEventsMap[accountDataEvent.getId()];
+                this.client.emit(ClientEvent.AccountData, accountDataEvent, prevEvent);
+                return accountDataEvent;
+            },
+        );
+    }
+}
+
+/**
+ * A copy of SyncApi such that it can be used as a drop-in replacement for sync v2. For the actual
+ * sliding sync API, see sliding-sync.ts or the class SlidingSync.
+ */
+export class SlidingSyncSdk {
+    private syncState: SyncState = null;
+    private syncStateData: ISyncStateData;
+    private lastPos: string = null;
+    private failCount = 0;
+    private notifEvents: MatrixEvent[] = []; // accumulator of sync events in the current sync response
+
+    constructor(
+        private readonly slidingSync: SlidingSync,
+        private readonly client: MatrixClient,
+        private readonly opts: Partial<IStoredClientOpts> = {},
+    ) {
+        this.opts.initialSyncLimit = this.opts.initialSyncLimit ?? 8;
+        this.opts.resolveInvitesToProfiles = this.opts.resolveInvitesToProfiles || false;
+        this.opts.pollTimeout = this.opts.pollTimeout || (30 * 1000);
+        this.opts.pendingEventOrdering = this.opts.pendingEventOrdering || PendingEventOrdering.Chronological;
+        this.opts.experimentalThreadSupport = this.opts.experimentalThreadSupport === true;
+
+        if (!opts.canResetEntireTimeline) {
+            opts.canResetEntireTimeline = (_roomId: string) => {
+                return false;
+            };
+        }
+
+        if (client.getNotifTimelineSet()) {
+            client.reEmitter.reEmit(client.getNotifTimelineSet(), [
+                RoomEvent.Timeline,
+                RoomEvent.TimelineReset,
+            ]);
+        }
+
+        this.slidingSync.on(SlidingSyncEvent.Lifecycle, this.onLifecycle.bind(this));
+        this.slidingSync.on(SlidingSyncEvent.RoomData, this.onRoomData.bind(this));
+        const extensions: Extension[] = [
+            new ExtensionToDevice(this.client),
+            new ExtensionAccountData(this.client),
+        ];
+        if (this.opts.crypto) {
+            extensions.push(
+                new ExtensionE2EE(this.opts.crypto),
+            );
+        }
+        extensions.forEach((ext) => {
+            this.slidingSync.registerExtension(ext);
+        });
+    }
+
+    private onRoomData(roomId: string, roomData: MSC3575RoomData): void {
+        let room = this.client.store.getRoom(roomId);
+        if (!room) {
+            if (!roomData.initial) {
+                logger.debug("initial flag not set but no stored room exists for room ", roomId, roomData);
+                return;
+            }
+            room = createRoom(this.client, roomId, this.opts);
+        }
+        this.processRoomData(this.client, room, roomData);
+    }
+
+    private onLifecycle(state: SlidingSyncState, resp: MSC3575SlidingSyncResponse, err?: Error): void {
+        if (err) {
+            logger.debug("onLifecycle", state, err);
+        }
+        switch (state) {
+            case SlidingSyncState.Complete:
+                this.purgeNotifications();
+                // Element won't stop showing the initial loading spinner unless we fire SyncState.Prepared
+                if (!this.lastPos) {
+                    this.updateSyncState(SyncState.Prepared, {
+                        oldSyncToken: this.lastPos,
+                        nextSyncToken: resp.pos,
+                        catchingUp: false,
+                        fromCache: false,
+                    });
+                }
+                // Conversely, Element won't show the room list unless there is at least 1x SyncState.Syncing
+                // so hence for the very first sync we will fire prepared then immediately syncing.
+                this.updateSyncState(SyncState.Syncing, {
+                    oldSyncToken: this.lastPos,
+                    nextSyncToken: resp.pos,
+                    catchingUp: false,
+                    fromCache: false,
+                });
+                this.lastPos = resp.pos;
+                break;
+            case SlidingSyncState.RequestFinished:
+                if (err) {
+                    this.failCount += 1;
+                    this.updateSyncState(
+                        this.failCount > FAILED_SYNC_ERROR_THRESHOLD ? SyncState.Error : SyncState.Reconnecting,
+                        {
+                            error: new MatrixError(err),
+                        },
+                    );
+                    if (this.shouldAbortSync(new MatrixError(err))) {
+                        return; // shouldAbortSync actually stops syncing too so we don't need to do anything.
+                    }
+                } else {
+                    this.failCount = 0;
+                }
+                break;
+        }
+    }
+
+    /**
+     * Sync rooms the user has left.
+     * @return {Promise} Resolved when they've been added to the store.
+     */
+    public async syncLeftRooms() {
+        return []; // TODO
+    }
+
+    /**
+     * Peek into a room. This will result in the room in question being synced so it
+     * is accessible via getRooms(). Live updates for the room will be provided.
+     * @param {string} roomId The room ID to peek into.
+     * @return {Promise} A promise which resolves once the room has been added to the
+     * store.
+     */
+    public async peek(_roomId: string): Promise<Room> {
+        return null; // TODO
+    }
+
+    /**
+     * Stop polling for updates in the peeked room. NOPs if there is no room being
+     * peeked.
+     */
+    public stopPeeking(): void {
+        // TODO
+    }
+
+    /**
+     * Returns the current state of this sync object
+     * @see module:client~MatrixClient#event:"sync"
+     * @return {?String}
+     */
+    public getSyncState(): SyncState {
+        return this.syncState;
+    }
+
+    /**
+     * Returns the additional data object associated with
+     * the current sync state, or null if there is no
+     * such data.
+     * Sync errors, if available, are put in the 'error' key of
+     * this object.
+     * @return {?Object}
+     */
+    public getSyncStateData(): ISyncStateData {
+        return this.syncStateData;
+    }
+
+    private shouldAbortSync(error: MatrixError): boolean {
+        if (error.errcode === "M_UNKNOWN_TOKEN") {
+            // The logout already happened, we just need to stop.
+            logger.warn("Token no longer valid - assuming logout");
+            this.stop();
+            this.updateSyncState(SyncState.Error, { error });
+            return true;
+        }
+        return false;
+    }
+
+    private async processRoomData(client: MatrixClient, room: Room, roomData: MSC3575RoomData) {
+        roomData = ensureNameEvent(client, room.roomId, roomData);
+        const stateEvents = mapEvents(this.client, room.roomId, roomData.required_state);
+        // Prevent events from being decrypted ahead of time
+        // this helps large account to speed up faster
+        // room::decryptCriticalEvent is in charge of decrypting all the events
+        // required for a client to function properly
+        const timelineEvents = mapEvents(this.client, room.roomId, roomData.timeline, false);
+        const ephemeralEvents = []; // TODO this.mapSyncEventsFormat(joinObj.ephemeral);
+
+        const encrypted = this.client.isRoomEncrypted(room.roomId);
+        // we do this first so it's correct when any of the events fire
+        if (roomData.notification_count != null) {
+            room.setUnreadNotificationCount(
+                NotificationCountType.Total,
+                roomData.notification_count,
+            );
+        }
+
+        if (roomData.highlight_count != null) {
+            // We track unread notifications ourselves in encrypted rooms, so don't
+            // bother setting it here. We trust our calculations better than the
+            // server's for this case, and therefore will assume that our non-zero
+            // count is accurate.
+            if (!encrypted
+                || (encrypted && room.getUnreadNotificationCount(NotificationCountType.Highlight) <= 0)) {
+                room.setUnreadNotificationCount(
+                    NotificationCountType.Highlight,
+                    roomData.highlight_count,
+                );
+            }
+        }
+
+        if (roomData.invite_state) {
+            const inviteStateEvents = mapEvents(this.client, room.roomId, roomData.invite_state);
+            this.processRoomEvents(room, inviteStateEvents);
+            if (roomData.initial) {
+                room.recalculate();
+                this.client.store.storeRoom(room);
+                this.client.emit(ClientEvent.Room, room);
+            }
+            inviteStateEvents.forEach((e) => {
+                this.client.emit(ClientEvent.Event, e);
+            });
+            room.updateMyMembership("invite");
+            return;
+        }
+
+        if (roomData.initial) {
+            // set the back-pagination token. Do this *before* adding any
+            // events so that clients can start back-paginating.
+            room.getLiveTimeline().setPaginationToken(
+                roomData.prev_batch, EventTimeline.BACKWARDS);
+        }
+
+        /* TODO
+        else if (roomData.limited) {
+
+            let limited = true;
+
+            // we've got a limited sync, so we *probably* have a gap in the
+            // timeline, so should reset. But we might have been peeking or
+            // paginating and already have some of the events, in which
+            // case we just want to append any subsequent events to the end
+            // of the existing timeline.
+            //
+            // This is particularly important in the case that we already have
+            // *all* of the events in the timeline - in that case, if we reset
+            // the timeline, we'll end up with an entirely empty timeline,
+            // which we'll try to paginate but not get any new events (which
+            // will stop us linking the empty timeline into the chain).
+            //
+            for (let i = timelineEvents.length - 1; i >= 0; i--) {
+                const eventId = timelineEvents[i].getId();
+                if (room.getTimelineForEvent(eventId)) {
+                    logger.debug("Already have event " + eventId + " in limited " +
+                        "sync - not resetting");
+                    limited = false;
+
+                    // we might still be missing some of the events before i;
+                    // we don't want to be adding them to the end of the
+                    // timeline because that would put them out of order.
+                    timelineEvents.splice(0, i);
+
+                    // XXX: there's a problem here if the skipped part of the
+                    // timeline modifies the state set in stateEvents, because
+                    // we'll end up using the state from stateEvents rather
+                    // than the later state from timelineEvents. We probably
+                    // need to wind stateEvents forward over the events we're
+                    // skipping.
+                    break;
+                }
+            }
+
+            if (limited) {
+                deregisterStateListeners(room);
+                room.resetLiveTimeline(
+                    roomData.prev_batch,
+                    null, // TODO this.opts.canResetEntireTimeline(room.roomId) ? null : syncEventData.oldSyncToken,
+                );
+
+                // We have to assume any gap in any timeline is
+                // reason to stop incrementally tracking notifications and
+                // reset the timeline.
+                this.client.resetNotifTimelineSet();
+                registerStateListeners(this.client, room);
+            }
+        } */
+
+        this.processRoomEvents(room, stateEvents, timelineEvents, false);
+
+        // we deliberately don't add ephemeral events to the timeline
+        room.addEphemeralEvents(ephemeralEvents);
+
+        room.recalculate();
+        if (roomData.initial) {
+            client.store.storeRoom(room);
+            client.emit(ClientEvent.Room, room);
+        }
+
+        // check if any timeline events should bing and add them to the notifEvents array:
+        // we'll purge this once we've fully processed the sync response
+        this.addNotifications(timelineEvents);
+
+        const processRoomEvent = async (e: MatrixEvent) => {
+            client.emit(ClientEvent.Event, e);
+            if (e.isState() && e.getType() == EventType.RoomEncryption && this.opts.crypto) {
+                await this.opts.crypto.onCryptoEvent(e);
+            }
+        };
+
+        await utils.promiseMapSeries(stateEvents, processRoomEvent);
+        await utils.promiseMapSeries(timelineEvents, processRoomEvent);
+        ephemeralEvents.forEach(function(e) {
+            client.emit(ClientEvent.Event, e);
+        });
+
+        room.updateMyMembership("join");
+
+        // Decrypt only the last message in all rooms to make sure we can generate a preview
+        // And decrypt all events after the recorded read receipt to ensure an accurate
+        // notification count
+        room.decryptCriticalEvents();
+    }
+
+    /**
+     * @param {Room} room
+     * @param {MatrixEvent[]} stateEventList A list of state events. This is the state
+     * at the *START* of the timeline list if it is supplied.
+     * @param {MatrixEvent[]} [timelineEventList] A list of timeline events. Lower index
+     * @param {boolean} fromCache whether the sync response came from cache
+     * is earlier in time. Higher index is later.
+     */
+    private processRoomEvents(
+        room: Room,
+        stateEventList: MatrixEvent[],
+        timelineEventList?: MatrixEvent[],
+        fromCache = false,
+    ): void {
+        timelineEventList = timelineEventList || [];
+        stateEventList = stateEventList || [];
+
+        // If there are no events in the timeline yet, initialise it with
+        // the given state events
+        const liveTimeline = room.getLiveTimeline();
+        const timelineWasEmpty = liveTimeline.getEvents().length == 0;
+        if (timelineWasEmpty) {
+            // Passing these events into initialiseState will freeze them, so we need
+            // to compute and cache the push actions for them now, otherwise sync dies
+            // with an attempt to assign to read only property.
+            // XXX: This is pretty horrible and is assuming all sorts of behaviour from
+            // these functions that it shouldn't be. We should probably either store the
+            // push actions cache elsewhere so we can freeze MatrixEvents, or otherwise
+            // find some solution where MatrixEvents are immutable but allow for a cache
+            // field.
+            for (const ev of stateEventList) {
+                this.client.getPushActionsForEvent(ev);
+            }
+            liveTimeline.initialiseState(stateEventList);
+        }
+
+        // If the timeline wasn't empty, we process the state events here: they're
+        // defined as updates to the state before the start of the timeline, so this
+        // starts to roll the state forward.
+        // XXX: That's what we *should* do, but this can happen if we were previously
+        // peeking in a room, in which case we obviously do *not* want to add the
+        // state events here onto the end of the timeline. Historically, the js-sdk
+        // has just set these new state events on the old and new state. This seems
+        // very wrong because there could be events in the timeline that diverge the
+        // state, in which case this is going to leave things out of sync. However,
+        // for now I think it;s best to behave the same as the code has done previously.
+        if (!timelineWasEmpty) {
+            // XXX: As above, don't do this...
+            //room.addLiveEvents(stateEventList || []);
+            // Do this instead...
+            room.oldState.setStateEvents(stateEventList);
+            room.currentState.setStateEvents(stateEventList);
+        }
+
+        // execute the timeline events. This will continue to diverge the current state
+        // if the timeline has any state events in it.
+        // This also needs to be done before running push rules on the events as they need
+        // to be decorated with sender etc.
+        room.addLiveEvents(timelineEventList, {
+            fromCache: fromCache,
+        });
+
+        room.recalculate();
+
+        // resolve invites now we have set the latest state
+        this.resolveInvites(room);
+    }
+
+    private resolveInvites(room: Room): void {
+        if (!room || !this.opts.resolveInvitesToProfiles) {
+            return;
+        }
+        const client = this.client;
+        // For each invited room member we want to give them a displayname/avatar url
+        // if they have one (the m.room.member invites don't contain this).
+        room.getMembersWithMembership("invite").forEach(function(member) {
+            if (member._requestedProfileInfo) return;
+            member._requestedProfileInfo = true;
+            // try to get a cached copy first.
+            const user = client.getUser(member.userId);
+            let promise;
+            if (user) {
+                promise = Promise.resolve({
+                    avatar_url: user.avatarUrl,
+                    displayname: user.displayName,
+                });
+            } else {
+                promise = client.getProfileInfo(member.userId);
+            }
+            promise.then(function(info) {
+                // slightly naughty by doctoring the invite event but this means all
+                // the code paths remain the same between invite/join display name stuff
+                // which is a worthy trade-off for some minor pollution.
+                const inviteEvent = member.events.member;
+                if (inviteEvent.getContent().membership !== "invite") {
+                    // between resolving and now they have since joined, so don't clobber
+                    return;
+                }
+                inviteEvent.getContent().avatar_url = info.avatar_url;
+                inviteEvent.getContent().displayname = info.displayname;
+                // fire listeners
+                member.setMembershipEvent(inviteEvent, room.currentState);
+            }, function(_err) {
+                // OH WELL.
+            });
+        });
+    }
+
+    public retryImmediately(): boolean {
+        return true;
+    }
+
+    /**
+     * Main entry point. Blocks until stop() is called.
+     */
+    public async sync() {
+        logger.debug("Sliding sync init loop");
+
+        //   1) We need to get push rules so we can check if events should bing as we get
+        //      them from /sync.
+        while (!this.client.isGuest()) {
+            try {
+                logger.debug("Getting push rules...");
+                const result = await this.client.getPushRules();
+                logger.debug("Got push rules");
+                this.client.pushRules = result;
+                break;
+            } catch (err) {
+                logger.error("Getting push rules failed", err);
+                if (this.shouldAbortSync(err)) {
+                    return;
+                }
+            }
+        }
+
+        // start syncing
+        await this.slidingSync.start();
+    }
+
+    /**
+     * Stops the sync object from syncing.
+     */
+    public stop(): void {
+        logger.debug("SyncApi.stop");
+        this.slidingSync.stop();
+    }
+
+    /**
+     * Sets the sync state and emits an event to say so
+     * @param {String} newState The new state string
+     * @param {Object} data Object of additional data to emit in the event
+     */
+    private updateSyncState(newState: SyncState, data?: ISyncStateData): void {
+        const old = this.syncState;
+        this.syncState = newState;
+        this.syncStateData = data;
+        this.client.emit(ClientEvent.Sync, this.syncState, old, data);
+    }
+
+    /**
+     * Takes a list of timelineEvents and adds and adds to notifEvents
+     * as appropriate.
+     * This must be called after the room the events belong to has been stored.
+     *
+     * @param {MatrixEvent[]} [timelineEventList] A list of timeline events. Lower index
+     * is earlier in time. Higher index is later.
+     */
+    private addNotifications(timelineEventList: MatrixEvent[]): void {
+        // gather our notifications into this.notifEvents
+        if (!this.client.getNotifTimelineSet()) {
+            return;
+        }
+        for (const timelineEvent of timelineEventList) {
+            const pushActions = this.client.getPushActionsForEvent(timelineEvent);
+            if (pushActions && pushActions.notify &&
+                pushActions.tweaks && pushActions.tweaks.highlight) {
+                this.notifEvents.push(timelineEvent);
+            }
+        }
+    }
+
+    /**
+     * Purge any events in the notifEvents array. Used after a /sync has been complete.
+     * This should not be called at a per-room scope (e.g in onRoomData) because otherwise the ordering
+     * will be messed up e.g room A gets a bing, room B gets a newer bing, but both in the same /sync
+     * response. If we purge at a per-room scope then we could process room B before room A leading to
+     * room B appearing earlier in the notifications timeline, even though it has the higher origin_server_ts.
+     */
+    private purgeNotifications(): void {
+        this.notifEvents.sort(function(a, b) {
+            return a.getTs() - b.getTs();
+        });
+        this.notifEvents.forEach((event) => {
+            this.client.getNotifTimelineSet().addLiveEvent(event);
+        });
+        this.notifEvents = [];
+    }
+}
+
+function ensureNameEvent(client: MatrixClient, roomId: string, roomData: MSC3575RoomData): MSC3575RoomData {
+    // make sure m.room.name is in required_state if there is a name, replacing anything previously
+    // there if need be. This ensures clients transparently 'calculate' the right room name. Native
+    // sliding sync clients should just read the "name" field.
+    if (!roomData.name) {
+        return roomData;
+    }
+    for (const stateEvent of roomData.required_state) {
+        if (stateEvent.type === EventType.RoomName && stateEvent.state_key === "") {
+            stateEvent.content = {
+                name: roomData.name,
+            };
+            return roomData;
+        }
+    }
+    roomData.required_state.push({
+        event_id: "$fake-sliding-sync-name-event-" + roomId,
+        state_key: "",
+        type: EventType.RoomName,
+        content: {
+            name: roomData.name,
+        },
+        sender: client.getUserId(),
+        origin_server_ts: new Date().getTime(),
+    });
+    return roomData;
+}
+
+// Helper functions which set up JS SDK structs are below and are identical to the sync v2 counterparts,
+// just outside the class.
+
+function createRoom(client: MatrixClient, roomId: string, opts: Partial<IStoredClientOpts>): Room { // XXX cargoculted from sync.ts
+    const { timelineSupport } = client;
+    const room = new Room(roomId, client, client.getUserId(), {
+        lazyLoadMembers: opts.lazyLoadMembers,
+        pendingEventOrdering: opts.pendingEventOrdering,
+        timelineSupport,
+    });
+    client.reEmitter.reEmit(room, [
+        RoomEvent.Name,
+        RoomEvent.Redaction,
+        RoomEvent.RedactionCancelled,
+        RoomEvent.Receipt,
+        RoomEvent.Tags,
+        RoomEvent.LocalEchoUpdated,
+        RoomEvent.AccountData,
+        RoomEvent.MyMembership,
+        RoomEvent.Timeline,
+        RoomEvent.TimelineReset,
+    ]);
+    registerStateListeners(client, room);
+    return room;
+}
+
+function registerStateListeners(client: MatrixClient, room: Room): void { // XXX cargoculted from sync.ts
+    // we need to also re-emit room state and room member events, so hook it up
+    // to the client now. We need to add a listener for RoomState.members in
+    // order to hook them correctly.
+    client.reEmitter.reEmit(room.currentState, [
+        RoomStateEvent.Events,
+        RoomStateEvent.Members,
+        RoomStateEvent.NewMember,
+        RoomStateEvent.Update,
+    ]);
+    room.currentState.on(RoomStateEvent.NewMember, function(event, state, member) {
+        member.user = client.getUser(member.userId);
+        client.reEmitter.reEmit(member, [
+            RoomMemberEvent.Name,
+            RoomMemberEvent.Typing,
+            RoomMemberEvent.PowerLevel,
+            RoomMemberEvent.Membership,
+        ]);
+    });
+}
+
+/*
+function deregisterStateListeners(room: Room): void { // XXX cargoculted from sync.ts
+    // could do with a better way of achieving this.
+    room.currentState.removeAllListeners(RoomStateEvent.Events);
+    room.currentState.removeAllListeners(RoomStateEvent.Members);
+    room.currentState.removeAllListeners(RoomStateEvent.NewMember);
+} */
+
+function mapEvents(client: MatrixClient, roomId: string, events: object[], decrypt = true): MatrixEvent[] {
+    const mapper = client.getEventMapper({ decrypt });
+    return (events as Array<IStrippedState | IRoomEvent | IStateEvent | IMinimalEvent>).map(function(e) {
+        e["room_id"] = roomId;
+        return mapper(e);
+    });
+}

src/sliding-sync.ts

@@ -0,0 +1,757 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { logger } from './logger';
+import { IAbortablePromise } from "./@types/partials";
+import { MatrixClient } from "./client";
+import { IRoomEvent, IStateEvent } from "./sync-accumulator";
+import { TypedEventEmitter } from "./models//typed-event-emitter";
+import { sleep } from "./utils";
+
+// /sync requests allow you to set a timeout= but the request may continue
+// beyond that and wedge forever, so we need to track how long we are willing
+// to keep open the connection. This constant is *ADDED* to the timeout= value
+// to determine the max time we're willing to wait.
+const BUFFER_PERIOD_MS = 10 * 1000;
+
+/**
+ * Represents a subscription to a room or set of rooms. Controls which events are returned.
+ */
+export interface MSC3575RoomSubscription {
+    required_state?: string[][];
+    timeline_limit?: number;
+}
+
+/**
+ * Controls which rooms are returned in a given list.
+ */
+export interface MSC3575Filter {
+    is_dm?: boolean;
+    is_encrypted?: boolean;
+    is_invite?: boolean;
+    is_tombstoned?: boolean;
+    room_name_like?: string;
+}
+
+/**
+ * Represents a list subscription.
+ */
+export interface MSC3575List extends MSC3575RoomSubscription {
+    ranges: number[][];
+    sort?: string[];
+    filters?: MSC3575Filter;
+    slow_get_all_rooms?: boolean;
+}
+
+/**
+ * A complete Sliding Sync request.
+ */
+export interface MSC3575SlidingSyncRequest {
+    // json body params
+    lists?: MSC3575List[];
+    unsubscribe_rooms?: string[];
+    room_subscriptions?: Record<string, MSC3575RoomSubscription>;
+    extensions?: object;
+
+    // query params
+    pos?: string;
+    timeout?: number;
+    clientTimeout?: number;
+}
+
+export interface MSC3575RoomData {
+    name: string;
+    required_state: IStateEvent[];
+    timeline: (IRoomEvent | IStateEvent)[];
+    notification_count?: number;
+    highlight_count?: number;
+    invite_state?: IStateEvent[];
+    initial?: boolean;
+    limited?: boolean;
+    is_dm?: boolean;
+    prev_batch?: string;
+}
+
+interface ListResponse {
+    count: number;
+    ops: Operation[];
+}
+
+interface BaseOperation {
+    op: string;
+}
+
+interface DeleteOperation extends BaseOperation {
+    op: "DELETE";
+    index: number;
+}
+
+interface InsertOperation extends BaseOperation {
+    op: "INSERT";
+    index: number;
+    room_id: string;
+}
+
+interface InvalidateOperation extends BaseOperation {
+    op: "INVALIDATE";
+    range: [number, number];
+}
+
+interface SyncOperation extends BaseOperation {
+    op: "SYNC";
+    range: [number, number];
+    room_ids: string[];
+}
+
+type Operation = DeleteOperation | InsertOperation | InvalidateOperation | SyncOperation;
+
+/**
+ * A complete Sliding Sync response
+ */
+export interface MSC3575SlidingSyncResponse {
+    pos: string;
+    lists: ListResponse[];
+    rooms: Record<string, MSC3575RoomData>;
+    extensions: object;
+}
+
+export enum SlidingSyncState {
+    /**
+     * Fired by SlidingSyncEvent.Lifecycle event immediately before processing the response.
+     */
+    RequestFinished = "FINISHED",
+    /**
+     * Fired by SlidingSyncEvent.Lifecycle event immediately after all room data listeners have been
+     * invoked, but before list listeners.
+     */
+    Complete = "COMPLETE",
+}
+
+/**
+ * Internal Class. SlidingList represents a single list in sliding sync. The list can have filters,
+ * multiple sliding windows, and maintains the index->room_id mapping.
+ */
+class SlidingList {
+    private list: MSC3575List;
+    private isModified: boolean;
+
+    // returned data
+    public roomIndexToRoomId: Record<number, string>;
+    public joinedCount: number;
+
+    /**
+     * Construct a new sliding list.
+     * @param {MSC3575List} list The range, sort and filter values to use for this list.
+     */
+    constructor(list: MSC3575List) {
+        this.replaceList(list);
+    }
+
+    /**
+     * Mark this list as modified or not. Modified lists will return sticky params with calls to getList.
+     * This is useful for the first time the list is sent, or if the list has changed in some way.
+     * @param modified True to mark this list as modified so all sticky parameters will be re-sent.
+     */
+    public setModified(modified: boolean): void {
+        this.isModified = modified;
+    }
+
+    /**
+     * Update the list range for this list. Does not affect modified status as list ranges are non-sticky.
+     * @param newRanges The new ranges for the list
+     */
+    public updateListRange(newRanges: number[][]): void {
+        this.list.ranges = JSON.parse(JSON.stringify(newRanges));
+    }
+
+    /**
+     * Replace list parameters. All fields will be replaced with the new list parameters.
+     * @param list The new list parameters
+     */
+    public replaceList(list: MSC3575List): void {
+        list.filters = list.filters || {};
+        list.ranges = list.ranges || [];
+        this.list = JSON.parse(JSON.stringify(list));
+        this.isModified = true;
+
+        // reset values as the join count may be very different (if filters changed) including the rooms
+        // (e.g. sort orders or sliding window ranges changed)
+
+        // the constantly changing sliding window ranges. Not an array for performance reasons
+        // E.g. tracking ranges 0-99, 500-599, we don't want to have a 600 element array
+        this.roomIndexToRoomId = {};
+        // the total number of joined rooms according to the server, always >= len(roomIndexToRoomId)
+        this.joinedCount = 0;
+    }
+
+    /**
+     * Return a copy of the list suitable for a request body.
+     * @param {boolean} forceIncludeAllParams True to forcibly include all params even if the list
+     * hasn't been modified. Callers may want to do this if they are modifying the list prior to calling
+     * updateList.
+     */
+    public getList(forceIncludeAllParams: boolean): MSC3575List {
+        let list = {
+            ranges: JSON.parse(JSON.stringify(this.list.ranges)),
+        };
+        if (this.isModified || forceIncludeAllParams) {
+            list = JSON.parse(JSON.stringify(this.list));
+        }
+        return list;
+    }
+
+    /**
+     * Check if a given index is within the list range. This is required even though the /sync API
+     * provides explicit updates with index positions because of the following situation:
+     *   0 1 2 3 4 5 6 7 8   indexes
+     *   a b c       d e f   COMMANDS: SYNC 0 2 a b c; SYNC 6 8 d e f;
+     *   a b c       d _ f   COMMAND: DELETE 7;
+     *   e a b c       d f   COMMAND: INSERT 0 e;
+     *   c=3 is wrong as we are not tracking it, ergo we need to see if `i` is in range else drop it
+     * @param i The index to check
+     * @returns True if the index is within a sliding window
+     */
+    public isIndexInRange(i: number): boolean {
+        for (const r of this.list.ranges) {
+            if (r[0] <= i && i <= r[1]) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
+
+/**
+ * When onResponse extensions should be invoked: before or after processing the main response.
+ */
+export enum ExtensionState {
+    // Call onResponse before processing the response body. This is useful when your extension is
+    // preparing the ground for the response body e.g. processing to-device messages before the
+    // encrypted event arrives.
+    PreProcess = "ExtState.PreProcess",
+    // Call onResponse after processing the response body. This is useful when your extension is
+    // decorating data from the client, and you rely on MatrixClient.getRoom returning the Room object
+    // e.g. room account data.
+    PostProcess = "ExtState.PostProcess",
+}
+
+/**
+ * An interface that must be satisfied to register extensions
+ */
+export interface Extension {
+    /**
+     * The extension name to go under 'extensions' in the request body.
+     * @returns The JSON key.
+     */
+    name(): string;
+    /**
+     * A function which is called when the request JSON is being formed.
+     * Returns the data to insert under this key.
+     * @param isInitial True when this is part of the initial request (send sticky params)
+     * @returns The request JSON to send.
+     */
+    onRequest(isInitial: boolean): object;
+    /**
+     * A function which is called when there is response JSON under this extension.
+     * @param data The response JSON under the extension name.
+     */
+    onResponse(data: object);
+    /**
+     * Controls when onResponse should be called.
+     * @returns The state when it should be called.
+     */
+    when(): ExtensionState;
+}
+
+/**
+ * Events which can be fired by the SlidingSync class. These are designed to provide different levels
+ * of information when processing sync responses.
+ *  - RoomData: concerns rooms, useful for SlidingSyncSdk to update its knowledge of rooms.
+ *  - Lifecycle: concerns callbacks at various well-defined points in the sync process.
+ *  - List: concerns lists, useful for UI layers to re-render room lists.
+ * Specifically, the order of event invocation is:
+ *  - Lifecycle (state=RequestFinished)
+ *  - RoomData (N times)
+ *  - Lifecycle (state=Complete)
+ *  - List (at most once per list)
+ */
+export enum SlidingSyncEvent {
+    /**
+     * This event fires when there are updates for a room. Fired as and when rooms are encountered
+     * in the response.
+     */
+    RoomData = "SlidingSync.RoomData",
+    /**
+     * This event fires at various points in the /sync loop lifecycle.
+     *  - SlidingSyncState.RequestFinished: Fires after we receive a valid response but before the
+     * response has been processed. Perform any pre-process steps here. If there was a problem syncing,
+     * `err` will be set (e.g network errors).
+     *  - SlidingSyncState.Complete: Fires after all SlidingSyncEvent.RoomData have been fired but before
+     * SlidingSyncEvent.List.
+     */
+    Lifecycle = "SlidingSync.Lifecycle",
+    /**
+     * This event fires whenever there has been a change to this list index. It fires exactly once
+     * per list, even if there were multiple operations for the list.
+     * It fires AFTER Lifecycle and RoomData events.
+     */
+    List = "SlidingSync.List",
+}
+
+export type SlidingSyncEventHandlerMap = {
+    [SlidingSyncEvent.RoomData]: (roomId: string, roomData: MSC3575RoomData) => void;
+    [SlidingSyncEvent.Lifecycle]: (state: SlidingSyncState, resp: MSC3575SlidingSyncResponse, err: Error) => void;
+    [SlidingSyncEvent.List]: (
+        listIndex: number, joinedCount: number, roomIndexToRoomId: Record<number, string>,
+    ) => void;
+};
+
+/**
+ * SlidingSync is a high-level data structure which controls the majority of sliding sync.
+ * It has no hooks into JS SDK except for needing a MatrixClient to perform the HTTP request.
+ * This means this class (and everything it uses) can be used in isolation from JS SDK if needed.
+ * To hook this up with the JS SDK, you need to use SlidingSyncSdk.
+ */
+export class SlidingSync extends TypedEventEmitter<SlidingSyncEvent, SlidingSyncEventHandlerMap> {
+    private lists: SlidingList[];
+    private listModifiedCount = 0;
+    private terminated = false;
+    // flag set when resend() is called because we cannot rely on detecting AbortError in JS SDK :(
+    private needsResend = false;
+    // map of extension name to req/resp handler
+    private extensions: Record<string, Extension> = {};
+
+    private desiredRoomSubscriptions = new Set<string>(); // the *desired* room subscriptions
+    private confirmedRoomSubscriptions = new Set<string>();
+
+    private pendingReq?: IAbortablePromise<MSC3575SlidingSyncResponse>;
+
+    /**
+     * Create a new sliding sync instance
+     * @param {string} proxyBaseUrl The base URL of the sliding sync proxy
+     * @param {MSC3575List[]} lists The lists to use for sliding sync.
+     * @param {MSC3575RoomSubscription} roomSubscriptionInfo The params to use for room subscriptions.
+     * @param {MatrixClient} client The client to use for /sync calls.
+     * @param {number} timeoutMS The number of milliseconds to wait for a response.
+     */
+    constructor(
+        private readonly proxyBaseUrl: string,
+        lists: MSC3575List[],
+        private roomSubscriptionInfo: MSC3575RoomSubscription,
+        private readonly client: MatrixClient,
+        private readonly timeoutMS: number,
+    ) {
+        super();
+        this.lists = lists.map((l) => new SlidingList(l));
+    }
+
+    /**
+     * Get the length of the sliding lists.
+     * @returns The number of lists in the sync request
+     */
+    public listLength(): number {
+        return this.lists.length;
+    }
+
+    /**
+     * Get the room data for a list.
+     * @param index The list index
+     * @returns The list data which contains the rooms in this list
+     */
+    public getListData(index: number): {joinedCount: number, roomIndexToRoomId: Record<number, string>} {
+        if (!this.lists[index]) {
+            return null;
+        }
+        return {
+            joinedCount: this.lists[index].joinedCount,
+            roomIndexToRoomId: Object.assign({}, this.lists[index].roomIndexToRoomId),
+        };
+    }
+
+    /**
+     * Get the full list parameters for a list index. This function is provided for callers to use
+     * in conjunction with setList to update fields on an existing list.
+     * @param index The list index to get the list for.
+     * @returns A copy of the list or undefined.
+     */
+    public getList(index: number): MSC3575List {
+        if (!this.lists[index]) {
+            return null;
+        }
+        return this.lists[index].getList(true);
+    }
+
+    /**
+     * Set new ranges for an existing list. Calling this function when _only_ the ranges have changed
+     * is more efficient than calling setList(index,list) as this function won't resend sticky params,
+     * whereas setList always will.
+     * @param index The list index to modify
+     * @param ranges The new ranges to apply.
+     */
+    public setListRanges(index: number, ranges: number[][]): void {
+        this.lists[index].updateListRange(ranges);
+        this.resend();
+    }
+
+    /**
+     * Add or replace a list. Calling this function will interrupt the /sync request to resend new
+     * lists.
+     * @param index The index to modify
+     * @param list The new list parameters.
+     */
+    public setList(index: number, list: MSC3575List): void {
+        if (this.lists[index]) {
+            this.lists[index].replaceList(list);
+        } else {
+            this.lists[index] = new SlidingList(list);
+        }
+        this.listModifiedCount += 1;
+        this.resend();
+    }
+
+    /**
+     * Get the room subscriptions for the sync API.
+     * @returns A copy of the desired room subscriptions.
+     */
+    public getRoomSubscriptions(): Set<string> {
+        return new Set(Array.from(this.desiredRoomSubscriptions));
+    }
+
+    /**
+     * Modify the room subscriptions for the sync API. Calling this function will interrupt the
+     * /sync request to resend new subscriptions. If the /sync stream has not started, this will
+     * prepare the room subscriptions for when start() is called.
+     * @param s The new desired room subscriptions.
+     */
+    public modifyRoomSubscriptions(s: Set<string>) {
+        this.desiredRoomSubscriptions = s;
+        this.resend();
+    }
+
+    /**
+     * Modify which events to retrieve for room subscriptions. Invalidates all room subscriptions
+     * such that they will be sent up afresh.
+     * @param rs The new room subscription fields to fetch.
+     */
+    public modifyRoomSubscriptionInfo(rs: MSC3575RoomSubscription): void {
+        this.roomSubscriptionInfo = rs;
+        this.confirmedRoomSubscriptions = new Set<string>();
+        this.resend();
+    }
+
+    /**
+     * Register an extension to send with the /sync request.
+     * @param ext The extension to register.
+     */
+    public registerExtension(ext: Extension): void {
+        if (this.extensions[ext.name()]) {
+            throw new Error(`registerExtension: ${ext.name()} already exists as an extension`);
+        }
+        this.extensions[ext.name()] = ext;
+    }
+
+    private getExtensionRequest(isInitial: boolean): object {
+        const ext = {};
+        Object.keys(this.extensions).forEach((extName) => {
+            ext[extName] = this.extensions[extName].onRequest(isInitial);
+        });
+        return ext;
+    }
+
+    private onPreExtensionsResponse(ext: object): void {
+        Object.keys(ext).forEach((extName) => {
+            if (this.extensions[extName].when() == ExtensionState.PreProcess) {
+                this.extensions[extName].onResponse(ext[extName]);
+            }
+        });
+    }
+
+    private onPostExtensionsResponse(ext: object): void {
+        Object.keys(ext).forEach((extName) => {
+            if (this.extensions[extName].when() == ExtensionState.PostProcess) {
+                this.extensions[extName].onResponse(ext[extName]);
+            }
+        });
+    }
+
+    /**
+     * Invoke all attached room data listeners.
+     * @param {string} roomId The room which received some data.
+     * @param {object} roomData The raw sliding sync response JSON.
+     */
+    private invokeRoomDataListeners(roomId: string, roomData: MSC3575RoomData): void {
+        if (!roomData.required_state) { roomData.required_state = []; }
+        if (!roomData.timeline) { roomData.timeline = []; }
+        this.emit(SlidingSyncEvent.RoomData, roomId, roomData);
+    }
+
+    /**
+     * Invoke all attached lifecycle listeners.
+     * @param {SlidingSyncState} state The Lifecycle state
+     * @param {object} resp The raw sync response JSON
+     * @param {Error?} err Any error that occurred when making the request e.g. network errors.
+     */
+    private invokeLifecycleListeners(state: SlidingSyncState, resp: MSC3575SlidingSyncResponse, err?: Error): void {
+        this.emit(SlidingSyncEvent.Lifecycle, state, resp, err);
+    }
+
+    private processListOps(list: ListResponse, listIndex: number): void {
+        let gapIndex = -1;
+        list.ops.forEach((op: Operation) => {
+            switch (op.op) {
+                case "DELETE": {
+                    logger.debug("DELETE", listIndex, op.index, ";");
+                    delete this.lists[listIndex].roomIndexToRoomId[op.index];
+                    gapIndex = op.index;
+                    break;
+                }
+                case "INSERT": {
+                    logger.debug(
+                        "INSERT",
+                        listIndex,
+                        op.index,
+                        op.room_id,
+                        ";",
+                    );
+                    if (this.lists[listIndex].roomIndexToRoomId[op.index]) {
+                        // something is in this space, shift items out of the way
+                        if (gapIndex < 0) {
+                            logger.debug(
+                                "cannot work out where gap is, INSERT without previous DELETE! List: ",
+                                listIndex,
+                            );
+                            return;
+                        }
+                        //  0,1,2,3  index
+                        // [A,B,C,D]
+                        //   DEL 3
+                        // [A,B,C,_]
+                        //   INSERT E 0
+                        // [E,A,B,C]
+                        // gapIndex=3, op.index=0
+                        if (gapIndex > op.index) {
+                            // the gap is further down the list, shift every element to the right
+                            // starting at the gap so we can just shift each element in turn:
+                            // [A,B,C,_] gapIndex=3, op.index=0
+                            // [A,B,C,C] i=3
+                            // [A,B,B,C] i=2
+                            // [A,A,B,C] i=1
+                            // Terminate. We'll assign into op.index next.
+                            for (let i = gapIndex; i > op.index; i--) {
+                                if (this.lists[listIndex].isIndexInRange(i)) {
+                                    this.lists[listIndex].roomIndexToRoomId[i] =
+                                        this.lists[listIndex].roomIndexToRoomId[
+                                            i - 1
+                                        ];
+                                }
+                            }
+                        } else if (gapIndex < op.index) {
+                            // the gap is further up the list, shift every element to the left
+                            // starting at the gap so we can just shift each element in turn
+                            for (let i = gapIndex; i < op.index; i++) {
+                                if (this.lists[listIndex].isIndexInRange(i)) {
+                                    this.lists[listIndex].roomIndexToRoomId[i] =
+                                        this.lists[listIndex].roomIndexToRoomId[
+                                            i + 1
+                                        ];
+                                }
+                            }
+                        }
+                    }
+                    this.lists[listIndex].roomIndexToRoomId[op.index] = op.room_id;
+                    break;
+                }
+                case "INVALIDATE": {
+                    const startIndex = op.range[0];
+                    for (let i = startIndex; i <= op.range[1]; i++) {
+                        delete this.lists[listIndex].roomIndexToRoomId[i];
+                    }
+                    logger.debug(
+                        "INVALIDATE",
+                        listIndex,
+                        op.range[0],
+                        op.range[1],
+                        ";",
+                    );
+                    break;
+                }
+                case "SYNC": {
+                    const startIndex = op.range[0];
+                    for (let i = startIndex; i <= op.range[1]; i++) {
+                        const roomId = op.room_ids[i - startIndex];
+                        if (!roomId) {
+                            break; // we are at the end of list
+                        }
+                        this.lists[listIndex].roomIndexToRoomId[i] = roomId;
+                    }
+                    logger.debug(
+                        "SYNC",
+                        listIndex,
+                        op.range[0],
+                        op.range[1],
+                        op.room_ids.join(" "),
+                        ";",
+                    );
+                    break;
+                }
+            }
+        });
+    }
+
+    /**
+     * Resend a Sliding Sync request. Used when something has changed in the request.
+     */
+    public resend(): void {
+        this.needsResend = true;
+        this.pendingReq?.abort();
+    }
+
+    /**
+     * Stop syncing with the server.
+     */
+    public stop(): void {
+        this.terminated = true;
+        this.pendingReq?.abort();
+        // remove all listeners so things can be GC'd
+        this.removeAllListeners(SlidingSyncEvent.Lifecycle);
+        this.removeAllListeners(SlidingSyncEvent.List);
+        this.removeAllListeners(SlidingSyncEvent.RoomData);
+    }
+
+    /**
+     * Start syncing with the server. Blocks until stopped.
+     */
+    public async start() {
+        let currentPos: string;
+        while (!this.terminated) {
+            this.needsResend = false;
+            let doNotUpdateList = false;
+            let resp: MSC3575SlidingSyncResponse;
+            try {
+                const listModifiedCount = this.listModifiedCount;
+                const reqBody: MSC3575SlidingSyncRequest = {
+                    lists: this.lists.map((l) => {
+                        return l.getList(false);
+                    }),
+                    pos: currentPos,
+                    timeout: this.timeoutMS,
+                    clientTimeout: this.timeoutMS + BUFFER_PERIOD_MS,
+                    extensions: this.getExtensionRequest(currentPos === undefined),
+                };
+                // check if we are (un)subscribing to a room and modify request this one time for it
+                const newSubscriptions = difference(this.desiredRoomSubscriptions, this.confirmedRoomSubscriptions);
+                const unsubscriptions = difference(this.confirmedRoomSubscriptions, this.desiredRoomSubscriptions);
+                if (unsubscriptions.size > 0) {
+                    reqBody.unsubscribe_rooms = Array.from(unsubscriptions);
+                }
+                if (newSubscriptions.size > 0) {
+                    reqBody.room_subscriptions = {};
+                    for (const roomId of newSubscriptions) {
+                        reqBody.room_subscriptions[roomId] = this.roomSubscriptionInfo;
+                    }
+                }
+                this.pendingReq = this.client.slidingSync(reqBody, this.proxyBaseUrl);
+                resp = await this.pendingReq;
+                logger.debug(resp);
+                currentPos = resp.pos;
+                // update what we think we're subscribed to.
+                for (const roomId of newSubscriptions) {
+                    this.confirmedRoomSubscriptions.add(roomId);
+                }
+                for (const roomId of unsubscriptions) {
+                    this.confirmedRoomSubscriptions.delete(roomId);
+                }
+                if (listModifiedCount !== this.listModifiedCount) {
+                    // the lists have been modified whilst we were waiting for 'await' to return, but the abort()
+                    // call did nothing. It is NOT SAFE to modify the list array now. We'll process the response but
+                    // not update list pointers.
+                    logger.debug("list modified during await call, not updating list");
+                    doNotUpdateList = true;
+                }
+                // mark all these lists as having been sent as sticky so we don't keep sending sticky params
+                this.lists.forEach((l) => {
+                    l.setModified(false);
+                });
+                // set default empty values so we don't need to null check
+                resp.lists = resp.lists || [];
+                resp.rooms = resp.rooms || {};
+                resp.extensions = resp.extensions || {};
+                resp.lists.forEach((val, i) => {
+                    this.lists[i].joinedCount = val.count;
+                });
+                this.invokeLifecycleListeners(
+                    SlidingSyncState.RequestFinished,
+                    resp,
+                );
+            } catch (err) {
+                if (err.httpStatus) {
+                    this.invokeLifecycleListeners(
+                        SlidingSyncState.RequestFinished,
+                        null,
+                        err,
+                    );
+                    await sleep(3000);
+                } else if (this.needsResend || err === "aborted") {
+                    // don't sleep as we caused this error by abort()ing the request.
+                    // we check for 'aborted' because that's the error Jest returns and without it
+                    // we get warnings about not exiting fast enough.
+                    continue;
+                } else {
+                    logger.error(err);
+                    await sleep(3000);
+                }
+            }
+            if (!resp) {
+                continue;
+            }
+            this.onPreExtensionsResponse(resp.extensions);
+
+            Object.keys(resp.rooms).forEach((roomId) => {
+                this.invokeRoomDataListeners(
+                    roomId,
+                    resp.rooms[roomId],
+                );
+            });
+
+            const listIndexesWithUpdates: Set<number> = new Set();
+            if (!doNotUpdateList) {
+                resp.lists.forEach((list, listIndex) => {
+                    list.ops = list.ops || [];
+                    if (list.ops.length > 0) {
+                        listIndexesWithUpdates.add(listIndex);
+                    }
+                    this.processListOps(list, listIndex);
+                });
+            }
+            this.invokeLifecycleListeners(SlidingSyncState.Complete, resp);
+            this.onPostExtensionsResponse(resp.extensions);
+            listIndexesWithUpdates.forEach((i) => {
+                this.emit(
+                    SlidingSyncEvent.List,
+                    i, this.lists[i].joinedCount, Object.assign({}, this.lists[i].roomIndexToRoomId),
+                );
+            });
+        }
+    }
+}
+
+const difference = (setA: Set<string>, setB: Set<string>): Set<string> => {
+    const diff = new Set(setA);
+    for (const elem of setB) {
+        diff.delete(elem);
+    }
+    return diff;
+};

src/store/index.ts

@@ -17,7 +17,7 @@ limitations under the License.
 import { EventType } from "../@types/event";
 import { Room } from "../models/room";
 import { User } from "../models/user";
-import { MatrixEvent } from "../models/event";
+import { IEvent, MatrixEvent } from "../models/event";
 import { Filter } from "../filter";
 import { RoomSummary } from "../models/room-summary";
 import { IMinimalEvent, IRooms, ISyncResponse } from "../sync-accumulator";
@@ -218,4 +218,8 @@ export interface IStore {
     getClientOptions(): Promise<IStartClientOpts>;
 
     storeClientOptions(options: IStartClientOpts): Promise<void>;
+
+    getPendingEvents(roomId: string): Promise<Partial<IEvent>[]>;
+
+    setPendingEvents(roomId: string, events: Partial<IEvent>[]): Promise<void>;
 }

src/store/indexeddb-local-backend.ts

@@ -127,6 +127,8 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend {
     private db: IDBDatabase = null;
     private disconnected = true;
     private _isNewlyCreated = false;
+    private isPersisting = false;
+    private pendingUserPresenceData: UserTuple[] = [];
 
     /**
      * Does the actual reading from and writing to the indexeddb
@@ -266,7 +268,7 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend {
                 reject(err);
             };
         }).then((events) => {
-            logger.log(`LL: got ${events && events.length} membershipEvents from storage for room ${roomId} ...`);
+            logger.log(`LL: got ${events?.length} membershipEvents from storage for room ${roomId} ...`);
             return events;
         });
     }
@@ -401,11 +403,24 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend {
     public async syncToDatabase(userTuples: UserTuple[]): Promise<void> {
         const syncData = this.syncAccumulator.getJSON(true);
 
-        await Promise.all([
-            this.persistUserPresenceEvents(userTuples),
-            this.persistAccountData(syncData.accountData),
-            this.persistSyncData(syncData.nextBatch, syncData.roomsData),
-        ]);
+        if (this.isPersisting) {
+            logger.warn("Skipping syncToDatabase() as persist already in flight");
+            this.pendingUserPresenceData.push(...userTuples);
+            return;
+        } else {
+            userTuples.unshift(...this.pendingUserPresenceData);
+            this.isPersisting = true;
+        }
+
+        try {
+            await Promise.all([
+                this.persistUserPresenceEvents(userTuples),
+                this.persistAccountData(syncData.accountData),
+                this.persistSyncData(syncData.nextBatch, syncData.roomsData),
+            ]);
+        } finally {
+            this.isPersisting = false;
+        }
     }
 
     /**
@@ -427,7 +442,9 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend {
                 nextBatch,
                 roomsData,
             }); // put == UPSERT
-            return txnAsPromise(txn).then();
+            return txnAsPromise(txn).then(() => {
+                logger.log("Persisted sync data up to", nextBatch);
+            });
         });
     }