v19.3.0-rc.1

This will be rolled out again later with more accompanying UI adjustments, including clearer error messages and possibly the option to disable it per-room.

.eslintrc.js

@@ -9,6 +9,8 @@ module.exports = {
         browser: true,
         node: true,
     },
+    // NOTE: These rules are frozen and new rules should not be added here.
+    // New changes belong in https://github.com/matrix-org/eslint-plugin-matrix-org/
     rules: {
         "no-var": ["warn"],
         "prefer-rest-params": ["warn"],

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,13 @@
-<!-- Please read https://github.com/matrix-org/matrix-js-sdk/blob/develop/CONTRIBUTING.md before submitting your pull request -->
+<!-- Thanks for submitting a PR! Please ensure the following requirements are met in order for us to review your PR -->
 
-<!-- Include a Sign-Off as described in https://github.com/matrix-org/matrix-js-sdk/blob/develop/CONTRIBUTING.md#sign-off -->
+## Checklist
 
-<!-- To specify text for the changelog entry (otherwise the PR title will be used):
-Notes:
+* [ ] Tests written for new code (and old code if feasible)
+* [ ] Linter and other CI checks pass
+* [ ] Sign-off given on the changes (see [CONTRIBUTING.md](https://github.com/matrix-org/matrix-js-sdk/blob/develop/CONTRIBUTING.md))
+
+<!--
+If you would like to specify text for the changelog entry other than your PR title, add the following:
+
+Notes: Add super cool feature
 -->

.github/renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "github>matrix-org/renovate-config-element-web"
+  ]
+}

.github/workflows/notify-downstream.yaml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Notify matrix-react-sdk repo that a new SDK build is on develop so it can CI against it
-        uses: peter-evans/repository-dispatch@v1
+        uses: peter-evans/repository-dispatch@v2
         with:
           token: ${{ secrets.ELEMENT_BOT_TOKEN }}
           repository: ${{ matrix.repo }}

.github/workflows/pull_request.yaml

@@ -16,7 +16,6 @@ concurrency: ${{ github.workflow }}-${{ github.event.pull_request.head.ref }}
 jobs:
   changelog:
     name: Preview Changelog
-    if: github.event.action != 'synchronize'
     runs-on: ubuntu-latest
     steps:
       - uses: matrix-org/allchange@main
@@ -31,7 +30,7 @@ jobs:
       pull-requests: read
     steps:
       - name: Add notice
-        uses: actions/github-script@v5
+        uses: actions/github-script@v6
         if: contains(github.event.pull_request.labels.*.name, 'X-Blocked')
         with:
           script: |
@@ -53,7 +52,7 @@ jobs:
 
       - name: Add label
         if: ${{ steps.teams.outputs.isTeamMember == 'false' }}
-        uses: actions/github-script@v5
+        uses: actions/github-script@v6
         with:
           script: |
             github.rest.issues.addLabels({
@@ -72,7 +71,7 @@ jobs:
       github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Close pull request
-        uses: actions/github-script@v5
+        uses: actions/github-script@v6
         with:
           script: |
             github.rest.issues.createComment({

.github/workflows/static_analysis.yml

@@ -11,7 +11,7 @@ jobs:
     name: "Typescript Syntax Check"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - uses: actions/setup-node@v3
         with:
@@ -27,7 +27,7 @@ jobs:
     name: "ESLint"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - uses: actions/setup-node@v3
         with:
@@ -43,7 +43,7 @@ jobs:
     name: "JSDoc Checker"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - uses: actions/setup-node@v3
         with:

.github/workflows/tests.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Yarn cache
         uses: actions/setup-node@v3
@@ -29,7 +29,7 @@ jobs:
         run: "yarn coverage --ci --reporters github-actions"
 
       - name: Upload Artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: coverage
           path: |

CHANGELOG.md

@@ -1,3 +1,49 @@
+Changes in [19.3.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v19.3.0-rc.1) (2022-08-09)
+============================================================================================================
+
+## ✨ Features
+ * Emit an event when the client receives TURN servers ([\#2529](https://github.com/matrix-org/matrix-js-sdk/pull/2529)).
+ * Add support for stable prefixes for MSC2285 ([\#2524](https://github.com/matrix-org/matrix-js-sdk/pull/2524)).
+ * Always block sending keys to unverified devices of verified users ([\#2562](https://github.com/matrix-org/matrix-js-sdk/pull/2562)). Contributed by @duxovni.
+ * Remove stream-replacement ([\#2551](https://github.com/matrix-org/matrix-js-sdk/pull/2551)).
+ * Add support for sending user-defined encrypted to-device messages ([\#2528](https://github.com/matrix-org/matrix-js-sdk/pull/2528)).
+ * Retry to-device messages ([\#2549](https://github.com/matrix-org/matrix-js-sdk/pull/2549)). Fixes vector-im/element-web#12851.
+ * Sliding sync: add missing filters from latest MSC ([\#2555](https://github.com/matrix-org/matrix-js-sdk/pull/2555)).
+ * Use stable prefixes for MSC3827 ([\#2537](https://github.com/matrix-org/matrix-js-sdk/pull/2537)).
+
+## 🐛 Bug Fixes
+ * Don't load the sync accumulator if there's already a sync persist in flight ([\#2569](https://github.com/matrix-org/matrix-js-sdk/pull/2569)).
+
+Changes in [19.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v19.2.0) (2022-08-02)
+==================================================================================================
+
+## 🦖 Deprecations
+ * Remove unstable support for `m.room_key.withheld` ([\#2512](https://github.com/matrix-org/matrix-js-sdk/pull/2512)). Fixes #2233.
+
+## ✨ Features
+ * Sliding sync: add missing filters from latest MSC ([\#2555](https://github.com/matrix-org/matrix-js-sdk/pull/2555)).
+ * Use stable prefixes for MSC3827 ([\#2537](https://github.com/matrix-org/matrix-js-sdk/pull/2537)).
+ * Add support for MSC3575: Sliding Sync ([\#2242](https://github.com/matrix-org/matrix-js-sdk/pull/2242)).
+
+## 🐛 Bug Fixes
+ * Correct the units in TURN servers expiry documentation ([\#2520](https://github.com/matrix-org/matrix-js-sdk/pull/2520)).
+ * Re-insert room IDs when decrypting bundled redaction events returned by `/sync` ([\#2531](https://github.com/matrix-org/matrix-js-sdk/pull/2531)). Contributed by @duxovni.
+
+Changes in [19.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v19.1.0) (2022-07-26)
+==================================================================================================
+
+## 🦖 Deprecations
+ * Remove MSC3244 support ([\#2504](https://github.com/matrix-org/matrix-js-sdk/pull/2504)).
+
+## ✨ Features
+ * `room` now exports `KNOWN_SAFE_ROOM_VERSION` ([\#2474](https://github.com/matrix-org/matrix-js-sdk/pull/2474)).
+
+## 🐛 Bug Fixes
+ * Don't crash with undefined room in `processBeaconEvents()` ([\#2500](https://github.com/matrix-org/matrix-js-sdk/pull/2500)). Fixes #2494.
+ * Properly re-insert room ID in bundled thread relation messages from sync ([\#2505](https://github.com/matrix-org/matrix-js-sdk/pull/2505)). Fixes vector-im/element-web#22094. Contributed by @duxovni.
+ * Actually store the identity server in the client when given as an option ([\#2503](https://github.com/matrix-org/matrix-js-sdk/pull/2503)). Fixes vector-im/element-web#22757.
+ * Fix call.collectCallStats() ([\#2480](https://github.com/matrix-org/matrix-js-sdk/pull/2480)).
+
 Changes in [19.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v19.0.0) (2022-07-05)
 ==================================================================================================
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-js-sdk",
-  "version": "19.0.0",
+  "version": "19.3.0-rc.1",
   "description": "Matrix Client-Server SDK for Javascript",
   "engines": {
     "node": ">=12.9.0"
@@ -56,11 +56,11 @@
     "@babel/runtime": "^7.12.5",
     "another-json": "^0.2.0",
     "browser-request": "^0.3.3",
-    "bs58": "^4.0.1",
+    "bs58": "^5.0.0",
     "content-type": "^1.0.4",
     "loglevel": "^1.7.1",
     "matrix-events-sdk": "^0.0.1-beta.7",
-    "p-retry": "^4.5.0",
+    "p-retry": "4",
     "qs": "^6.9.6",
     "request": "^2.88.2",
     "unhomoglyph": "^1.0.6"
@@ -78,11 +78,11 @@
     "@babel/preset-env": "^7.12.11",
     "@babel/preset-typescript": "^7.12.7",
     "@babel/register": "^7.12.10",
-    "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz",
+    "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.12.tgz",
     "@types/bs58": "^4.0.1",
     "@types/content-type": "^1.1.5",
-    "@types/jest": "^27.0.0",
-    "@types/node": "12",
+    "@types/jest": "^28.0.0",
+    "@types/node": "16",
     "@types/request": "^2.48.5",
     "@typescript-eslint/eslint-plugin": "^5.6.0",
     "@typescript-eslint/parser": "^5.6.0",
@@ -92,17 +92,17 @@
     "better-docs": "^2.4.0-beta.9",
     "browserify": "^17.0.0",
     "docdash": "^1.2.0",
-    "eslint": "8.16.0",
+    "eslint": "8.20.0",
     "eslint-config-google": "^0.14.0",
     "eslint-plugin-import": "^2.25.4",
     "eslint-plugin-matrix-org": "^0.5.0",
-    "exorcist": "^1.0.1",
-    "fake-indexeddb": "^3.1.2",
+    "exorcist": "^2.0.0",
+    "fake-indexeddb": "^4.0.0",
     "jest": "^28.0.0",
     "jest-localstorage-mock": "^2.4.6",
     "jest-sonar-reporter": "^2.0.0",
     "jsdoc": "^3.6.6",
-    "matrix-mock-request": "^2.0.1",
+    "matrix-mock-request": "^2.1.2",
     "rimraf": "^3.0.2",
     "terser": "^5.5.1",
     "tsify": "^5.0.2",

release.sh

@@ -121,14 +121,16 @@ echo $release | grep -q '-' && prerelease=1
 
 if [ $prerelease -eq 1 ]; then
     echo Making a PRE-RELEASE
+else
+    read -p "Making a FINAL RELEASE, press enter to continue " REPLY
 fi
 
-# we might already be on the release branch, in which case, yay
-# If we're on any branch starting with 'release', we don't create
-# a separate release branch (this allows us to use the same
+# We might already be on the release branch, in which case, yay
+# If we're on any branch starting with 'release', or the staging branch
+# we don't create a separate release branch (this allows us to use the same
 # release branch for releases and release candidates).
 curbranch=$(git symbolic-ref --short HEAD)
-if [[ "$curbranch" != release* ]]; then
+if [[ "$curbranch" != release* && "$curbranch" != "staging" ]]; then
     echo "Creating release branch"
     git checkout -b "$rel_branch"
 else

renovate.json

@@ -1,16 +0,0 @@
-{
-    "extends": [
-        "config:base",
-        ":dependencyDashboardApproval"
-    ],
-    "labels": ["T-Task", "Dependencies"],
-    "lockFileMaintenance": { "enabled": true },
-    "groupName": "all",
-    "packageRules": [{
-        "matchFiles": ["package.json"],
-        "rangeStrategy": "update-lockfile"
-    }],
-    "platformAutomerge": true,
-    "automerge": true,
-    "automergeType": "pr"
-}

spec/TestClient.ts

@@ -39,8 +39,8 @@ import { IKeysUploadResponse, IUploadKeysRequest } from '../src/client';
 export class TestClient {
     public readonly httpBackend: MockHttpBackend;
     public readonly client: MatrixClient;
-    private deviceKeys: IDeviceKeys;
-    private oneTimeKeys: Record<string, IOneTimeKey>;
+    public deviceKeys: IDeviceKeys;
+    public oneTimeKeys: Record<string, IOneTimeKey>;
 
     constructor(
         public readonly userId?: string,
@@ -236,4 +236,8 @@ export class TestClient {
     public isFallbackICEServerAllowed(): boolean {
         return true;
     }
+
+    public getUserId(): string {
+        return this.userId;
+    }
 }

spec/integ/matrix-client-crypto.spec.js

@@ -1,758 +0,0 @@
-/*
-Copyright 2016 OpenMarket Ltd
-Copyright 2017 Vector Creations Ltd
-Copyright 2018 New Vector Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/* This file consists of a set of integration tests which try to simulate
- * communication via an Olm-encrypted room between two users, Alice and Bob.
- *
- * Note that megolm (group) conversation is not tested here.
- *
- * See also `megolm.spec.js`.
- */
-
-// load olm before the sdk if possible
-import '../olm-loader';
-
-import { logger } from '../../src/logger';
-import * as testUtils from "../test-utils/test-utils";
-import { TestClient } from "../TestClient";
-import { CRYPTO_ENABLED } from "../../src/client";
-
-let aliTestClient;
-const roomId = "!room:localhost";
-const aliUserId = "@ali:localhost";
-const aliDeviceId = "zxcvb";
-const aliAccessToken = "aseukfgwef";
-let bobTestClient;
-const bobUserId = "@bob:localhost";
-const bobDeviceId = "bvcxz";
-const bobAccessToken = "fewgfkuesa";
-let aliMessages;
-let bobMessages;
-
-function bobUploadsDeviceKeys() {
-    bobTestClient.expectDeviceKeyUpload();
-    return Promise.all([
-        bobTestClient.client.uploadKeys(),
-        bobTestClient.httpBackend.flush(),
-    ]).then(() => {
-        expect(Object.keys(bobTestClient.deviceKeys).length).not.toEqual(0);
-    });
-}
-
-/**
- * Set an expectation that ali will query bobs keys; then flush the http request.
- *
- * @return {promise} resolves once the http request has completed.
- */
-function expectAliQueryKeys() {
-    // can't query keys before bob has uploaded them
-    expect(bobTestClient.deviceKeys).toBeTruthy();
-
-    const bobKeys = {};
-    bobKeys[bobDeviceId] = bobTestClient.deviceKeys;
-    aliTestClient.httpBackend.when("POST", "/keys/query")
-            .respond(200, function(path, content) {
-        expect(content.device_keys[bobUserId]).toEqual(
-            [],
-            "Expected Alice to key query for " + bobUserId + ", got " +
-            Object.keys(content.device_keys),
-        );
-        const result = {};
-        result[bobUserId] = bobKeys;
-        return { device_keys: result };
-    });
-    return aliTestClient.httpBackend.flush("/keys/query", 1);
-}
-
-/**
- * Set an expectation that bob will query alis keys; then flush the http request.
- *
- * @return {promise} which resolves once the http request has completed.
- */
-function expectBobQueryKeys() {
-    // can't query keys before ali has uploaded them
-    expect(aliTestClient.deviceKeys).toBeTruthy();
-
-    const aliKeys = {};
-    aliKeys[aliDeviceId] = aliTestClient.deviceKeys;
-    logger.log("query result will be", aliKeys);
-
-    bobTestClient.httpBackend.when(
-        "POST", "/keys/query",
-    ).respond(200, function(path, content) {
-        expect(content.device_keys[aliUserId]).toEqual(
-            [],
-            "Expected Bob to key query for " + aliUserId + ", got " +
-            Object.keys(content.device_keys),
-        );
-        const result = {};
-        result[aliUserId] = aliKeys;
-        return { device_keys: result };
-    });
-    return bobTestClient.httpBackend.flush("/keys/query", 1);
-}
-
-/**
- * Set an expectation that ali will claim one of bob's keys; then flush the http request.
- *
- * @return {promise} resolves once the http request has completed.
- */
-function expectAliClaimKeys() {
-    return bobTestClient.awaitOneTimeKeyUpload().then((keys) => {
-        aliTestClient.httpBackend.when(
-            "POST", "/keys/claim",
-        ).respond(200, function(path, content) {
-            const claimType = content.one_time_keys[bobUserId][bobDeviceId];
-            expect(claimType).toEqual("signed_curve25519");
-            let keyId = null;
-            for (keyId in keys) {
-                if (bobTestClient.oneTimeKeys.hasOwnProperty(keyId)) {
-                    if (keyId.indexOf(claimType + ":") === 0) {
-                        break;
-                    }
-                }
-            }
-            const result = {};
-            result[bobUserId] = {};
-            result[bobUserId][bobDeviceId] = {};
-            result[bobUserId][bobDeviceId][keyId] = keys[keyId];
-            return { one_time_keys: result };
-        });
-    }).then(() => {
-        // it can take a while to process the key query, so give it some extra
-        // time, and make sure the claim actually happens rather than ploughing on
-        // confusingly.
-        return aliTestClient.httpBackend.flush("/keys/claim", 1, 500).then((r) => {
-            expect(r).toEqual(1, "Ali did not claim Bob's keys");
-        });
-    });
-}
-
-function aliDownloadsKeys() {
-    // can't query keys before bob has uploaded them
-    expect(bobTestClient.getSigningKey()).toBeTruthy();
-
-    const p1 = aliTestClient.client.downloadKeys([bobUserId]).then(function() {
-        return aliTestClient.client.getStoredDevicesForUser(bobUserId);
-    }).then((devices) => {
-        expect(devices.length).toEqual(1);
-        expect(devices[0].deviceId).toEqual("bvcxz");
-    });
-    const p2 = expectAliQueryKeys();
-
-    // check that the localStorage is updated as we expect (not sure this is
-    // an integration test, but meh)
-    return Promise.all([p1, p2]).then(() => {
-        return aliTestClient.client.crypto.deviceList.saveIfDirty();
-    }).then(() => {
-        aliTestClient.client.cryptoStore.getEndToEndDeviceData(null, (data) => {
-            const devices = data.devices[bobUserId];
-            expect(devices[bobDeviceId].keys).toEqual(bobTestClient.deviceKeys.keys);
-            expect(devices[bobDeviceId].verified).
-                toBe(0); // DeviceVerification.UNVERIFIED
-        });
-    });
-}
-
-function aliEnablesEncryption() {
-    return aliTestClient.client.setRoomEncryption(roomId, {
-        algorithm: "m.olm.v1.curve25519-aes-sha2",
-    }).then(function() {
-        expect(aliTestClient.client.isRoomEncrypted(roomId)).toBeTruthy();
-    });
-}
-
-function bobEnablesEncryption() {
-    return bobTestClient.client.setRoomEncryption(roomId, {
-        algorithm: "m.olm.v1.curve25519-aes-sha2",
-    }).then(function() {
-       expect(bobTestClient.client.isRoomEncrypted(roomId)).toBeTruthy();
-    });
-}
-
-/**
- * Ali sends a message, first claiming e2e keys. Set the expectations and
- * check the results.
- *
- * @return {promise} which resolves to the ciphertext for Bob's device.
- */
-function aliSendsFirstMessage() {
-    return Promise.all([
-        sendMessage(aliTestClient.client),
-        expectAliQueryKeys()
-            .then(expectAliClaimKeys)
-            .then(expectAliSendMessageRequest),
-    ]).then(function([_, ciphertext]) {
-        return ciphertext;
-    });
-}
-
-/**
- * Ali sends a message without first claiming e2e keys. Set the expectations
- * and check the results.
- *
- * @return {promise} which resolves to the ciphertext for Bob's device.
- */
-function aliSendsMessage() {
-    return Promise.all([
-        sendMessage(aliTestClient.client),
-        expectAliSendMessageRequest(),
-    ]).then(function([_, ciphertext]) {
-        return ciphertext;
-    });
-}
-
-/**
- * Bob sends a message, first querying (but not claiming) e2e keys. Set the
- * expectations and check the results.
- *
- * @return {promise} which resolves to the ciphertext for Ali's device.
- */
-function bobSendsReplyMessage() {
-    return Promise.all([
-        sendMessage(bobTestClient.client),
-        expectBobQueryKeys()
-            .then(expectBobSendMessageRequest),
-    ]).then(function([_, ciphertext]) {
-        return ciphertext;
-    });
-}
-
-/**
- * Set an expectation that Ali will send a message, and flush the request
- *
- * @return {promise} which resolves to the ciphertext for Bob's device.
- */
-function expectAliSendMessageRequest() {
-    return expectSendMessageRequest(aliTestClient.httpBackend).then(function(content) {
-        aliMessages.push(content);
-        expect(Object.keys(content.ciphertext)).toEqual([bobTestClient.getDeviceKey()]);
-        const ciphertext = content.ciphertext[bobTestClient.getDeviceKey()];
-        expect(ciphertext).toBeTruthy();
-        return ciphertext;
-    });
-}
-
-/**
- * Set an expectation that Bob will send a message, and flush the request
- *
- * @return {promise} which resolves to the ciphertext for Bob's device.
- */
-function expectBobSendMessageRequest() {
-    return expectSendMessageRequest(bobTestClient.httpBackend).then(function(content) {
-        bobMessages.push(content);
-        const aliKeyId = "curve25519:" + aliDeviceId;
-        const aliDeviceCurve25519Key = aliTestClient.deviceKeys.keys[aliKeyId];
-        expect(Object.keys(content.ciphertext)).toEqual([aliDeviceCurve25519Key]);
-        const ciphertext = content.ciphertext[aliDeviceCurve25519Key];
-        expect(ciphertext).toBeTruthy();
-        return ciphertext;
-    });
-}
-
-function sendMessage(client) {
-    return client.sendMessage(
-        roomId, { msgtype: "m.text", body: "Hello, World" },
-    );
-}
-
-function expectSendMessageRequest(httpBackend) {
-    const path = "/send/m.room.encrypted/";
-    const prom = new Promise((resolve) => {
-        httpBackend.when("PUT", path).respond(200, function(path, content) {
-            resolve(content);
-            return {
-                event_id: "asdfgh",
-            };
-        });
-    });
-
-    // it can take a while to process the key query
-    return httpBackend.flush(path, 1).then(() => prom);
-}
-
-function aliRecvMessage() {
-    const message = bobMessages.shift();
-    return recvMessage(
-        aliTestClient.httpBackend, aliTestClient.client, bobUserId, message,
-    );
-}
-
-function bobRecvMessage() {
-    const message = aliMessages.shift();
-    return recvMessage(
-        bobTestClient.httpBackend, bobTestClient.client, aliUserId, message,
-    );
-}
-
-function recvMessage(httpBackend, client, sender, message) {
-    const syncData = {
-        next_batch: "x",
-        rooms: {
-            join: {
-
-            },
-        },
-    };
-    syncData.rooms.join[roomId] = {
-        timeline: {
-            events: [
-                testUtils.mkEvent({
-                    type: "m.room.encrypted",
-                    room: roomId,
-                    content: message,
-                    sender: sender,
-                }),
-            ],
-        },
-    };
-    httpBackend.when("GET", "/sync").respond(200, syncData);
-
-    const eventPromise = new Promise((resolve, reject) => {
-        const onEvent = function(event) {
-            // ignore the m.room.member events
-            if (event.getType() == "m.room.member") {
-                return;
-            }
-            logger.log(client.credentials.userId + " received event",
-                        event);
-
-            client.removeListener("event", onEvent);
-            resolve(event);
-        };
-        client.on("event", onEvent);
-    });
-
-    httpBackend.flush();
-
-    return eventPromise.then((event) => {
-        expect(event.isEncrypted()).toBeTruthy();
-
-        // it may still be being decrypted
-        return testUtils.awaitDecryption(event);
-    }).then((event) => {
-        expect(event.getType()).toEqual("m.room.message");
-        expect(event.getContent()).toMatchObject({
-            msgtype: "m.text",
-            body: "Hello, World",
-        });
-        expect(event.isEncrypted()).toBeTruthy();
-    });
-}
-
-/**
- * Send an initial sync response to the client (which just includes the member
- * list for our test room).
- *
- * @param {TestClient} testClient
- * @returns {Promise} which resolves when the sync has been flushed.
- */
-function firstSync(testClient) {
-    // send a sync response including our test room.
-    const syncData = {
-        next_batch: "x",
-        rooms: {
-            join: { },
-        },
-    };
-    syncData.rooms.join[roomId] = {
-        state: {
-            events: [
-                testUtils.mkMembership({
-                    mship: "join",
-                    user: aliUserId,
-                }),
-                testUtils.mkMembership({
-                    mship: "join",
-                    user: bobUserId,
-                }),
-            ],
-        },
-        timeline: {
-            events: [],
-        },
-    };
-
-    testClient.httpBackend.when("GET", "/sync").respond(200, syncData);
-    return testClient.flushSync();
-}
-
-describe("MatrixClient crypto", function() {
-    if (!CRYPTO_ENABLED) {
-        return;
-    }
-
-    beforeEach(async function() {
-        aliTestClient = new TestClient(aliUserId, aliDeviceId, aliAccessToken);
-        await aliTestClient.client.initCrypto();
-
-        bobTestClient = new TestClient(bobUserId, bobDeviceId, bobAccessToken);
-        await bobTestClient.client.initCrypto();
-
-        aliMessages = [];
-        bobMessages = [];
-    });
-
-    afterEach(function() {
-        aliTestClient.httpBackend.verifyNoOutstandingExpectation();
-        bobTestClient.httpBackend.verifyNoOutstandingExpectation();
-
-        return Promise.all([aliTestClient.stop(), bobTestClient.stop()]);
-    });
-
-    it("Bob uploads device keys", function() {
-        return Promise.resolve()
-            .then(bobUploadsDeviceKeys);
-    });
-
-    it("Ali downloads Bobs device keys", function() {
-        return Promise.resolve()
-            .then(bobUploadsDeviceKeys)
-            .then(aliDownloadsKeys);
-    });
-
-    it("Ali gets keys with an invalid signature", function() {
-        return Promise.resolve()
-            .then(bobUploadsDeviceKeys)
-            .then(function() {
-                // tamper bob's keys
-                const bobDeviceKeys = bobTestClient.deviceKeys;
-                expect(bobDeviceKeys.keys["curve25519:" + bobDeviceId]).toBeTruthy();
-                bobDeviceKeys.keys["curve25519:" + bobDeviceId] += "abc";
-
-                return Promise.all([
-                    aliTestClient.client.downloadKeys([bobUserId]),
-                    expectAliQueryKeys(),
-                ]);
-            }).then(function() {
-                return aliTestClient.client.getStoredDevicesForUser(bobUserId);
-            }).then((devices) => {
-                // should get an empty list
-                expect(devices).toEqual([]);
-            });
-    });
-
-    it("Ali gets keys with an incorrect userId", function() {
-        const eveUserId = "@eve:localhost";
-
-        const bobDeviceKeys = {
-            algorithms: ['m.olm.v1.curve25519-aes-sha2', 'm.megolm.v1.aes-sha2'],
-            device_id: 'bvcxz',
-            keys: {
-                'ed25519:bvcxz': 'pYuWKMCVuaDLRTM/eWuB8OlXEb61gZhfLVJ+Y54tl0Q',
-                'curve25519:bvcxz': '7Gni0loo/nzF0nFp9847RbhElGewzwUXHPrljjBGPTQ',
-            },
-            user_id: '@eve:localhost',
-            signatures: {
-                '@eve:localhost': {
-                    'ed25519:bvcxz': 'CliUPZ7dyVPBxvhSA1d+X+LYa5b2AYdjcTwG' +
-                        '0stXcIxjaJNemQqtdgwKDtBFl3pN2I13SEijRDCf1A8bYiQMDg',
-                },
-            },
-        };
-
-        const bobKeys = {};
-        bobKeys[bobDeviceId] = bobDeviceKeys;
-        aliTestClient.httpBackend.when(
-            "POST", "/keys/query",
-        ).respond(200, function(path, content) {
-            const result = {};
-            result[bobUserId] = bobKeys;
-            return { device_keys: result };
-        });
-
-        return Promise.all([
-            aliTestClient.client.downloadKeys([bobUserId, eveUserId]),
-            aliTestClient.httpBackend.flush("/keys/query", 1),
-        ]).then(function() {
-            return Promise.all([
-                aliTestClient.client.getStoredDevicesForUser(bobUserId),
-                aliTestClient.client.getStoredDevicesForUser(eveUserId),
-            ]);
-        }).then(([bobDevices, eveDevices]) => {
-            // should get an empty list
-            expect(bobDevices).toEqual([]);
-            expect(eveDevices).toEqual([]);
-        });
-    });
-
-    it("Ali gets keys with an incorrect deviceId", function() {
-        const bobDeviceKeys = {
-            algorithms: ['m.olm.v1.curve25519-aes-sha2', 'm.megolm.v1.aes-sha2'],
-            device_id: 'bad_device',
-            keys: {
-                'ed25519:bad_device': 'e8XlY5V8x2yJcwa5xpSzeC/QVOrU+D5qBgyTK0ko+f0',
-                'curve25519:bad_device': 'YxuuLG/4L5xGeP8XPl5h0d7DzyYVcof7J7do+OXz0xc',
-            },
-            user_id: '@bob:localhost',
-            signatures: {
-                '@bob:localhost': {
-                    'ed25519:bad_device': 'fEFTq67RaSoIEVBJ8DtmRovbwUBKJ0A' +
-                        'me9m9PDzM9azPUwZ38Xvf6vv1A7W1PSafH4z3Y2ORIyEnZgHaNby3CQ',
-                },
-            },
-        };
-
-        const bobKeys = {};
-        bobKeys[bobDeviceId] = bobDeviceKeys;
-        aliTestClient.httpBackend.when(
-            "POST", "/keys/query",
-        ).respond(200, function(path, content) {
-            const result = {};
-            result[bobUserId] = bobKeys;
-            return { device_keys: result };
-        });
-
-        return Promise.all([
-            aliTestClient.client.downloadKeys([bobUserId]),
-            aliTestClient.httpBackend.flush("/keys/query", 1),
-        ]).then(function() {
-            return aliTestClient.client.getStoredDevicesForUser(bobUserId);
-        }).then((devices) => {
-            // should get an empty list
-            expect(devices).toEqual([]);
-        });
-    });
-
-    it("Bob starts his client and uploads device keys and one-time keys", function() {
-        return Promise.resolve()
-            .then(() => bobTestClient.start())
-            .then(() => bobTestClient.awaitOneTimeKeyUpload())
-            .then((keys) => {
-                expect(Object.keys(keys).length).toEqual(5);
-                expect(Object.keys(bobTestClient.deviceKeys).length).not.toEqual(0);
-            });
-    });
-
-    it("Ali sends a message", function() {
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => bobTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(aliEnablesEncryption)
-            .then(aliSendsFirstMessage);
-    });
-
-    it("Bob receives a message", function() {
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => bobTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(aliEnablesEncryption)
-            .then(aliSendsFirstMessage)
-            .then(bobRecvMessage);
-    });
-
-    it("Bob receives a message with a bogus sender", function() {
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => bobTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(aliEnablesEncryption)
-            .then(aliSendsFirstMessage)
-            .then(function() {
-                const message = aliMessages.shift();
-                const syncData = {
-                    next_batch: "x",
-                    rooms: {
-                        join: {
-
-                        },
-                    },
-                };
-                syncData.rooms.join[roomId] = {
-                    timeline: {
-                        events: [
-                            testUtils.mkEvent({
-                                type: "m.room.encrypted",
-                                room: roomId,
-                                content: message,
-                                sender: "@bogus:sender",
-                            }),
-                        ],
-                    },
-                };
-                bobTestClient.httpBackend.when("GET", "/sync").respond(200, syncData);
-
-                const eventPromise = new Promise((resolve, reject) => {
-                    const onEvent = function(event) {
-                        logger.log(bobUserId + " received event",
-                                    event);
-                        resolve(event);
-                    };
-                    bobTestClient.client.once("event", onEvent);
-                });
-
-                bobTestClient.httpBackend.flush();
-                return eventPromise;
-            }).then((event) => {
-                expect(event.isEncrypted()).toBeTruthy();
-
-                // it may still be being decrypted
-                return testUtils.awaitDecryption(event);
-            }).then((event) => {
-                expect(event.getType()).toEqual("m.room.message");
-                expect(event.getContent().msgtype).toEqual("m.bad.encrypted");
-            });
-    });
-
-    it("Ali blocks Bob's device", function() {
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => bobTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(aliEnablesEncryption)
-            .then(aliDownloadsKeys)
-            .then(function() {
-                aliTestClient.client.setDeviceBlocked(bobUserId, bobDeviceId, true);
-                const p1 = sendMessage(aliTestClient.client);
-                const p2 = expectSendMessageRequest(aliTestClient.httpBackend)
-                      .then(function(sentContent) {
-                          // no unblocked devices, so the ciphertext should be empty
-                          expect(sentContent.ciphertext).toEqual({});
-                      });
-                return Promise.all([p1, p2]);
-            });
-    });
-
-    it("Bob receives two pre-key messages", function() {
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => bobTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(aliEnablesEncryption)
-            .then(aliSendsFirstMessage)
-            .then(bobRecvMessage)
-            .then(aliSendsMessage)
-            .then(bobRecvMessage);
-    });
-
-    it("Bob replies to the message", function() {
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        bobTestClient.expectKeyQuery({ device_keys: { [bobUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => bobTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(() => firstSync(bobTestClient))
-            .then(aliEnablesEncryption)
-            .then(aliSendsFirstMessage)
-            .then(bobRecvMessage)
-            .then(bobEnablesEncryption)
-            .then(bobSendsReplyMessage).then(function(ciphertext) {
-                expect(ciphertext.type).toEqual(1, "Unexpected cipghertext type.");
-            }).then(aliRecvMessage);
-    });
-
-    it("Ali does a key query when encryption is enabled", function() {
-        // enabling encryption in the room should make alice download devices
-        // for both members.
-        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
-        return Promise.resolve()
-            .then(() => aliTestClient.start())
-            .then(() => firstSync(aliTestClient))
-            .then(() => {
-                const syncData = {
-                    next_batch: '2',
-                    rooms: {
-                        join: {},
-                    },
-                };
-                syncData.rooms.join[roomId] = {
-                    state: {
-                        events: [
-                            testUtils.mkEvent({
-                                type: 'm.room.encryption',
-                                skey: '',
-                                content: {
-                                    algorithm: 'm.olm.v1.curve25519-aes-sha2',
-                                },
-                            }),
-                        ],
-                    },
-                };
-
-                aliTestClient.httpBackend.when('GET', '/sync').respond(
-                    200, syncData);
-                return aliTestClient.httpBackend.flush('/sync', 1);
-            }).then(() => {
-                aliTestClient.expectKeyQuery({
-                    device_keys: {
-                        [bobUserId]: {},
-                    },
-                });
-                return aliTestClient.httpBackend.flushAllExpected();
-            });
-    });
-
-    it("Upload new oneTimeKeys based on a /sync request - no count-asking", function() {
-        // Send a response which causes a key upload
-        const httpBackend = aliTestClient.httpBackend;
-        const syncDataEmpty = {
-            next_batch: "a",
-            device_one_time_keys_count: {
-                signed_curve25519: 0,
-            },
-        };
-
-        // enqueue expectations:
-        // * Sync with empty one_time_keys => upload keys
-
-        return Promise.resolve()
-            .then(() => {
-                logger.log(aliTestClient + ': starting');
-                httpBackend.when("GET", "/versions").respond(200, {});
-                httpBackend.when("GET", "/pushrules").respond(200, {});
-                httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
-                aliTestClient.expectDeviceKeyUpload();
-
-                // we let the client do a very basic initial sync, which it needs before
-                // it will upload one-time keys.
-                httpBackend.when("GET", "/sync").respond(200, syncDataEmpty);
-
-                aliTestClient.client.startClient({});
-
-                return httpBackend.flushAllExpected().then(() => {
-                    logger.log(aliTestClient + ': started');
-                });
-            })
-            .then(() => httpBackend.when("POST", "/keys/upload")
-                .respond(200, (path, content) => {
-                    expect(content.one_time_keys).toBeTruthy();
-                    expect(content.one_time_keys).not.toEqual({});
-                    expect(Object.keys(content.one_time_keys).length)
-                        .toBeGreaterThanOrEqual(1);
-                    logger.log('received %i one-time keys',
-                                Object.keys(content.one_time_keys).length);
-                    // cancel futher calls by telling the client
-                    // we have more than we need
-                    return {
-                       one_time_key_counts: {
-                           signed_curve25519: 70,
-                       },
-                    };
-                }))
-            .then(() => httpBackend.flushAllExpected());
-    });
-});

spec/integ/matrix-client-crypto.spec.ts

@@ -0,0 +1,676 @@
+/*
+Copyright 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/* This file consists of a set of integration tests which try to simulate
+ * communication via an Olm-encrypted room between two users, Alice and Bob.
+ *
+ * Note that megolm (group) conversation is not tested here.
+ *
+ * See also `megolm.spec.js`.
+ */
+
+// load olm before the sdk if possible
+import '../olm-loader';
+
+import { logger } from '../../src/logger';
+import * as testUtils from "../test-utils/test-utils";
+import { TestClient } from "../TestClient";
+import { CRYPTO_ENABLED } from "../../src/client";
+import { ClientEvent, IContent, ISendEventResponse, MatrixClient, MatrixEvent } from "../../src/matrix";
+
+let aliTestClient: TestClient;
+const roomId = "!room:localhost";
+const aliUserId = "@ali:localhost";
+const aliDeviceId = "zxcvb";
+const aliAccessToken = "aseukfgwef";
+let bobTestClient: TestClient;
+const bobUserId = "@bob:localhost";
+const bobDeviceId = "bvcxz";
+const bobAccessToken = "fewgfkuesa";
+let aliMessages: IContent[];
+let bobMessages: IContent[];
+
+// IMessage isn't exported by src/crypto/algorithms/olm.ts
+interface OlmPayload {
+    type: number;
+    body: string;
+}
+
+async function bobUploadsDeviceKeys(): Promise<void> {
+    bobTestClient.expectDeviceKeyUpload();
+    await Promise.all([
+        bobTestClient.client.uploadKeys(),
+        bobTestClient.httpBackend.flushAllExpected(),
+    ]);
+    expect(Object.keys(bobTestClient.deviceKeys).length).not.toEqual(0);
+}
+
+/**
+ * Set an expectation that querier will query uploader's keys; then flush the http request.
+ *
+ * @return {promise} resolves once the http request has completed.
+ */
+function expectQueryKeys(querier: TestClient, uploader: TestClient): Promise<number> {
+    // can't query keys before bob has uploaded them
+    expect(uploader.deviceKeys).toBeTruthy();
+
+    const uploaderKeys = {};
+    uploaderKeys[uploader.deviceId] = uploader.deviceKeys;
+    querier.httpBackend.when("POST", "/keys/query")
+        .respond(200, function(_path, content) {
+            expect(content.device_keys[uploader.userId]).toEqual([]);
+            const result = {};
+            result[uploader.userId] = uploaderKeys;
+            return { device_keys: result };
+        });
+    return querier.httpBackend.flush("/keys/query", 1);
+}
+const expectAliQueryKeys = () => expectQueryKeys(aliTestClient, bobTestClient);
+const expectBobQueryKeys = () => expectQueryKeys(bobTestClient, aliTestClient);
+
+/**
+ * Set an expectation that ali will claim one of bob's keys; then flush the http request.
+ *
+ * @return {promise} resolves once the http request has completed.
+ */
+async function expectAliClaimKeys(): Promise<void> {
+    const keys = await bobTestClient.awaitOneTimeKeyUpload();
+    aliTestClient.httpBackend.when(
+        "POST", "/keys/claim",
+    ).respond(200, function(_path, content) {
+        const claimType = content.one_time_keys[bobUserId][bobDeviceId];
+        expect(claimType).toEqual("signed_curve25519");
+        let keyId = null;
+        for (keyId in keys) {
+            if (bobTestClient.oneTimeKeys.hasOwnProperty(keyId)) {
+                if (keyId.indexOf(claimType + ":") === 0) {
+                    break;
+                }
+            }
+        }
+        const result = {};
+        result[bobUserId] = {};
+        result[bobUserId][bobDeviceId] = {};
+        result[bobUserId][bobDeviceId][keyId] = keys[keyId];
+        return { one_time_keys: result };
+    });
+    // it can take a while to process the key query, so give it some extra
+    // time, and make sure the claim actually happens rather than ploughing on
+    // confusingly.
+    const r = await aliTestClient.httpBackend.flush("/keys/claim", 1, 500);
+    expect(r).toEqual(1);
+}
+
+async function aliDownloadsKeys(): Promise<void> {
+    // can't query keys before bob has uploaded them
+    expect(bobTestClient.getSigningKey()).toBeTruthy();
+
+    const p1 = async () => {
+        await aliTestClient.client.downloadKeys([bobUserId]);
+        const devices = aliTestClient.client.getStoredDevicesForUser(bobUserId);
+        expect(devices.length).toEqual(1);
+        expect(devices[0].deviceId).toEqual("bvcxz");
+    };
+    const p2 = expectAliQueryKeys;
+
+    // check that the localStorage is updated as we expect (not sure this is
+    // an integration test, but meh)
+    await Promise.all([p1(), p2()]);
+    await aliTestClient.client.crypto.deviceList.saveIfDirty();
+    // @ts-ignore - protected
+    aliTestClient.client.cryptoStore.getEndToEndDeviceData(null, (data) => {
+        const devices = data.devices[bobUserId];
+        expect(devices[bobDeviceId].keys).toEqual(bobTestClient.deviceKeys.keys);
+        expect(devices[bobDeviceId].verified).
+            toBe(0); // DeviceVerification.UNVERIFIED
+    });
+}
+
+async function clientEnablesEncryption(client: MatrixClient): Promise<void> {
+    await client.setRoomEncryption(roomId, {
+        algorithm: "m.olm.v1.curve25519-aes-sha2",
+    });
+    expect(client.isRoomEncrypted(roomId)).toBeTruthy();
+}
+const aliEnablesEncryption = () => clientEnablesEncryption(aliTestClient.client);
+const bobEnablesEncryption = () => clientEnablesEncryption(bobTestClient.client);
+
+/**
+ * Ali sends a message, first claiming e2e keys. Set the expectations and
+ * check the results.
+ *
+ * @return {promise} which resolves to the ciphertext for Bob's device.
+ */
+async function aliSendsFirstMessage(): Promise<OlmPayload> {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const [_, ciphertext] = await Promise.all([
+        sendMessage(aliTestClient.client),
+        expectAliQueryKeys()
+            .then(expectAliClaimKeys)
+            .then(expectAliSendMessageRequest),
+    ]);
+    return ciphertext;
+}
+
+/**
+ * Ali sends a message without first claiming e2e keys. Set the expectations
+ * and check the results.
+ *
+ * @return {promise} which resolves to the ciphertext for Bob's device.
+ */
+async function aliSendsMessage(): Promise<OlmPayload> {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const [_, ciphertext] = await Promise.all([
+        sendMessage(aliTestClient.client),
+        expectAliSendMessageRequest(),
+    ]);
+    return ciphertext;
+}
+
+/**
+ * Bob sends a message, first querying (but not claiming) e2e keys. Set the
+ * expectations and check the results.
+ *
+ * @return {promise} which resolves to the ciphertext for Ali's device.
+ */
+async function bobSendsReplyMessage(): Promise<OlmPayload> {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const [_, ciphertext] = await Promise.all([
+        sendMessage(bobTestClient.client),
+        expectBobQueryKeys()
+            .then(expectBobSendMessageRequest),
+    ]);
+    return ciphertext;
+}
+
+/**
+ * Set an expectation that Ali will send a message, and flush the request
+ *
+ * @return {promise} which resolves to the ciphertext for Bob's device.
+ */
+async function expectAliSendMessageRequest(): Promise<OlmPayload> {
+    const content = await expectSendMessageRequest(aliTestClient.httpBackend);
+    aliMessages.push(content);
+    expect(Object.keys(content.ciphertext)).toEqual([bobTestClient.getDeviceKey()]);
+    const ciphertext = content.ciphertext[bobTestClient.getDeviceKey()];
+    expect(ciphertext).toBeTruthy();
+    return ciphertext;
+}
+
+/**
+ * Set an expectation that Bob will send a message, and flush the request
+ *
+ * @return {promise} which resolves to the ciphertext for Bob's device.
+ */
+async function expectBobSendMessageRequest(): Promise<OlmPayload> {
+    const content = await expectSendMessageRequest(bobTestClient.httpBackend);
+    bobMessages.push(content);
+    const aliKeyId = "curve25519:" + aliDeviceId;
+    const aliDeviceCurve25519Key = aliTestClient.deviceKeys.keys[aliKeyId];
+    expect(Object.keys(content.ciphertext)).toEqual([aliDeviceCurve25519Key]);
+    const ciphertext = content.ciphertext[aliDeviceCurve25519Key];
+    expect(ciphertext).toBeTruthy();
+    return ciphertext;
+}
+
+function sendMessage(client: MatrixClient): Promise<ISendEventResponse> {
+    return client.sendMessage(
+        roomId, { msgtype: "m.text", body: "Hello, World" },
+    );
+}
+
+async function expectSendMessageRequest(httpBackend: TestClient["httpBackend"]): Promise<IContent> {
+    const path = "/send/m.room.encrypted/";
+    const prom = new Promise((resolve) => {
+        httpBackend.when("PUT", path).respond(200, function(_path, content) {
+            resolve(content);
+            return {
+                event_id: "asdfgh",
+            };
+        });
+    });
+
+    // it can take a while to process the key query
+    await httpBackend.flush(path, 1);
+    return prom;
+}
+
+function aliRecvMessage(): Promise<void> {
+    const message = bobMessages.shift();
+    return recvMessage(
+        aliTestClient.httpBackend, aliTestClient.client, bobUserId, message,
+    );
+}
+
+function bobRecvMessage(): Promise<void> {
+    const message = aliMessages.shift();
+    return recvMessage(
+        bobTestClient.httpBackend, bobTestClient.client, aliUserId, message,
+    );
+}
+
+async function recvMessage(
+    httpBackend: TestClient["httpBackend"],
+    client: MatrixClient,
+    sender: string,
+    message: IContent,
+): Promise<void> {
+    const syncData = {
+        next_batch: "x",
+        rooms: {
+            join: {
+
+            },
+        },
+    };
+    syncData.rooms.join[roomId] = {
+        timeline: {
+            events: [
+                testUtils.mkEvent({
+                    type: "m.room.encrypted",
+                    room: roomId,
+                    content: message,
+                    sender: sender,
+                }),
+            ],
+        },
+    };
+    httpBackend.when("GET", "/sync").respond(200, syncData);
+
+    const eventPromise = new Promise<MatrixEvent>((resolve) => {
+        const onEvent = function(event: MatrixEvent) {
+            // ignore the m.room.member events
+            if (event.getType() == "m.room.member") {
+                return;
+            }
+            logger.log(client.credentials.userId + " received event",
+                event);
+
+            client.removeListener(ClientEvent.Event, onEvent);
+            resolve(event);
+        };
+        client.on(ClientEvent.Event, onEvent);
+    });
+
+    await httpBackend.flushAllExpected();
+
+    const preDecryptionEvent = await eventPromise;
+    expect(preDecryptionEvent.isEncrypted()).toBeTruthy();
+    // it may still be being decrypted
+    const event = await testUtils.awaitDecryption(preDecryptionEvent);
+    expect(event.getType()).toEqual("m.room.message");
+    expect(event.getContent()).toMatchObject({
+        msgtype: "m.text",
+        body: "Hello, World",
+    });
+    expect(event.isEncrypted()).toBeTruthy();
+}
+
+/**
+ * Send an initial sync response to the client (which just includes the member
+ * list for our test room).
+ *
+ * @param {TestClient} testClient
+ * @returns {Promise} which resolves when the sync has been flushed.
+ */
+function firstSync(testClient: TestClient): Promise<void> {
+    // send a sync response including our test room.
+    const syncData = {
+        next_batch: "x",
+        rooms: {
+            join: { },
+        },
+    };
+    syncData.rooms.join[roomId] = {
+        state: {
+            events: [
+                testUtils.mkMembership({
+                    mship: "join",
+                    user: aliUserId,
+                }),
+                testUtils.mkMembership({
+                    mship: "join",
+                    user: bobUserId,
+                }),
+            ],
+        },
+        timeline: {
+            events: [],
+        },
+    };
+
+    testClient.httpBackend.when("GET", "/sync").respond(200, syncData);
+    return testClient.flushSync();
+}
+
+describe("MatrixClient crypto", () => {
+    if (!CRYPTO_ENABLED) {
+        return;
+    }
+
+    beforeEach(async () => {
+        aliTestClient = new TestClient(aliUserId, aliDeviceId, aliAccessToken);
+        await aliTestClient.client.initCrypto();
+
+        bobTestClient = new TestClient(bobUserId, bobDeviceId, bobAccessToken);
+        await bobTestClient.client.initCrypto();
+
+        aliMessages = [];
+        bobMessages = [];
+    });
+
+    afterEach(() => {
+        aliTestClient.httpBackend.verifyNoOutstandingExpectation();
+        bobTestClient.httpBackend.verifyNoOutstandingExpectation();
+
+        return Promise.all([aliTestClient.stop(), bobTestClient.stop()]);
+    });
+
+    it("Bob uploads device keys", bobUploadsDeviceKeys);
+
+    it("Ali downloads Bobs device keys", async () => {
+        await bobUploadsDeviceKeys();
+        await aliDownloadsKeys();
+    });
+
+    it("Ali gets keys with an invalid signature", async () => {
+        await bobUploadsDeviceKeys();
+        // tamper bob's keys
+        const bobDeviceKeys = bobTestClient.deviceKeys;
+        expect(bobDeviceKeys.keys["curve25519:" + bobDeviceId]).toBeTruthy();
+        bobDeviceKeys.keys["curve25519:" + bobDeviceId] += "abc";
+        await Promise.all([
+            aliTestClient.client.downloadKeys([bobUserId]),
+            expectAliQueryKeys(),
+        ]);
+        const devices = aliTestClient.client.getStoredDevicesForUser(bobUserId);
+        // should get an empty list
+        expect(devices).toEqual([]);
+    });
+
+    it("Ali gets keys with an incorrect userId", async () => {
+        const eveUserId = "@eve:localhost";
+
+        const bobDeviceKeys = {
+            algorithms: ['m.olm.v1.curve25519-aes-sha2', 'm.megolm.v1.aes-sha2'],
+            device_id: 'bvcxz',
+            keys: {
+                'ed25519:bvcxz': 'pYuWKMCVuaDLRTM/eWuB8OlXEb61gZhfLVJ+Y54tl0Q',
+                'curve25519:bvcxz': '7Gni0loo/nzF0nFp9847RbhElGewzwUXHPrljjBGPTQ',
+            },
+            user_id: '@eve:localhost',
+            signatures: {
+                '@eve:localhost': {
+                    'ed25519:bvcxz': 'CliUPZ7dyVPBxvhSA1d+X+LYa5b2AYdjcTwG' +
+                        '0stXcIxjaJNemQqtdgwKDtBFl3pN2I13SEijRDCf1A8bYiQMDg',
+                },
+            },
+        };
+
+        const bobKeys = {};
+        bobKeys[bobDeviceId] = bobDeviceKeys;
+        aliTestClient.httpBackend.when(
+            "POST", "/keys/query",
+        ).respond(200, { device_keys: { [bobUserId]: bobKeys } });
+
+        await Promise.all([
+            aliTestClient.client.downloadKeys([bobUserId, eveUserId]),
+            aliTestClient.httpBackend.flush("/keys/query", 1),
+        ]);
+        const [bobDevices, eveDevices] = await Promise.all([
+            aliTestClient.client.getStoredDevicesForUser(bobUserId),
+            aliTestClient.client.getStoredDevicesForUser(eveUserId),
+        ]);
+        // should get an empty list
+        expect(bobDevices).toEqual([]);
+        expect(eveDevices).toEqual([]);
+    });
+
+    it("Ali gets keys with an incorrect deviceId", async () => {
+        const bobDeviceKeys = {
+            algorithms: ['m.olm.v1.curve25519-aes-sha2', 'm.megolm.v1.aes-sha2'],
+            device_id: 'bad_device',
+            keys: {
+                'ed25519:bad_device': 'e8XlY5V8x2yJcwa5xpSzeC/QVOrU+D5qBgyTK0ko+f0',
+                'curve25519:bad_device': 'YxuuLG/4L5xGeP8XPl5h0d7DzyYVcof7J7do+OXz0xc',
+            },
+            user_id: '@bob:localhost',
+            signatures: {
+                '@bob:localhost': {
+                    'ed25519:bad_device': 'fEFTq67RaSoIEVBJ8DtmRovbwUBKJ0A' +
+                        'me9m9PDzM9azPUwZ38Xvf6vv1A7W1PSafH4z3Y2ORIyEnZgHaNby3CQ',
+                },
+            },
+        };
+
+        const bobKeys = {};
+        bobKeys[bobDeviceId] = bobDeviceKeys;
+        aliTestClient.httpBackend.when(
+            "POST", "/keys/query",
+        ).respond(200, { device_keys: { [bobUserId]: bobKeys } });
+
+        await Promise.all([
+            aliTestClient.client.downloadKeys([bobUserId]),
+            aliTestClient.httpBackend.flush("/keys/query", 1),
+        ]);
+        const devices = aliTestClient.client.getStoredDevicesForUser(bobUserId);
+        // should get an empty list
+        expect(devices).toEqual([]);
+    });
+
+    it("Bob starts his client and uploads device keys and one-time keys", async () => {
+        await bobTestClient.start();
+        const keys = await bobTestClient.awaitOneTimeKeyUpload();
+        expect(Object.keys(keys).length).toEqual(5);
+        expect(Object.keys(bobTestClient.deviceKeys).length).not.toEqual(0);
+    });
+
+    it("Ali sends a message", async () => {
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await bobTestClient.start();
+        await firstSync(aliTestClient);
+        await aliEnablesEncryption();
+        await aliSendsFirstMessage();
+    });
+
+    it("Bob receives a message", async () => {
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await bobTestClient.start();
+        await firstSync(aliTestClient);
+        await aliEnablesEncryption();
+        await aliSendsFirstMessage();
+        await bobRecvMessage();
+    });
+
+    it("Bob receives a message with a bogus sender", async () => {
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await bobTestClient.start();
+        await firstSync(aliTestClient);
+        await aliEnablesEncryption();
+        await aliSendsFirstMessage();
+        const message = aliMessages.shift();
+        const syncData = {
+            next_batch: "x",
+            rooms: {
+                join: {
+
+                },
+            },
+        };
+        syncData.rooms.join[roomId] = {
+            timeline: {
+                events: [
+                    testUtils.mkEvent({
+                        type: "m.room.encrypted",
+                        room: roomId,
+                        content: message,
+                        sender: "@bogus:sender",
+                    }),
+                ],
+            },
+        };
+        bobTestClient.httpBackend.when("GET", "/sync").respond(200, syncData);
+
+        const eventPromise = new Promise<MatrixEvent>((resolve) => {
+            const onEvent = function(event: MatrixEvent) {
+                logger.log(bobUserId + " received event", event);
+                resolve(event);
+            };
+            bobTestClient.client.once(ClientEvent.Event, onEvent);
+        });
+        await bobTestClient.httpBackend.flushAllExpected();
+        const preDecryptionEvent = await eventPromise;
+        expect(preDecryptionEvent.isEncrypted()).toBeTruthy();
+        // it may still be being decrypted
+        const event = await testUtils.awaitDecryption(preDecryptionEvent);
+        expect(event.getType()).toEqual("m.room.message");
+        expect(event.getContent().msgtype).toEqual("m.bad.encrypted");
+    });
+
+    it("Ali blocks Bob's device", async () => {
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await bobTestClient.start();
+        await firstSync(aliTestClient);
+        await aliEnablesEncryption();
+        await aliDownloadsKeys();
+        aliTestClient.client.setDeviceBlocked(bobUserId, bobDeviceId, true);
+        const p1 = sendMessage(aliTestClient.client);
+        const p2 = expectSendMessageRequest(aliTestClient.httpBackend)
+            .then(function(sentContent) {
+                // no unblocked devices, so the ciphertext should be empty
+                expect(sentContent.ciphertext).toEqual({});
+            });
+        await Promise.all([p1, p2]);
+    });
+
+    it("Bob receives two pre-key messages", async () => {
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await bobTestClient.start();
+        await firstSync(aliTestClient);
+        await aliEnablesEncryption();
+        await aliSendsFirstMessage();
+        await bobRecvMessage();
+        await aliSendsMessage();
+        await bobRecvMessage();
+    });
+
+    it("Bob replies to the message", async () => {
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        bobTestClient.expectKeyQuery({ device_keys: { [bobUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await bobTestClient.start();
+        await firstSync(aliTestClient);
+        await firstSync(bobTestClient);
+        await aliEnablesEncryption();
+        await aliSendsFirstMessage();
+        await bobRecvMessage();
+        await bobEnablesEncryption();
+        const ciphertext = await bobSendsReplyMessage();
+        expect(ciphertext.type).toEqual(1);
+        await aliRecvMessage();
+    });
+
+    it("Ali does a key query when encryption is enabled", async () => {
+        // enabling encryption in the room should make alice download devices
+        // for both members.
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} }, failures: {} });
+        await aliTestClient.start();
+        await firstSync(aliTestClient);
+        const syncData = {
+            next_batch: '2',
+            rooms: {
+                join: {},
+            },
+        };
+        syncData.rooms.join[roomId] = {
+            state: {
+                events: [
+                    testUtils.mkEvent({
+                        type: 'm.room.encryption',
+                        skey: '',
+                        content: {
+                            algorithm: 'm.olm.v1.curve25519-aes-sha2',
+                        },
+                    }),
+                ],
+            },
+        };
+
+        aliTestClient.httpBackend.when('GET', '/sync').respond(
+            200, syncData);
+        await aliTestClient.httpBackend.flush('/sync', 1);
+        aliTestClient.expectKeyQuery({
+            device_keys: {
+                [bobUserId]: {},
+            },
+            failures: {},
+        });
+        await aliTestClient.httpBackend.flushAllExpected();
+    });
+
+    it("Upload new oneTimeKeys based on a /sync request - no count-asking", async () => {
+        // Send a response which causes a key upload
+        const httpBackend = aliTestClient.httpBackend;
+        const syncDataEmpty = {
+            next_batch: "a",
+            device_one_time_keys_count: {
+                signed_curve25519: 0,
+            },
+        };
+
+        // enqueue expectations:
+        // * Sync with empty one_time_keys => upload keys
+
+        logger.log(aliTestClient + ': starting');
+        httpBackend.when("GET", "/versions").respond(200, {});
+        httpBackend.when("GET", "/pushrules").respond(200, {});
+        httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+        aliTestClient.expectDeviceKeyUpload();
+
+        // we let the client do a very basic initial sync, which it needs before
+        // it will upload one-time keys.
+        httpBackend.when("GET", "/sync").respond(200, syncDataEmpty);
+
+        await Promise.all([
+            aliTestClient.client.startClient({}),
+            httpBackend.flushAllExpected(),
+        ]);
+        logger.log(aliTestClient + ': started');
+        httpBackend.when("POST", "/keys/upload")
+            .respond(200, (_path, content) => {
+                expect(content.one_time_keys).toBeTruthy();
+                expect(content.one_time_keys).not.toEqual({});
+                expect(Object.keys(content.one_time_keys).length).toBeGreaterThanOrEqual(1);
+                logger.log('received %i one-time keys', Object.keys(content.one_time_keys).length);
+                // cancel futher calls by telling the client
+                // we have more than we need
+                return {
+                    one_time_key_counts: {
+                        signed_curve25519: 70,
+                    },
+                };
+            });
+        await httpBackend.flushAllExpected();
+    });
+});

spec/integ/matrix-client-event-timeline.spec.ts

@@ -1,5 +1,21 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 import * as utils from "../test-utils/test-utils";
-import { EventTimeline, Filter, MatrixEvent } from "../../src/matrix";
+import { ClientEvent, EventTimeline, Filter, IEvent, MatrixClient, MatrixEvent, Room } from "../../src/matrix";
 import { logger } from "../../src/logger";
 import { TestClient } from "../TestClient";
 import { Thread, THREAD_RELATION_TYPE } from "../../src/models/thread";
@@ -10,8 +26,14 @@ const accessToken = "aseukfgwef";
 const roomId = "!foo:bar";
 const otherUserId = "@bob:localhost";
 
+const withoutRoomId = (e: Partial<IEvent>): Partial<IEvent> => {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const { room_id: _, ...copy } = e;
+    return copy;
+};
+
 const USER_MEMBERSHIP_EVENT = utils.mkMembership({
-    room: roomId, mship: "join", user: userId, name: userName,
+    room: roomId, mship: "join", user: userId, name: userName, event: false,
 });
 
 const ROOM_NAME_EVENT = utils.mkEvent({
@@ -19,34 +41,37 @@ const ROOM_NAME_EVENT = utils.mkEvent({
     content: {
         name: "Old room name",
     },
+    event: false,
 });
 
 const INITIAL_SYNC_DATA = {
     next_batch: "s_5_3",
     rooms: {
         join: {
-            "!foo:bar": {  // roomId
+            [roomId]: {
                 timeline: {
                     events: [
                         utils.mkMessage({
-                            room: roomId, user: otherUserId, msg: "hello",
+                            user: otherUserId, msg: "hello", event: false,
                         }),
                     ],
                     prev_batch: "f_1_1",
                 },
                 state: {
                     events: [
-                        ROOM_NAME_EVENT,
+                        withoutRoomId(ROOM_NAME_EVENT),
                         utils.mkMembership({
-                            room: roomId, mship: "join",
+                            mship: "join",
                             user: otherUserId, name: "Bob",
+                            event: false,
                         }),
-                        USER_MEMBERSHIP_EVENT,
+                        withoutRoomId(USER_MEMBERSHIP_EVENT),
                         utils.mkEvent({
-                            type: "m.room.create", room: roomId, user: userId,
+                            type: "m.room.create", user: userId,
                             content: {
                                 creator: userId,
                             },
+                            event: false,
                         }),
                     ],
                 },
@@ -57,16 +82,16 @@ const INITIAL_SYNC_DATA = {
 
 const EVENTS = [
     utils.mkMessage({
-        room: roomId, user: userId, msg: "we",
+        room: roomId, user: userId, msg: "we", event: false,
     }),
     utils.mkMessage({
-        room: roomId, user: userId, msg: "could",
+        room: roomId, user: userId, msg: "could", event: false,
     }),
     utils.mkMessage({
-        room: roomId, user: userId, msg: "be",
+        room: roomId, user: userId, msg: "be", event: false,
     }),
     utils.mkMessage({
-        room: roomId, user: userId, msg: "heroes",
+        room: roomId, user: userId, msg: "heroes", event: false,
     }),
 ];
 
@@ -81,12 +106,13 @@ const THREAD_ROOT = utils.mkEvent({
     unsigned: {
         "m.relations": {
             "io.element.thread": {
-                "latest_event": undefined,
+                //"latest_event": undefined,
                 "count": 1,
                 "current_user_participated": true,
             },
         },
     },
+    event: false,
 });
 
 const THREAD_REPLY = utils.mkEvent({
@@ -102,12 +128,25 @@ const THREAD_REPLY = utils.mkEvent({
             event_id: THREAD_ROOT.event_id,
         },
     },
+    event: false,
 });
 
 THREAD_ROOT.unsigned["m.relations"]["io.element.thread"].latest_event = THREAD_REPLY;
 
+const SYNC_THREAD_ROOT = withoutRoomId(THREAD_ROOT);
+const SYNC_THREAD_REPLY = withoutRoomId(THREAD_REPLY);
+SYNC_THREAD_ROOT.unsigned = {
+    "m.relations": {
+        "io.element.thread": {
+            "latest_event": SYNC_THREAD_REPLY,
+            "count": 1,
+            "current_user_participated": true,
+        },
+    },
+};
+
 // start the client, and wait for it to initialise
-function startClient(httpBackend, client) {
+function startClient(httpBackend: TestClient["httpBackend"], client: MatrixClient) {
     httpBackend.when("GET", "/versions").respond(200, {});
     httpBackend.when("GET", "/pushrules").respond(200, {});
     httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
@@ -116,8 +155,8 @@ function startClient(httpBackend, client) {
     client.startClient();
 
     // set up a promise which will resolve once the client is initialised
-    const prom = new Promise((resolve) => {
-        client.on("sync", function(state) {
+    const prom = new Promise<void>((resolve) => {
+        client.on(ClientEvent.Sync, function(state) {
             logger.log("sync", state);
             if (state != "SYNCING") {
                 return;
@@ -133,8 +172,8 @@ function startClient(httpBackend, client) {
 }
 
 describe("getEventTimeline support", function() {
-    let httpBackend;
-    let client;
+    let httpBackend: TestClient["httpBackend"];
+    let client: MatrixClient;
 
     beforeEach(function() {
         const testClient = new TestClient(userId, "DEVICE", accessToken);
@@ -177,7 +216,7 @@ describe("getEventTimeline support", function() {
 
     it("scrollback should be able to scroll back to before a gappy /sync", function() {
         // need a client with timelineSupport disabled to make this work
-        let room;
+        let room: Room;
 
         return startClient(httpBackend, client).then(function() {
             room = client.getRoom(roomId);
@@ -189,7 +228,7 @@ describe("getEventTimeline support", function() {
                         "!foo:bar": {
                             timeline: {
                                 events: [
-                                    EVENTS[0],
+                                    withoutRoomId(EVENTS[0]),
                                 ],
                                 prev_batch: "f_1_1",
                             },
@@ -205,7 +244,7 @@ describe("getEventTimeline support", function() {
                         "!foo:bar": {
                             timeline: {
                                 events: [
-                                    EVENTS[1],
+                                    withoutRoomId(EVENTS[1]),
                                 ],
                                 limited: true,
                                 prev_batch: "f_1_2",
@@ -240,8 +279,8 @@ describe("getEventTimeline support", function() {
 });
 
 describe("MatrixClient event timelines", function() {
-    let client = null;
-    let httpBackend = null;
+    let client: MatrixClient;
+    let httpBackend: TestClient["httpBackend"];
 
     beforeEach(function() {
         const testClient = new TestClient(
@@ -260,7 +299,7 @@ describe("MatrixClient event timelines", function() {
     afterEach(function() {
         httpBackend.verifyNoOutstandingExpectation();
         client.stopClient();
-        Thread.setServerSideSupport(false);
+        Thread.setServerSideSupport(false, false);
     });
 
     describe("getEventTimeline", function() {
@@ -308,7 +347,7 @@ describe("MatrixClient event timelines", function() {
                         "!foo:bar": {
                             timeline: {
                                 events: [
-                                    EVENTS[0],
+                                    withoutRoomId(EVENTS[0]),
                                 ],
                                 prev_batch: "f_1_2",
                             },
@@ -343,7 +382,7 @@ describe("MatrixClient event timelines", function() {
                         "!foo:bar": {
                             timeline: {
                                 events: [
-                                    EVENTS[3],
+                                    withoutRoomId(EVENTS[3]),
                                 ],
                                 prev_batch: "f_1_2",
                             },
@@ -366,7 +405,7 @@ describe("MatrixClient event timelines", function() {
                 });
 
             const prom = new Promise((resolve, reject) => {
-                client.on("sync", function() {
+                client.on(ClientEvent.Sync, function() {
                     client.getEventTimeline(timelineSet, EVENTS[2].event_id,
                     ).then(function(tl) {
                         expect(tl.getEvents().length).toEqual(4);
@@ -511,8 +550,9 @@ describe("MatrixClient event timelines", function() {
         });
 
         it("should handle thread replies with server support by fetching a contiguous thread timeline", async () => {
+            // @ts-ignore
             client.clientOpts.experimentalThreadSupport = true;
-            Thread.setServerSideSupport(true);
+            Thread.setServerSideSupport(true, false);
             client.stopClient(); // we don't need the client to be syncing at this time
             const room = client.getRoom(roomId);
             const thread = room.createThread(THREAD_ROOT.event_id, undefined, [], false);
@@ -556,8 +596,9 @@ describe("MatrixClient event timelines", function() {
         });
 
         it("should return relevant timeline from non-thread timelineSet when asking for the thread root", async () => {
+            // @ts-ignore
             client.clientOpts.experimentalThreadSupport = true;
-            Thread.setServerSideSupport(true);
+            Thread.setServerSideSupport(true, false);
             client.stopClient(); // we don't need the client to be syncing at this time
             const room = client.getRoom(roomId);
             const threadRoot = new MatrixEvent(THREAD_ROOT);
@@ -587,8 +628,9 @@ describe("MatrixClient event timelines", function() {
         });
 
         it("should return undefined when event is not in the thread that the given timelineSet is representing", () => {
+            // @ts-ignore
             client.clientOpts.experimentalThreadSupport = true;
-            Thread.setServerSideSupport(true);
+            Thread.setServerSideSupport(true, false);
             client.stopClient(); // we don't need the client to be syncing at this time
             const room = client.getRoom(roomId);
             const threadRoot = new MatrixEvent(THREAD_ROOT);
@@ -614,8 +656,9 @@ describe("MatrixClient event timelines", function() {
         });
 
         it("should return undefined when event is within a thread but timelineSet is not", () => {
+            // @ts-ignore
             client.clientOpts.experimentalThreadSupport = true;
-            Thread.setServerSideSupport(true);
+            Thread.setServerSideSupport(true, false);
             client.stopClient(); // we don't need the client to be syncing at this time
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
@@ -639,6 +682,7 @@ describe("MatrixClient event timelines", function() {
         });
 
         it("should should add lazy loading filter when requested", async () => {
+            // @ts-ignore
             client.clientOpts.lazyLoadMembers = true;
             client.stopClient(); // we don't need the client to be syncing at this time
             const room = client.getRoom(roomId);
@@ -656,7 +700,7 @@ describe("MatrixClient event timelines", function() {
                 };
             });
             req.check((request) => {
-                expect(request.opts.qs.filter).toEqual(JSON.stringify(Filter.LAZY_LOADING_MESSAGES_FILTER));
+                expect(request.queryParams.filter).toEqual(JSON.stringify(Filter.LAZY_LOADING_MESSAGES_FILTER));
             });
 
             await Promise.all([
@@ -863,7 +907,7 @@ describe("MatrixClient event timelines", function() {
                         "!foo:bar": {
                             timeline: {
                                 events: [
-                                    event,
+                                    withoutRoomId(event),
                                 ],
                                 prev_batch: "f_1_1",
                             },
@@ -941,11 +985,10 @@ describe("MatrixClient event timelines", function() {
 
         // a state event, followed by a redaction thereof
         const event = utils.mkMembership({
-            room: roomId, mship: "join", user: otherUserId,
+            mship: "join", user: otherUserId,
         });
         const redaction = utils.mkEvent({
             type: "m.room.redaction",
-            room_id: roomId,
             sender: otherUserId,
             content: {},
         });
@@ -987,7 +1030,7 @@ describe("MatrixClient event timelines", function() {
                 timeline: {
                     events: [
                         utils.mkMessage({
-                            room: roomId, user: otherUserId, msg: "world",
+                            user: otherUserId, msg: "world",
                         }),
                     ],
                     limited: true,
@@ -1006,4 +1049,75 @@ describe("MatrixClient event timelines", function() {
             expect(tl.getEvents().length).toEqual(1);
         });
     });
+
+    it("should re-insert room IDs for bundled thread relation events", async () => {
+        // @ts-ignore
+        client.clientOpts.experimentalThreadSupport = true;
+        Thread.setServerSideSupport(true, false);
+
+        httpBackend.when("GET", "/sync").respond(200, {
+            next_batch: "s_5_4",
+            rooms: {
+                join: {
+                    [roomId]: {
+                        timeline: {
+                            events: [
+                                SYNC_THREAD_ROOT,
+                            ],
+                            prev_batch: "f_1_1",
+                        },
+                    },
+                },
+            },
+        });
+        await Promise.all([httpBackend.flushAllExpected(), utils.syncPromise(client)]);
+
+        const room = client.getRoom(roomId);
+        const thread = room.getThread(THREAD_ROOT.event_id);
+        const timelineSet = thread.timelineSet;
+
+        httpBackend.when("GET", "/rooms/!foo%3Abar/context/" + encodeURIComponent(THREAD_ROOT.event_id))
+            .respond(200, {
+                start: "start_token",
+                events_before: [],
+                event: THREAD_ROOT,
+                events_after: [],
+                state: [],
+                end: "end_token",
+            });
+        httpBackend.when("GET", "/rooms/!foo%3Abar/relations/" +
+            encodeURIComponent(THREAD_ROOT.event_id) + "/" +
+            encodeURIComponent(THREAD_RELATION_TYPE.name) + "?limit=20")
+            .respond(200, function() {
+                return {
+                    original_event: THREAD_ROOT,
+                    chunk: [THREAD_REPLY],
+                    // no next batch as this is the oldest end of the timeline
+                };
+            });
+        await Promise.all([
+            client.getEventTimeline(timelineSet, THREAD_ROOT.event_id),
+            httpBackend.flushAllExpected(),
+        ]);
+
+        httpBackend.when("GET", "/sync").respond(200, {
+            next_batch: "s_5_5",
+            rooms: {
+                join: {
+                    [roomId]: {
+                        timeline: {
+                            events: [
+                                SYNC_THREAD_REPLY,
+                            ],
+                            prev_batch: "f_1_2",
+                        },
+                    },
+                },
+            },
+        });
+
+        await Promise.all([httpBackend.flushAllExpected(), utils.syncPromise(client)]);
+
+        expect(thread.liveTimeline.getEvents()[1].event).toEqual(THREAD_REPLY);
+    });
 });

spec/integ/matrix-client-methods.spec.js

@@ -27,11 +27,19 @@ describe("MatrixClient", function() {
     let store = null;
     const userId = "@alice:localhost";
     const accessToken = "aseukfgwef";
+    const idServerDomain = "identity.localhost"; // not a real server
+    const identityAccessToken = "woop-i-am-a-secret";
 
     beforeEach(function() {
         store = new MemoryStore();
 
-        const testClient = new TestClient(userId, "aliceDevice", accessToken, undefined, { store });
+        const testClient = new TestClient(userId, "aliceDevice", accessToken, undefined, {
+            store,
+            identityServer: {
+                getAccessToken: () => Promise.resolve(identityAccessToken),
+            },
+            idBaseUrl: `https://${idServerDomain}`,
+        });
         httpBackend = testClient.httpBackend;
         client = testClient.client;
     });
@@ -993,7 +1001,7 @@ describe("MatrixClient", function() {
             };
 
             httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
-                versions: ["r0.5.0"],
+                versions: ["r0.6.0"],
             });
 
             const prom = client.requestRegisterEmailToken("bob@email", "secret", 1);
@@ -1008,6 +1016,64 @@ describe("MatrixClient", function() {
             expect(await prom).toStrictEqual(response);
         });
     });
+
+    describe("inviteByThreePid", () => {
+        it("should supply an id_access_token", async () => {
+            const targetEmail = "gerald@example.org";
+
+            httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
+                versions: ["r0.6.0"],
+            });
+
+            httpBackend.when("POST", "/invite").check(req => {
+                expect(req.data).toStrictEqual({
+                    id_server: idServerDomain,
+                    id_access_token: identityAccessToken,
+                    medium: "email",
+                    address: targetEmail,
+                });
+            }).respond(200, {});
+
+            const prom = client.inviteByThreePid("!room:example.org", "email", targetEmail);
+            await httpBackend.flush();
+            await prom; // returns empty object, so no validation needed
+        });
+    });
+
+    describe("createRoom", () => {
+        it("should populate id_access_token on 3pid invites", async () => {
+            const targetEmail = "gerald@example.org";
+            const response = {
+                room_id: "!room:localhost",
+            };
+            const input = {
+                invite_3pid: [{
+                    // we intentionally exclude the access token here, so it can be populated for us
+                    id_server: idServerDomain,
+                    medium: "email",
+                    address: targetEmail,
+                }],
+            };
+
+            httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
+                versions: ["r0.6.0"],
+            });
+
+            httpBackend.when("POST", "/createRoom").check(req => {
+                expect(req.data).toMatchObject({
+                    invite_3pid: expect.arrayContaining([{
+                        ...input.invite_3pid[0],
+                        id_access_token: identityAccessToken,
+                    }]),
+                });
+                expect(req.data.invite_3pid.length).toBe(1);
+            }).respond(200, response);
+
+            const prom = client.createRoom(input);
+            await httpBackend.flush();
+            expect(await prom).toStrictEqual(response);
+        });
+    });
 });
 
 function withThreadId(event, newThreadId) {

spec/integ/sliding-sync-sdk.spec.ts

@@ -0,0 +1,732 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// eslint-disable-next-line no-restricted-imports
+import MockHttpBackend from "matrix-mock-request";
+import { fail } from "assert";
+
+import { SlidingSync, SlidingSyncEvent, MSC3575RoomData, SlidingSyncState, Extension } from "../../src/sliding-sync";
+import { TestClient } from "../TestClient";
+import { IRoomEvent, IStateEvent } from "../../src/sync-accumulator";
+import {
+    MatrixClient, MatrixEvent, NotificationCountType, JoinRule, MatrixError,
+    EventType, IPushRules, PushRuleKind, TweakName, ClientEvent,
+} from "../../src";
+import { SlidingSyncSdk } from "../../src/sliding-sync-sdk";
+import { SyncState } from "../../src/sync";
+import { IStoredClientOpts } from "../../src/client";
+
+describe("SlidingSyncSdk", () => {
+    let client: MatrixClient = null;
+    let httpBackend: MockHttpBackend = null;
+    let sdk: SlidingSyncSdk = null;
+    let mockSlidingSync: SlidingSync = null;
+    const selfUserId = "@alice:localhost";
+    const selfAccessToken = "aseukfgwef";
+
+    const mockifySlidingSync = (s: SlidingSync): SlidingSync => {
+        s.getList = jest.fn();
+        s.getListData = jest.fn();
+        s.getRoomSubscriptions = jest.fn();
+        s.listLength = jest.fn();
+        s.modifyRoomSubscriptionInfo = jest.fn();
+        s.modifyRoomSubscriptions = jest.fn();
+        s.registerExtension = jest.fn();
+        s.setList = jest.fn();
+        s.setListRanges = jest.fn();
+        s.start = jest.fn();
+        s.stop = jest.fn();
+        s.resend = jest.fn();
+        return s;
+    };
+
+    // shorthand way to make events without filling in all the fields
+    let eventIdCounter = 0;
+    const mkOwnEvent = (evType: string, content: object): IRoomEvent => {
+        eventIdCounter++;
+        return {
+            type: evType,
+            content: content,
+            sender: selfUserId,
+            origin_server_ts: Date.now(),
+            event_id: "$" + eventIdCounter,
+        };
+    };
+    const mkOwnStateEvent = (evType: string, content: object, stateKey?: string): IStateEvent => {
+        eventIdCounter++;
+        return {
+            type: evType,
+            state_key: stateKey,
+            content: content,
+            sender: selfUserId,
+            origin_server_ts: Date.now(),
+            event_id: "$" + eventIdCounter,
+        };
+    };
+    const assertTimelineEvents = (got: MatrixEvent[], want: IRoomEvent[]): void => {
+        expect(got.length).toEqual(want.length);
+        got.forEach((m, i) => {
+            expect(m.getType()).toEqual(want[i].type);
+            expect(m.getSender()).toEqual(want[i].sender);
+            expect(m.getId()).toEqual(want[i].event_id);
+            expect(m.getContent()).toEqual(want[i].content);
+            expect(m.getTs()).toEqual(want[i].origin_server_ts);
+            if (want[i].unsigned) {
+                expect(m.getUnsigned()).toEqual(want[i].unsigned);
+            }
+            const maybeStateEvent = want[i] as IStateEvent;
+            if (maybeStateEvent.state_key) {
+                expect(m.getStateKey()).toEqual(maybeStateEvent.state_key);
+            }
+        });
+    };
+
+    // assign client/httpBackend globals
+    const setupClient = async (testOpts?: Partial<IStoredClientOpts&{withCrypto: boolean}>) => {
+        testOpts = testOpts || {};
+        const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
+        httpBackend = testClient.httpBackend;
+        client = testClient.client;
+        mockSlidingSync = mockifySlidingSync(new SlidingSync("", [], {}, client, 0));
+        if (testOpts.withCrypto) {
+            httpBackend.when("GET", "/room_keys/version").respond(404, {});
+            await client.initCrypto();
+            testOpts.crypto = client.crypto;
+        }
+        httpBackend.when("GET", "/_matrix/client/r0/pushrules").respond(200, {});
+        sdk = new SlidingSyncSdk(mockSlidingSync, client, testOpts);
+    };
+
+    // tear down client/httpBackend globals
+    const teardownClient = () => {
+        client.stopClient();
+        return httpBackend.stop();
+    };
+
+    // find an extension on a SlidingSyncSdk instance
+    const findExtension = (name: string): Extension => {
+        expect(mockSlidingSync.registerExtension).toHaveBeenCalled();
+        const mockFn = mockSlidingSync.registerExtension as jest.Mock;
+        // find the extension
+        for (let i = 0; i < mockFn.mock.calls.length; i++) {
+            const calledExtension = mockFn.mock.calls[i][0] as Extension;
+            if (calledExtension && calledExtension.name() === name) {
+                return calledExtension;
+            }
+        }
+        fail("cannot find extension " + name);
+    };
+
+    describe("sync/stop", () => {
+        beforeAll(async () => {
+            await setupClient();
+        });
+        afterAll(teardownClient);
+        it("can sync()", async () => {
+            const hasSynced = sdk.sync();
+            await httpBackend.flushAllExpected();
+            await hasSynced;
+            expect(mockSlidingSync.start).toBeCalled();
+        });
+        it("can stop()", async () => {
+            sdk.stop();
+            expect(mockSlidingSync.stop).toBeCalled();
+        });
+    });
+
+    describe("rooms", () => {
+        beforeAll(async () => {
+            await setupClient();
+        });
+        afterAll(teardownClient);
+
+        describe("initial", () => {
+            beforeAll(async () => {
+                const hasSynced = sdk.sync();
+                await httpBackend.flushAllExpected();
+                await hasSynced;
+            });
+            // inject some rooms with different fields set.
+            // All rooms are new so they all have initial: true
+            const roomA = "!a_state_and_timeline:localhost";
+            const roomB = "!b_timeline_only:localhost";
+            const roomC = "!c_with_highlight_count:localhost";
+            const roomD = "!d_with_notif_count:localhost";
+            const roomE = "!e_with_invite:localhost";
+            const roomF = "!f_calc_room_name:localhost";
+            const data: Record<string, MSC3575RoomData> = {
+                [roomA]: {
+                    name: "A",
+                    required_state: [
+                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                        mkOwnStateEvent(EventType.RoomName, { name: "A" }, ""),
+                    ],
+                    timeline: [
+                        mkOwnEvent(EventType.RoomMessage, { body: "hello A" }),
+                        mkOwnEvent(EventType.RoomMessage, { body: "world A" }),
+                    ],
+                    initial: true,
+                },
+                [roomB]: {
+                    name: "B",
+                    required_state: [],
+                    timeline: [
+                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                        mkOwnEvent(EventType.RoomMessage, { body: "hello B" }),
+                        mkOwnEvent(EventType.RoomMessage, { body: "world B" }),
+
+                    ],
+                    initial: true,
+                },
+                [roomC]: {
+                    name: "C",
+                    required_state: [],
+                    timeline: [
+                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                        mkOwnEvent(EventType.RoomMessage, { body: "hello C" }),
+                        mkOwnEvent(EventType.RoomMessage, { body: "world C" }),
+                    ],
+                    highlight_count: 5,
+                    initial: true,
+                },
+                [roomD]: {
+                    name: "D",
+                    required_state: [],
+                    timeline: [
+                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                        mkOwnEvent(EventType.RoomMessage, { body: "hello D" }),
+                        mkOwnEvent(EventType.RoomMessage, { body: "world D" }),
+                    ],
+                    notification_count: 5,
+                    initial: true,
+                },
+                [roomE]: {
+                    name: "E",
+                    required_state: [],
+                    timeline: [],
+                    invite_state: [
+                        {
+                            type: EventType.RoomMember,
+                            content: { membership: "invite" },
+                            state_key: selfUserId,
+                            sender: "@bob:localhost",
+                            event_id: "$room_e_invite",
+                            origin_server_ts: 123456,
+                        },
+                        {
+                            type: "m.room.join_rules",
+                            content: { join_rule: "invite" },
+                            state_key: "",
+                            sender: "@bob:localhost",
+                            event_id: "$room_e_join_rule",
+                            origin_server_ts: 123456,
+                        },
+                    ],
+                    initial: true,
+                },
+                [roomF]: {
+                    name: "#foo:localhost",
+                    required_state: [
+                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                        mkOwnStateEvent(EventType.RoomCanonicalAlias, { alias: "#foo:localhost" }, ""),
+                        mkOwnStateEvent(EventType.RoomName, { name: "This should be ignored" }, ""),
+                    ],
+                    timeline: [
+                        mkOwnEvent(EventType.RoomMessage, { body: "hello A" }),
+                        mkOwnEvent(EventType.RoomMessage, { body: "world A" }),
+                    ],
+                    initial: true,
+                },
+            };
+
+            it("can be created with required_state and timeline", () => {
+                mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomA, data[roomA]);
+                const gotRoom = client.getRoom(roomA);
+                expect(gotRoom).toBeDefined();
+                expect(gotRoom.name).toEqual(data[roomA].name);
+                expect(gotRoom.getMyMembership()).toEqual("join");
+                assertTimelineEvents(gotRoom.getLiveTimeline().getEvents().slice(-2), data[roomA].timeline);
+            });
+
+            it("can be created with timeline only", () => {
+                mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomB, data[roomB]);
+                const gotRoom = client.getRoom(roomB);
+                expect(gotRoom).toBeDefined();
+                expect(gotRoom.name).toEqual(data[roomB].name);
+                expect(gotRoom.getMyMembership()).toEqual("join");
+                assertTimelineEvents(gotRoom.getLiveTimeline().getEvents().slice(-5), data[roomB].timeline);
+            });
+
+            it("can be created with a highlight_count", () => {
+                mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomC, data[roomC]);
+                const gotRoom = client.getRoom(roomC);
+                expect(gotRoom).toBeDefined();
+                expect(
+                    gotRoom.getUnreadNotificationCount(NotificationCountType.Highlight),
+                ).toEqual(data[roomC].highlight_count);
+            });
+
+            it("can be created with a notification_count", () => {
+                mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomD, data[roomD]);
+                const gotRoom = client.getRoom(roomD);
+                expect(gotRoom).toBeDefined();
+                expect(
+                    gotRoom.getUnreadNotificationCount(NotificationCountType.Total),
+                ).toEqual(data[roomD].notification_count);
+            });
+
+            it("can be created with invite_state", () => {
+                mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomE, data[roomE]);
+                const gotRoom = client.getRoom(roomE);
+                expect(gotRoom).toBeDefined();
+                expect(gotRoom.getMyMembership()).toEqual("invite");
+                expect(gotRoom.currentState.getJoinRule()).toEqual(JoinRule.Invite);
+            });
+
+            it("uses the 'name' field to caluclate the room name", () => {
+                mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomF, data[roomF]);
+                const gotRoom = client.getRoom(roomF);
+                expect(gotRoom).toBeDefined();
+                expect(
+                    gotRoom.name,
+                ).toEqual(data[roomF].name);
+            });
+
+            describe("updating", () => {
+                it("can update with a new timeline event", async () => {
+                    const newEvent = mkOwnEvent(EventType.RoomMessage, { body: "new event A" });
+                    mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomA, {
+                        timeline: [newEvent],
+                        required_state: [],
+                        name: data[roomA].name,
+                    });
+                    const gotRoom = client.getRoom(roomA);
+                    expect(gotRoom).toBeDefined();
+                    const newTimeline = data[roomA].timeline;
+                    newTimeline.push(newEvent);
+                    assertTimelineEvents(gotRoom.getLiveTimeline().getEvents().slice(-3), newTimeline);
+                });
+
+                it("can update with a new required_state event", async () => {
+                    let gotRoom = client.getRoom(roomB);
+                    expect(gotRoom.getJoinRule()).toEqual(JoinRule.Invite); // default
+                    mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomB, {
+                        required_state: [
+                            mkOwnStateEvent("m.room.join_rules", { join_rule: "restricted" }, ""),
+                        ],
+                        timeline: [],
+                        name: data[roomB].name,
+                    });
+                    gotRoom = client.getRoom(roomB);
+                    expect(gotRoom).toBeDefined();
+                    expect(gotRoom.getJoinRule()).toEqual(JoinRule.Restricted);
+                });
+
+                it("can update with a new highlight_count", async () => {
+                    mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomC, {
+                        name: data[roomC].name,
+                        required_state: [],
+                        timeline: [],
+                        highlight_count: 1,
+                    });
+                    const gotRoom = client.getRoom(roomC);
+                    expect(gotRoom).toBeDefined();
+                    expect(
+                        gotRoom.getUnreadNotificationCount(NotificationCountType.Highlight),
+                    ).toEqual(1);
+                });
+
+                it("can update with a new notification_count", async () => {
+                    mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomD, {
+                        name: data[roomD].name,
+                        required_state: [],
+                        timeline: [],
+                        notification_count: 1,
+                    });
+                    const gotRoom = client.getRoom(roomD);
+                    expect(gotRoom).toBeDefined();
+                    expect(
+                        gotRoom.getUnreadNotificationCount(NotificationCountType.Total),
+                    ).toEqual(1);
+                });
+            });
+        });
+    });
+
+    describe("lifecycle", () => {
+        beforeAll(async () => {
+            await setupClient();
+            const hasSynced = sdk.sync();
+            await httpBackend.flushAllExpected();
+            await hasSynced;
+        });
+        const FAILED_SYNC_ERROR_THRESHOLD = 3; // would be nice to export the const in the actual class...
+
+        it("emits SyncState.Reconnecting when < FAILED_SYNC_ERROR_THRESHOLD & SyncState.Error when over", async () => {
+            mockSlidingSync.emit(
+                SlidingSyncEvent.Lifecycle, SlidingSyncState.Complete,
+                { pos: "h", lists: [], rooms: {}, extensions: {} }, null,
+            );
+            expect(sdk.getSyncState()).toEqual(SyncState.Syncing);
+
+            mockSlidingSync.emit(
+                SlidingSyncEvent.Lifecycle, SlidingSyncState.RequestFinished, null, new Error("generic"),
+            );
+            expect(sdk.getSyncState()).toEqual(SyncState.Reconnecting);
+
+            for (let i = 0; i < FAILED_SYNC_ERROR_THRESHOLD; i++) {
+                mockSlidingSync.emit(
+                    SlidingSyncEvent.Lifecycle, SlidingSyncState.RequestFinished, null, new Error("generic"),
+                );
+            }
+            expect(sdk.getSyncState()).toEqual(SyncState.Error);
+        });
+
+        it("emits SyncState.Syncing after a previous SyncState.Error", async () => {
+            mockSlidingSync.emit(
+                SlidingSyncEvent.Lifecycle,
+                SlidingSyncState.Complete,
+                { pos: "i", lists: [], rooms: {}, extensions: {} },
+                null,
+            );
+            expect(sdk.getSyncState()).toEqual(SyncState.Syncing);
+        });
+
+        it("emits SyncState.Error immediately when receiving M_UNKNOWN_TOKEN and stops syncing", async () => {
+            expect(mockSlidingSync.stop).not.toBeCalled();
+            mockSlidingSync.emit(SlidingSyncEvent.Lifecycle, SlidingSyncState.RequestFinished, null, new MatrixError({
+                errcode: "M_UNKNOWN_TOKEN",
+                message: "Oh no your access token is no longer valid",
+            }));
+            expect(sdk.getSyncState()).toEqual(SyncState.Error);
+            expect(mockSlidingSync.stop).toBeCalled();
+        });
+    });
+
+    describe("opts", () => {
+        afterEach(teardownClient);
+        it("can resolveProfilesToInvites", async () => {
+            await setupClient({
+                resolveInvitesToProfiles: true,
+            });
+            const roomId = "!resolveProfilesToInvites:localhost";
+            const invitee = "@invitee:localhost";
+            const inviteeProfile = {
+                avatar_url: "mxc://foobar",
+                displayname: "The Invitee",
+            };
+            httpBackend.when("GET", "/profile").respond(200, inviteeProfile);
+            mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomId, {
+                initial: true,
+                name: "Room with Invite",
+                required_state: [],
+                timeline: [
+                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                    mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                    mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                    mkOwnStateEvent(EventType.RoomMember, { membership: "invite" }, invitee),
+                ],
+            });
+            await httpBackend.flush("/profile", 1, 1000);
+            const room = client.getRoom(roomId);
+            expect(room).toBeDefined();
+            const inviteeMember = room.getMember(invitee);
+            expect(inviteeMember).toBeDefined();
+            expect(inviteeMember.getMxcAvatarUrl()).toEqual(inviteeProfile.avatar_url);
+            expect(inviteeMember.name).toEqual(inviteeProfile.displayname);
+        });
+    });
+
+    describe("ExtensionE2EE", () => {
+        let ext: Extension;
+        beforeAll(async () => {
+            await setupClient({
+                withCrypto: true,
+            });
+            const hasSynced = sdk.sync();
+            await httpBackend.flushAllExpected();
+            await hasSynced;
+            ext = findExtension("e2ee");
+        });
+        afterAll(async () => {
+            // needed else we do some async operations in the background which can cause Jest to whine:
+            // "Cannot log after tests are done. Did you forget to wait for something async in your test?"
+            // Attempted to log "Saving device tracking data null"."
+            client.crypto.stop();
+        });
+        it("gets enabled on the initial request only", () => {
+            expect(ext.onRequest(true)).toEqual({
+                enabled: true,
+            });
+            expect(ext.onRequest(false)).toEqual(undefined);
+        });
+        it("can update device lists", () => {
+            ext.onResponse({
+                device_lists: {
+                    changed: ["@alice:localhost"],
+                    left: ["@bob:localhost"],
+                },
+            });
+            // TODO: more assertions?
+        });
+        it("can update OTK counts", () => {
+            client.crypto.updateOneTimeKeyCount = jest.fn();
+            ext.onResponse({
+                device_one_time_keys_count: {
+                    signed_curve25519: 42,
+                },
+            });
+            expect(client.crypto.updateOneTimeKeyCount).toHaveBeenCalledWith(42);
+            ext.onResponse({
+                device_one_time_keys_count: {
+                    not_signed_curve25519: 42,
+                    // missing field -> default to 0
+                },
+            });
+            expect(client.crypto.updateOneTimeKeyCount).toHaveBeenCalledWith(0);
+        });
+        it("can update fallback keys", () => {
+            ext.onResponse({
+                device_unused_fallback_key_types: ["signed_curve25519"],
+            });
+            expect(client.crypto.getNeedsNewFallback()).toEqual(false);
+            ext.onResponse({
+                device_unused_fallback_key_types: ["not_signed_curve25519"],
+            });
+            expect(client.crypto.getNeedsNewFallback()).toEqual(true);
+        });
+    });
+    describe("ExtensionAccountData", () => {
+        let ext: Extension;
+        beforeAll(async () => {
+            await setupClient();
+            const hasSynced = sdk.sync();
+            await httpBackend.flushAllExpected();
+            await hasSynced;
+            ext = findExtension("account_data");
+        });
+        it("gets enabled on the initial request only", () => {
+            expect(ext.onRequest(true)).toEqual({
+                enabled: true,
+            });
+            expect(ext.onRequest(false)).toEqual(undefined);
+        });
+        it("processes global account data", async () => {
+            const globalType = "global_test";
+            const globalContent = {
+                info: "here",
+            };
+            let globalData = client.getAccountData(globalType);
+            expect(globalData).toBeUndefined();
+            ext.onResponse({
+                global: [
+                    {
+                        type: globalType,
+                        content: globalContent,
+                    },
+                ],
+            });
+            globalData = client.getAccountData(globalType);
+            expect(globalData).toBeDefined();
+            expect(globalData.getContent()).toEqual(globalContent);
+        });
+        it("processes rooms account data", async () => {
+            const roomId = "!room:id";
+            mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomId, {
+                name: "Room with account data",
+                required_state: [],
+                timeline: [
+                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                    mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                    mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
+                    mkOwnEvent(EventType.RoomMessage, { body: "hello" }),
+
+                ],
+                initial: true,
+            });
+            const roomContent = {
+                foo: "bar",
+            };
+            const roomType = "test";
+            ext.onResponse({
+                rooms: {
+                    [roomId]: [
+                        {
+                            type: roomType,
+                            content: roomContent,
+                        },
+                    ],
+                },
+            });
+            const room = client.getRoom(roomId);
+            expect(room).toBeDefined();
+            const event = room.getAccountData(roomType);
+            expect(event).toBeDefined();
+            expect(event.getContent()).toEqual(roomContent);
+        });
+        it("doesn't crash for unknown room account data", async () => {
+            const unknownRoomId = "!unknown:id";
+            const roomType = "tester";
+            ext.onResponse({
+                rooms: {
+                    [unknownRoomId]: [
+                        {
+                            type: roomType,
+                            content: {
+                                foo: "Bar",
+                            },
+                        },
+                    ],
+                },
+            });
+            const room = client.getRoom(unknownRoomId);
+            expect(room).toBeNull();
+            expect(client.getAccountData(roomType)).toBeUndefined();
+        });
+        it("can update push rules via account data", async () => {
+            const roomId = "!foo:bar";
+            const pushRulesContent: IPushRules = {
+                global: {
+                    [PushRuleKind.RoomSpecific]: [{
+                        enabled: true,
+                        default: true,
+                        pattern: "monkey",
+                        actions: [
+                            {
+                                set_tweak: TweakName.Sound,
+                                value: "default",
+                            },
+                        ],
+                        rule_id: roomId,
+                    }],
+                },
+            };
+            let pushRule = client.getRoomPushRule("global", roomId);
+            expect(pushRule).toBeUndefined();
+            ext.onResponse({
+                global: [
+                    {
+                        type: EventType.PushRules,
+                        content: pushRulesContent,
+                    },
+                ],
+            });
+            pushRule = client.getRoomPushRule("global", roomId);
+            expect(pushRule).toEqual(pushRulesContent.global[PushRuleKind.RoomSpecific][0]);
+        });
+    });
+    describe("ExtensionToDevice", () => {
+        let ext: Extension;
+        beforeAll(async () => {
+            await setupClient();
+            const hasSynced = sdk.sync();
+            await httpBackend.flushAllExpected();
+            await hasSynced;
+            ext = findExtension("to_device");
+        });
+        it("gets enabled with a limit on the initial request only", () => {
+            const reqJson: any = ext.onRequest(true);
+            expect(reqJson.enabled).toEqual(true);
+            expect(reqJson.limit).toBeGreaterThan(0);
+            expect(reqJson.since).toBeUndefined();
+        });
+        it("updates the since value", async () => {
+            ext.onResponse({
+                next_batch: "12345",
+                events: [],
+            });
+            expect(ext.onRequest(false)).toEqual({
+                since: "12345",
+            });
+        });
+        it("can handle missing fields", async () => {
+            ext.onResponse({
+                next_batch: "23456",
+                // no events array
+            });
+        });
+        it("emits to-device events on the client", async () => {
+            const toDeviceType = "custom_test";
+            const toDeviceContent = {
+                foo: "bar",
+            };
+            let called = false;
+            client.once(ClientEvent.ToDeviceEvent, (ev) => {
+                expect(ev.getContent()).toEqual(toDeviceContent);
+                expect(ev.getType()).toEqual(toDeviceType);
+                called = true;
+            });
+            ext.onResponse({
+                next_batch: "34567",
+                events: [
+                    {
+                        type: toDeviceType,
+                        content: toDeviceContent,
+                    },
+                ],
+            });
+            expect(called).toBe(true);
+        });
+        it("can cancel key verification requests", async () => {
+            const seen: Record<string, boolean> = {};
+            client.on(ClientEvent.ToDeviceEvent, (ev) => {
+                const evType = ev.getType();
+                expect(seen[evType]).toBeFalsy();
+                seen[evType] = true;
+                if (evType === "m.key.verification.start" || evType === "m.key.verification.request") {
+                    expect(ev.isCancelled()).toEqual(true);
+                } else {
+                    expect(ev.isCancelled()).toEqual(false);
+                }
+            });
+            ext.onResponse({
+                next_batch: "45678",
+                events: [
+                    // someone tries to verify keys
+                    {
+                        type: "m.key.verification.start",
+                        content: {
+                            transaction_id: "a",
+                        },
+                    },
+                    {
+                        type: "m.key.verification.request",
+                        content: {
+                            transaction_id: "a",
+                        },
+                    },
+                    // then gives up
+                    {
+                        type: "m.key.verification.cancel",
+                        content: {
+                            transaction_id: "a",
+                        },
+                    },
+                ],
+            });
+        });
+    });
+});

spec/integ/sliding-sync.spec.ts

@@ -0,0 +1,758 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// eslint-disable-next-line no-restricted-imports
+import EventEmitter from "events";
+import MockHttpBackend from "matrix-mock-request";
+
+import { SlidingSync, SlidingSyncState, ExtensionState, SlidingSyncEvent } from "../../src/sliding-sync";
+import { TestClient } from "../TestClient";
+import { logger } from "../../src/logger";
+import { MatrixClient } from "../../src";
+import { sleep } from "../../src/utils";
+
+/**
+ * Tests for sliding sync. These tests are broken down into sub-tests which are reliant upon one another.
+ * Each test suite (describe block) uses a single MatrixClient/HTTPBackend and a single SlidingSync class.
+ * Each test will call different functions on SlidingSync which may depend on state from previous tests.
+ */
+describe("SlidingSync", () => {
+    let client: MatrixClient = null;
+    let httpBackend: MockHttpBackend = null;
+    const selfUserId = "@alice:localhost";
+    const selfAccessToken = "aseukfgwef";
+    const proxyBaseUrl = "http://localhost:8008";
+    const syncUrl = proxyBaseUrl + "/_matrix/client/unstable/org.matrix.msc3575/sync";
+
+    // assign client/httpBackend globals
+    const setupClient = () => {
+        const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
+        httpBackend = testClient.httpBackend;
+        client = testClient.client;
+    };
+
+    // tear down client/httpBackend globals
+    const teardownClient = () => {
+        httpBackend.verifyNoOutstandingExpectation();
+        client.stopClient();
+        return httpBackend.stop();
+    };
+
+    describe("start/stop", () => {
+        beforeAll(setupClient);
+        afterAll(teardownClient);
+        let slidingSync: SlidingSync;
+
+        it("should start the sync loop upon calling start()", async () => {
+            slidingSync = new SlidingSync(proxyBaseUrl, [], {}, client, 1);
+            const fakeResp = {
+                pos: "a",
+                lists: [],
+                rooms: {},
+                extensions: {},
+            };
+            httpBackend.when("POST", syncUrl).respond(200, fakeResp);
+            const p = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state, resp, err) => {
+                expect(state).toEqual(SlidingSyncState.RequestFinished);
+                expect(resp).toEqual(fakeResp);
+                expect(err).toBeFalsy();
+                return true;
+            });
+            slidingSync.start();
+            await httpBackend.flushAllExpected();
+            await p;
+        });
+
+        it("should stop the sync loop upon calling stop()", () => {
+            slidingSync.stop();
+            httpBackend.verifyNoOutstandingExpectation();
+        });
+    });
+
+    describe("room subscriptions", () => {
+        beforeAll(setupClient);
+        afterAll(teardownClient);
+        const roomId = "!foo:bar";
+        const anotherRoomID = "!another:room";
+        let roomSubInfo = {
+            timeline_limit: 1,
+            required_state: [
+                ["m.room.name", ""],
+            ],
+        };
+        const wantRoomData = {
+            name: "foo bar",
+            required_state: [],
+            timeline: [],
+        };
+
+        let slidingSync: SlidingSync;
+
+        it("should be able to subscribe to a room", async () => {
+            // add the subscription
+            slidingSync = new SlidingSync(proxyBaseUrl, [], roomSubInfo, client, 1);
+            slidingSync.modifyRoomSubscriptions(new Set([roomId]));
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("room sub", body);
+                expect(body.room_subscriptions).toBeTruthy();
+                expect(body.room_subscriptions[roomId]).toEqual(roomSubInfo);
+            }).respond(200, {
+                pos: "a",
+                lists: [],
+                extensions: {},
+                rooms: {
+                    [roomId]: wantRoomData,
+                },
+            });
+
+            const p = listenUntil(slidingSync, "SlidingSync.RoomData", (gotRoomId, gotRoomData) => {
+                expect(gotRoomId).toEqual(roomId);
+                expect(gotRoomData).toEqual(wantRoomData);
+                return true;
+            });
+            slidingSync.start();
+            await httpBackend.flushAllExpected();
+            await p;
+        });
+
+        it("should be possible to adjust room subscription info whilst syncing", async () => {
+            // listen for updated request
+            const newSubInfo = {
+                timeline_limit: 100,
+                required_state: [
+                    ["m.room.member", "*"],
+                ],
+            };
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("adjusted sub", body);
+                expect(body.room_subscriptions).toBeTruthy();
+                expect(body.room_subscriptions[roomId]).toEqual(newSubInfo);
+            }).respond(200, {
+                pos: "a",
+                lists: [],
+                extensions: {},
+                rooms: {
+                    [roomId]: wantRoomData,
+                },
+            });
+
+            const p = listenUntil(slidingSync, "SlidingSync.RoomData", (gotRoomId, gotRoomData) => {
+                expect(gotRoomId).toEqual(roomId);
+                expect(gotRoomData).toEqual(wantRoomData);
+                return true;
+            });
+
+            slidingSync.modifyRoomSubscriptionInfo(newSubInfo);
+            await httpBackend.flushAllExpected();
+            await p;
+            // need to set what the new subscription info is for subsequent tests
+            roomSubInfo = newSubInfo;
+        });
+
+        it("should be possible to add room subscriptions whilst syncing", async () => {
+            // listen for updated request
+            const anotherRoomData = {
+                name: "foo bar 2",
+                room_id: anotherRoomID,
+                // we should not fall over if fields are missing.
+                // required_state: [],
+                // timeline: [],
+            };
+            const anotherRoomDataFixed = {
+                name: anotherRoomData.name,
+                room_id: anotherRoomID,
+                required_state: [],
+                timeline: [],
+            };
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("new subs", body);
+                expect(body.room_subscriptions).toBeTruthy();
+                // only the new room is sent, the other is sticky
+                expect(body.room_subscriptions[anotherRoomID]).toEqual(roomSubInfo);
+                expect(body.room_subscriptions[roomId]).toBeUndefined();
+            }).respond(200, {
+                pos: "b",
+                lists: [],
+                extensions: {},
+                rooms: {
+                    [anotherRoomID]: anotherRoomData,
+                },
+            });
+
+            const p = listenUntil(slidingSync, "SlidingSync.RoomData", (gotRoomId, gotRoomData) => {
+                expect(gotRoomId).toEqual(anotherRoomID);
+                expect(gotRoomData).toEqual(anotherRoomDataFixed);
+                return true;
+            });
+
+            const subs = slidingSync.getRoomSubscriptions();
+            subs.add(anotherRoomID);
+            slidingSync.modifyRoomSubscriptions(subs);
+            await httpBackend.flushAllExpected();
+            await p;
+        });
+
+        it("should be able to unsubscribe from a room", async () => {
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("unsub request", body);
+                expect(body.room_subscriptions).toBeFalsy();
+                expect(body.unsubscribe_rooms).toEqual([roomId]);
+            }).respond(200, {
+                pos: "b",
+                lists: [],
+            });
+
+            const p = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+
+            // remove the subscription for the first room
+            slidingSync.modifyRoomSubscriptions(new Set([anotherRoomID]));
+
+            await httpBackend.flushAllExpected();
+            await p;
+
+            slidingSync.stop();
+        });
+    });
+
+    describe("lists", () => {
+        beforeAll(setupClient);
+        afterAll(teardownClient);
+
+        const roomA = "!a:localhost";
+        const roomB = "!b:localhost";
+        const roomC = "!c:localhost";
+        const rooms = {
+            [roomA]: {
+                name: "A",
+                required_state: [],
+                timeline: [],
+            },
+            [roomB]: {
+                name: "B",
+                required_state: [],
+                timeline: [],
+            },
+            [roomC]: {
+                name: "C",
+                required_state: [],
+                timeline: [],
+            },
+        };
+        const newRanges = [[0, 2], [3, 5]];
+
+        let slidingSync: SlidingSync;
+        it("should be possible to subscribe to a list", async () => {
+            // request first 3 rooms
+            const listReq = {
+                ranges: [[0, 2]],
+                sort: ["by_name"],
+                timeline_limit: 1,
+                required_state: [
+                    ["m.room.topic", ""],
+                ],
+                filters: {
+                    is_dm: true,
+                },
+            };
+            slidingSync = new SlidingSync(proxyBaseUrl, [listReq], {}, client, 1);
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("list", body);
+                expect(body.lists).toBeTruthy();
+                expect(body.lists[0]).toEqual(listReq);
+            }).respond(200, {
+                pos: "a",
+                lists: [{
+                    count: 500,
+                    ops: [{
+                        op: "SYNC",
+                        range: [0, 2],
+                        room_ids: Object.keys(rooms),
+                    }],
+                }],
+                rooms: rooms,
+            });
+            const listenerData = {};
+            const dataListener = (roomId, roomData) => {
+                expect(listenerData[roomId]).toBeFalsy();
+                listenerData[roomId] = roomData;
+            };
+            slidingSync.on(SlidingSyncEvent.RoomData, dataListener);
+            const responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+            slidingSync.start();
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+
+            expect(listenerData[roomA]).toEqual(rooms[roomA]);
+            expect(listenerData[roomB]).toEqual(rooms[roomB]);
+            expect(listenerData[roomC]).toEqual(rooms[roomC]);
+
+            expect(slidingSync.listLength()).toEqual(1);
+            slidingSync.off(SlidingSyncEvent.RoomData, dataListener);
+        });
+
+        it("should be possible to retrieve list data", () => {
+            expect(slidingSync.getList(0)).toBeDefined();
+            expect(slidingSync.getList(5)).toBeNull();
+            expect(slidingSync.getListData(5)).toBeNull();
+            const syncData = slidingSync.getListData(0);
+            expect(syncData.joinedCount).toEqual(500); // from previous test
+            expect(syncData.roomIndexToRoomId).toEqual({
+                0: roomA,
+                1: roomB,
+                2: roomC,
+            });
+        });
+
+        it("should be possible to adjust list ranges", async () => {
+            // modify the list ranges
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("next ranges", body.lists[0].ranges);
+                expect(body.lists).toBeTruthy();
+                expect(body.lists[0]).toEqual({
+                    // only the ranges should be sent as the rest are unchanged and sticky
+                    ranges: newRanges,
+                });
+            }).respond(200, {
+                pos: "b",
+                lists: [{
+                    count: 500,
+                    ops: [{
+                        op: "SYNC",
+                        range: [0, 2],
+                        room_ids: Object.keys(rooms),
+                    }],
+                }],
+            });
+
+            const responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.RequestFinished;
+            });
+            slidingSync.setListRanges(0, newRanges);
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+        });
+
+        it("should be possible to add an extra list", async () => {
+            // add extra list
+            const extraListReq = {
+                ranges: [[0, 100]],
+                sort: ["by_name"],
+                filters: {
+                    "is_dm": true,
+                },
+            };
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("extra list", body);
+                expect(body.lists).toBeTruthy();
+                expect(body.lists[0]).toEqual({
+                    // only the ranges should be sent as the rest are unchanged and sticky
+                    ranges: newRanges,
+                });
+                expect(body.lists[1]).toEqual(extraListReq);
+            }).respond(200, {
+                pos: "c",
+                lists: [
+                    {
+                        count: 500,
+                    },
+                    {
+                        count: 50,
+                        ops: [{
+                            op: "SYNC",
+                            range: [0, 2],
+                            room_ids: Object.keys(rooms),
+                        }],
+                    },
+                ],
+            });
+            listenUntil(slidingSync, "SlidingSync.List", (listIndex, joinedCount, roomIndexToRoomId) => {
+                expect(listIndex).toEqual(1);
+                expect(joinedCount).toEqual(50);
+                expect(roomIndexToRoomId).toEqual({
+                    0: roomA,
+                    1: roomB,
+                    2: roomC,
+                });
+                return true;
+            });
+            const responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+            slidingSync.setList(1, extraListReq);
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+        });
+
+        it("should be possible to get list DELETE/INSERTs", async () => {
+            // move C (2) to A (0)
+            httpBackend.when("POST", syncUrl).respond(200, {
+                pos: "e",
+                lists: [{
+                    count: 500,
+                    ops: [{
+                        op: "DELETE",
+                        index: 2,
+                    }, {
+                        op: "INSERT",
+                        index: 0,
+                        room_id: roomC,
+                    }],
+                },
+                {
+                    count: 50,
+                }],
+            });
+            let listPromise = listenUntil(slidingSync, "SlidingSync.List",
+                (listIndex, joinedCount, roomIndexToRoomId) => {
+                    expect(listIndex).toEqual(0);
+                    expect(joinedCount).toEqual(500);
+                    expect(roomIndexToRoomId).toEqual({
+                        0: roomC,
+                        1: roomA,
+                        2: roomB,
+                    });
+                    return true;
+                });
+            let responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+            await listPromise;
+
+            // move C (0) back to A (2)
+            httpBackend.when("POST", syncUrl).respond(200, {
+                pos: "f",
+                lists: [{
+                    count: 500,
+                    ops: [{
+                        op: "DELETE",
+                        index: 0,
+                    }, {
+                        op: "INSERT",
+                        index: 2,
+                        room_id: roomC,
+                    }],
+                },
+                {
+                    count: 50,
+                }],
+            });
+            listPromise = listenUntil(slidingSync, "SlidingSync.List",
+                (listIndex, joinedCount, roomIndexToRoomId) => {
+                    expect(listIndex).toEqual(0);
+                    expect(joinedCount).toEqual(500);
+                    expect(roomIndexToRoomId).toEqual({
+                        0: roomA,
+                        1: roomB,
+                        2: roomC,
+                    });
+                    return true;
+                });
+            responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+            await listPromise;
+        });
+
+        it("should ignore invalid list indexes", async () => {
+            httpBackend.when("POST", syncUrl).respond(200, {
+                pos: "e",
+                lists: [{
+                    count: 500,
+                    ops: [{
+                        op: "DELETE",
+                        index: 2324324,
+                    }],
+                },
+                {
+                    count: 50,
+                }],
+            });
+            const listPromise = listenUntil(slidingSync, "SlidingSync.List",
+                (listIndex, joinedCount, roomIndexToRoomId) => {
+                    expect(listIndex).toEqual(0);
+                    expect(joinedCount).toEqual(500);
+                    expect(roomIndexToRoomId).toEqual({
+                        0: roomA,
+                        1: roomB,
+                        2: roomC,
+                    });
+                    return true;
+                });
+            const responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+            await listPromise;
+        });
+
+        it("should be possible to update a list", async () => {
+            httpBackend.when("POST", syncUrl).respond(200, {
+                pos: "g",
+                lists: [{
+                    count: 42,
+                    ops: [
+                        {
+                            op: "INVALIDATE",
+                            range: [0, 2],
+                        },
+                        {
+                            op: "SYNC",
+                            range: [0, 1],
+                            room_ids: [roomB, roomC],
+                        },
+                    ],
+                },
+                {
+                    count: 50,
+                }],
+            });
+            // update the list with a new filter
+            slidingSync.setList(0, {
+                filters: {
+                    is_encrypted: true,
+                },
+                ranges: [[0, 100]],
+            });
+            const listPromise = listenUntil(slidingSync, "SlidingSync.List",
+                (listIndex, joinedCount, roomIndexToRoomId) => {
+                    expect(listIndex).toEqual(0);
+                    expect(joinedCount).toEqual(42);
+                    expect(roomIndexToRoomId).toEqual({
+                        0: roomB,
+                        1: roomC,
+                    });
+                    return true;
+                });
+            const responseProcessed = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state) => {
+                return state === SlidingSyncState.Complete;
+            });
+            await httpBackend.flushAllExpected();
+            await responseProcessed;
+            await listPromise;
+            slidingSync.stop();
+        });
+    });
+
+    describe("extensions", () => {
+        beforeAll(setupClient);
+        afterAll(teardownClient);
+        let slidingSync: SlidingSync;
+        const extReq = {
+            foo: "bar",
+        };
+        const extResp = {
+            baz: "quuz",
+        };
+
+        // Pre-extensions get called BEFORE processing the sync response
+        const preExtName = "foobar";
+        let onPreExtensionRequest;
+        let onPreExtensionResponse;
+
+        // Post-extensions get called AFTER processing the sync response
+        const postExtName = "foobar2";
+        let onPostExtensionRequest;
+        let onPostExtensionResponse;
+
+        const extPre = {
+            name: () => preExtName,
+            onRequest: (initial) => { return onPreExtensionRequest(initial); },
+            onResponse: (res) => { return onPreExtensionResponse(res); },
+            when: () => ExtensionState.PreProcess,
+        };
+        const extPost = {
+            name: () => postExtName,
+            onRequest: (initial) => { return onPostExtensionRequest(initial); },
+            onResponse: (res) => { return onPostExtensionResponse(res); },
+            when: () => ExtensionState.PostProcess,
+        };
+
+        it("should be able to register an extension", async () => {
+            slidingSync = new SlidingSync(proxyBaseUrl, [], {}, client, 1);
+            slidingSync.registerExtension(extPre);
+
+            const callbackOrder = [];
+            let extensionOnResponseCalled = false;
+            onPreExtensionRequest = () => {
+                return extReq;
+            };
+            onPreExtensionResponse = (resp) => {
+                extensionOnResponseCalled = true;
+                callbackOrder.push("onPreExtensionResponse");
+                expect(resp).toEqual(extResp);
+            };
+
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("ext req", body);
+                expect(body.extensions).toBeTruthy();
+                expect(body.extensions[preExtName]).toEqual(extReq);
+            }).respond(200, {
+                pos: "a",
+                ops: [],
+                counts: [],
+                extensions: {
+                    [preExtName]: extResp,
+                },
+            });
+
+            const p = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state, resp, err) => {
+                if (state === SlidingSyncState.Complete) {
+                    callbackOrder.push("Lifecycle");
+                    return true;
+                }
+            });
+            slidingSync.start();
+            await httpBackend.flushAllExpected();
+            await p;
+            expect(extensionOnResponseCalled).toBe(true);
+            expect(callbackOrder).toEqual(["onPreExtensionResponse", "Lifecycle"]);
+        });
+
+        it("should be able to send nothing in an extension request/response", async () => {
+            onPreExtensionRequest = () => {
+                return undefined;
+            };
+            let responseCalled = false;
+            onPreExtensionResponse = (resp) => {
+                responseCalled = true;
+            };
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("ext req nothing", body);
+                expect(body.extensions).toBeTruthy();
+                expect(body.extensions[preExtName]).toBeUndefined();
+            }).respond(200, {
+                pos: "a",
+                ops: [],
+                counts: [],
+                extensions: {},
+            });
+            // we need to resend as sliding sync will already have a buffered request with the old
+            // extension values from the previous test.
+            slidingSync.resend();
+
+            const p = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state, resp, err) => {
+                return state === SlidingSyncState.Complete;
+            });
+            await httpBackend.flushAllExpected();
+            await p;
+            expect(responseCalled).toBe(false);
+        });
+
+        it("is possible to register extensions after start() has been called", async () => {
+            slidingSync.registerExtension(extPost);
+            onPostExtensionRequest = () => {
+                return extReq;
+            };
+            let responseCalled = false;
+            const callbackOrder = [];
+            onPostExtensionResponse = (resp) => {
+                expect(resp).toEqual(extResp);
+                responseCalled = true;
+                callbackOrder.push("onPostExtensionResponse");
+            };
+            httpBackend.when("POST", syncUrl).check(function(req) {
+                const body = req.data;
+                logger.log("ext req after start", body);
+                expect(body.extensions).toBeTruthy();
+                expect(body.extensions[preExtName]).toBeUndefined(); // from the earlier test
+                expect(body.extensions[postExtName]).toEqual(extReq);
+            }).respond(200, {
+                pos: "c",
+                ops: [],
+                counts: [],
+                extensions: {
+                    [postExtName]: extResp,
+                },
+            });
+            // we need to resend as sliding sync will already have a buffered request with the old
+            // extension values from the previous test.
+            slidingSync.resend();
+
+            const p = listenUntil(slidingSync, "SlidingSync.Lifecycle", (state, resp, err) => {
+                if (state === SlidingSyncState.Complete) {
+                    callbackOrder.push("Lifecycle");
+                    return true;
+                }
+            });
+            await httpBackend.flushAllExpected();
+            await p;
+            expect(responseCalled).toBe(true);
+            expect(callbackOrder).toEqual(["Lifecycle", "onPostExtensionResponse"]);
+            slidingSync.stop();
+        });
+
+        it("is not possible to register the same extension name twice", async () => {
+            slidingSync = new SlidingSync(proxyBaseUrl, [], {}, client, 1);
+            slidingSync.registerExtension(extPre);
+            expect(() => { slidingSync.registerExtension(extPre); }).toThrow();
+        });
+    });
+});
+
+async function timeout(delayMs: number, reason: string): Promise<never> {
+    await sleep(delayMs);
+    throw new Error(`timeout: ${delayMs}ms - ${reason}`);
+}
+
+/**
+ * Listen until a callback returns data.
+ * @param {EventEmitter} emitter The event emitter
+ * @param {string} eventName The event to listen for
+ * @param {function} callback The callback which will be invoked when events fire. Return something truthy from this to resolve the promise.
+ * @param {number} timeoutMs The number of milliseconds to wait for the callback to return data. Default: 500ms.
+ * @returns {Promise} A promise which will be resolved when the callback returns data. If the callback throws or the timeout is reached,
+ * the promise is rejected.
+ */
+function listenUntil<T>(
+    emitter: EventEmitter,
+    eventName: string,
+    callback: (...args: any[]) => T,
+    timeoutMs = 500,
+): Promise<T> {
+    const trace = new Error().stack.split(`\n`)[2];
+    return Promise.race([new Promise<T>((resolve, reject) => {
+        const wrapper = (...args) => {
+            try {
+                const data = callback(...args);
+                if (data) {
+                    emitter.off(eventName, wrapper);
+                    resolve(data);
+                }
+            } catch (err) {
+                reject(err);
+            }
+        };
+        emitter.on(eventName, wrapper);
+    }), timeout(timeoutMs, "timed out waiting for event " + eventName + " " + trace)]);
+}

spec/test-utils/test-utils.ts

@@ -70,7 +70,7 @@ export function mock<T>(constr: { new(...args: any[]): T }, name: string): T {
 
 interface IEventOpts {
     type: EventType | string;
-    room: string;
+    room?: string;
     sender?: string;
     skey?: string;
     content: IContent;
@@ -93,8 +93,8 @@ let testEventIndex = 1; // counter for events, easier for comparison of randomly
  * @return {Object} a JSON object representing this event.
  */
 export function mkEvent(opts: IEventOpts & { event: true }, client?: MatrixClient): MatrixEvent;
-export function mkEvent(opts: IEventOpts & { event?: false }, client?: MatrixClient): object;
-export function mkEvent(opts: IEventOpts & { event?: boolean }, client?: MatrixClient): object | MatrixEvent {
+export function mkEvent(opts: IEventOpts & { event?: false }, client?: MatrixClient): Partial<IEvent>;
+export function mkEvent(opts: IEventOpts & { event?: boolean }, client?: MatrixClient): Partial<IEvent> | MatrixEvent {
     if (!opts.type || !opts.content) {
         throw new Error("Missing .type or .content =>" + JSON.stringify(opts));
     }
@@ -129,6 +129,21 @@ export function mkEvent(opts: IEventOpts & { event?: boolean }, client?: MatrixC
     return opts.event ? new MatrixEvent(event) : event;
 }
 
+type GeneratedMetadata = {
+    event_id: string;
+    txn_id: string;
+    origin_server_ts: number;
+};
+
+export function mkEventCustom<T>(base: T): T & GeneratedMetadata {
+    return {
+        event_id: "$" + testEventIndex++ + "-" + Math.random() + "-" + Math.random(),
+        txn_id: "~" + Math.random(),
+        origin_server_ts: Date.now(),
+        ...base,
+    };
+}
+
 interface IPresenceOpts {
     user?: string;
     sender?: string;
@@ -145,8 +160,8 @@ interface IPresenceOpts {
  * @return {Object|MatrixEvent} The event
  */
 export function mkPresence(opts: IPresenceOpts & { event: true }): MatrixEvent;
-export function mkPresence(opts: IPresenceOpts & { event?: false }): object;
-export function mkPresence(opts: IPresenceOpts & { event?: boolean }): object | MatrixEvent {
+export function mkPresence(opts: IPresenceOpts & { event?: false }): Partial<IEvent>;
+export function mkPresence(opts: IPresenceOpts & { event?: boolean }): Partial<IEvent> | MatrixEvent {
     const event = {
         event_id: "$" + Math.random() + "-" + Math.random(),
         type: "m.presence",
@@ -162,7 +177,7 @@ export function mkPresence(opts: IPresenceOpts & { event?: boolean }): object |
 }
 
 interface IMembershipOpts {
-    room: string;
+    room?: string;
     mship: string;
     sender?: string;
     user?: string;
@@ -186,8 +201,8 @@ interface IMembershipOpts {
  * @return {Object|MatrixEvent} The event
  */
 export function mkMembership(opts: IMembershipOpts & { event: true }): MatrixEvent;
-export function mkMembership(opts: IMembershipOpts & { event?: false }): object;
-export function mkMembership(opts: IMembershipOpts & { event?: boolean }): object | MatrixEvent {
+export function mkMembership(opts: IMembershipOpts & { event?: false }): Partial<IEvent>;
+export function mkMembership(opts: IMembershipOpts & { event?: boolean }): Partial<IEvent> | MatrixEvent {
     const eventOpts: IEventOpts = {
         ...opts,
         type: EventType.RoomMember,
@@ -208,8 +223,20 @@ export function mkMembership(opts: IMembershipOpts & { event?: boolean }): objec
     return mkEvent(eventOpts);
 }
 
+export function mkMembershipCustom<T>(
+    base: T & { membership: string, sender: string, content?: IContent },
+): T & { type: EventType, sender: string, state_key: string, content: IContent } & GeneratedMetadata {
+    const content = base.content || {};
+    return mkEventCustom({
+        ...base,
+        content: { ...content, membership: base.membership },
+        type: EventType.RoomMember,
+        state_key: base.sender,
+    });
+}
+
 interface IMessageOpts {
-    room: string;
+    room?: string;
     user: string;
     msg?: string;
     event?: boolean;
@@ -226,8 +253,11 @@ interface IMessageOpts {
  * @return {Object|MatrixEvent} The event
  */
 export function mkMessage(opts: IMessageOpts & { event: true }, client?: MatrixClient): MatrixEvent;
-export function mkMessage(opts: IMessageOpts & { event?: false }, client?: MatrixClient): object;
-export function mkMessage(opts: IMessageOpts & { event?: boolean }, client?: MatrixClient): object | MatrixEvent {
+export function mkMessage(opts: IMessageOpts & { event?: false }, client?: MatrixClient): Partial<IEvent>;
+export function mkMessage(
+    opts: IMessageOpts & { event?: boolean },
+    client?: MatrixClient,
+): Partial<IEvent> | MatrixEvent {
     const eventOpts: IEventOpts = {
         ...opts,
         type: EventType.RoomMessage,
@@ -260,11 +290,11 @@ interface IReplyMessageOpts extends IMessageOpts {
  * @return {Object|MatrixEvent} The event
  */
 export function mkReplyMessage(opts: IReplyMessageOpts & { event: true }, client?: MatrixClient): MatrixEvent;
-export function mkReplyMessage(opts: IReplyMessageOpts & { event?: false }, client?: MatrixClient): object;
+export function mkReplyMessage(opts: IReplyMessageOpts & { event?: false }, client?: MatrixClient): Partial<IEvent>;
 export function mkReplyMessage(
     opts: IReplyMessageOpts & { event?: boolean },
     client?: MatrixClient,
-): object | MatrixEvent {
+): Partial<IEvent> | MatrixEvent {
     const eventOpts: IEventOpts = {
         ...opts,
         type: EventType.RoomMessage,

spec/test-utils/webrtc.ts

@@ -0,0 +1,146 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export const DUMMY_SDP = (
+    "v=0\r\n" +
+    "o=- 5022425983810148698 2 IN IP4 127.0.0.1\r\n" +
+    "s=-\r\nt=0 0\r\na=group:BUNDLE 0\r\n" +
+    "a=msid-semantic: WMS h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA\r\n" +
+    "m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126\r\n" +
+    "c=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:hLDR\r\n" +
+    "a=ice-pwd:bMGD9aOldHWiI+6nAq/IIlRw\r\n" +
+    "a=ice-options:trickle\r\n" +
+    "a=fingerprint:sha-256 E4:94:84:F9:4A:98:8A:56:F5:5F:FD:AF:72:B9:32:89:49:5C:4B:9A:" +
+    "4A:15:8E:41:8A:F3:69:E4:39:52:DC:D6\r\n" +
+    "a=setup:active\r\n" +
+    "a=mid:0\r\n" +
+    "a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n" +
+    "a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n" +
+    "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n" +
+    "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n" +
+    "a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n" +
+    "a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n" +
+    "a=sendrecv\r\n" +
+    "a=msid:h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA 4357098f-3795-4131-bff4-9ba9c0348c49\r\n" +
+    "a=rtcp-mux\r\n" +
+    "a=rtpmap:111 opus/48000/2\r\n" +
+    "a=rtcp-fb:111 transport-cc\r\n" +
+    "a=fmtp:111 minptime=10;useinbandfec=1\r\n" +
+    "a=rtpmap:103 ISAC/16000\r\n" +
+    "a=rtpmap:104 ISAC/32000\r\n" +
+    "a=rtpmap:9 G722/8000\r\n" +
+    "a=rtpmap:0 PCMU/8000\r\n" +
+    "a=rtpmap:8 PCMA/8000\r\n" +
+    "a=rtpmap:106 CN/32000\r\n" +
+    "a=rtpmap:105 CN/16000\r\n" +
+    "a=rtpmap:13 CN/8000\r\n" +
+    "a=rtpmap:110 telephone-event/48000\r\n" +
+    "a=rtpmap:112 telephone-event/32000\r\n" +
+    "a=rtpmap:113 telephone-event/16000\r\n" +
+    "a=rtpmap:126 telephone-event/8000\r\n" +
+    "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n"
+);
+
+export class MockRTCPeerConnection {
+    localDescription: RTCSessionDescription;
+
+    constructor() {
+        this.localDescription = {
+            sdp: DUMMY_SDP,
+            type: 'offer',
+            toJSON: function() { },
+        };
+    }
+
+    addEventListener() { }
+    createDataChannel(label: string, opts: RTCDataChannelInit) { return { label, ...opts }; }
+    createOffer() {
+        return Promise.resolve({});
+    }
+    setRemoteDescription() {
+        return Promise.resolve();
+    }
+    setLocalDescription() {
+        return Promise.resolve();
+    }
+    close() { }
+    getStats() { return []; }
+    addTrack(track: MockMediaStreamTrack) { return new MockRTCRtpSender(track); }
+}
+
+export class MockRTCRtpSender {
+    constructor(public track: MockMediaStreamTrack) { }
+
+    replaceTrack(track: MockMediaStreamTrack) { this.track = track; }
+}
+
+export class MockMediaStreamTrack {
+    constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) { }
+
+    stop() { }
+}
+
+// XXX: Using EventTarget in jest doesn't seem to work, so we write our own
+// implementation
+export class MockMediaStream {
+    constructor(
+        public id: string,
+        private tracks: MockMediaStreamTrack[] = [],
+    ) {}
+
+    listeners: [string, (...args: any[]) => any][] = [];
+
+    dispatchEvent(eventType: string) {
+        this.listeners.forEach(([t, c]) => {
+            if (t !== eventType) return;
+            c();
+        });
+    }
+    getTracks() { return this.tracks; }
+    getAudioTracks() { return this.tracks.filter((track) => track.kind === "audio"); }
+    getVideoTracks() { return this.tracks.filter((track) => track.kind === "video"); }
+    addEventListener(eventType: string, callback: (...args: any[]) => any) {
+        this.listeners.push([eventType, callback]);
+    }
+    removeEventListener(eventType: string, callback: (...args: any[]) => any) {
+        this.listeners.filter(([t, c]) => {
+            return t !== eventType || c !== callback;
+        });
+    }
+    addTrack(track: MockMediaStreamTrack) {
+        this.tracks.push(track);
+        this.dispatchEvent("addtrack");
+    }
+    removeTrack(track: MockMediaStreamTrack) { this.tracks.splice(this.tracks.indexOf(track), 1); }
+}
+
+export class MockMediaDeviceInfo {
+    constructor(
+        public kind: "audio" | "video",
+    ) { }
+}
+
+export class MockMediaHandler {
+    getUserMediaStream(audio: boolean, video: boolean) {
+        const tracks = [];
+        if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio"));
+        if (video) tracks.push(new MockMediaStreamTrack("video_track", "video"));
+
+        return new MockMediaStream("mock_stream_from_media_handler", tracks);
+    }
+    stopUserMediaStream() { }
+    hasAudioDevice() { return true; }
+}

spec/unit/autodiscovery.spec.ts

@@ -1,6 +1,6 @@
 /*
 Copyright 2018 New Vector Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2022 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,19 +17,20 @@ limitations under the License.
 
 import MockHttpBackend from "matrix-mock-request";
 
-import * as sdk from "../../src";
+import { request } from "../../src/matrix";
 import { AutoDiscovery } from "../../src/autodiscovery";
 
 describe("AutoDiscovery", function() {
-    let httpBackend = null;
-
-    beforeEach(function() {
-        httpBackend = new MockHttpBackend();
-        sdk.request(httpBackend.requestFn);
-    });
+    const getHttpBackend = (): MockHttpBackend => {
+        const httpBackend = new MockHttpBackend();
+        request(httpBackend.requestFn);
+        return httpBackend;
+    };
 
     it("should throw an error when no domain is specified", function() {
+        getHttpBackend();
         return Promise.all([
+            // @ts-ignore testing no args
             AutoDiscovery.findClientConfig(/* no args */).then(() => {
                 throw new Error("Expected a failure, not success with no args");
             }, () => {
@@ -42,13 +43,13 @@ describe("AutoDiscovery", function() {
                 return true;
             }),
 
-            AutoDiscovery.findClientConfig(null).then(() => {
+            AutoDiscovery.findClientConfig(null as any).then(() => {
                 throw new Error("Expected a failure, not success with null");
             }, () => {
                 return true;
             }),
 
-            AutoDiscovery.findClientConfig(true).then(() => {
+            AutoDiscovery.findClientConfig(true as any).then(() => {
                 throw new Error("Expected a failure, not success with a non-string");
             }, () => {
                 return true;
@@ -57,6 +58,7 @@ describe("AutoDiscovery", function() {
     });
 
     it("should return PROMPT when .well-known 404s", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/.well-known/matrix/client").respond(404, {});
         return Promise.all([
             httpBackend.flushAllExpected(),
@@ -80,6 +82,7 @@ describe("AutoDiscovery", function() {
     });
 
     it("should return FAIL_PROMPT when .well-known returns a 500 error", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/.well-known/matrix/client").respond(500, {});
         return Promise.all([
             httpBackend.flushAllExpected(),
@@ -103,6 +106,7 @@ describe("AutoDiscovery", function() {
     });
 
     it("should return FAIL_PROMPT when .well-known returns a 400 error", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/.well-known/matrix/client").respond(400, {});
         return Promise.all([
             httpBackend.flushAllExpected(),
@@ -126,6 +130,7 @@ describe("AutoDiscovery", function() {
     });
 
     it("should return FAIL_PROMPT when .well-known returns an empty body", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, "");
         return Promise.all([
             httpBackend.flushAllExpected(),
@@ -149,6 +154,7 @@ describe("AutoDiscovery", function() {
     });
 
     it("should return FAIL_PROMPT when .well-known returns not-JSON", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, "abc");
         return Promise.all([
             httpBackend.flushAllExpected(),
@@ -173,6 +179,7 @@ describe("AutoDiscovery", function() {
 
     it("should return FAIL_PROMPT when .well-known does not have a base_url for " +
         "m.homeserver (empty string)", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {
                 base_url: "",
@@ -201,6 +208,7 @@ describe("AutoDiscovery", function() {
 
     it("should return FAIL_PROMPT when .well-known does not have a base_url for " +
         "m.homeserver (no property)", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {},
         });
@@ -227,6 +235,7 @@ describe("AutoDiscovery", function() {
 
     it("should return FAIL_ERROR when .well-known has an invalid base_url for " +
         "m.homeserver (disallowed scheme)", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {
                 base_url: "mxc://example.org",
@@ -255,6 +264,7 @@ describe("AutoDiscovery", function() {
 
     it("should return FAIL_ERROR when .well-known has an invalid base_url for " +
         "m.homeserver (verification failure: 404)", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/_matrix/client/versions").respond(404, {});
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {
@@ -284,6 +294,7 @@ describe("AutoDiscovery", function() {
 
     it("should return FAIL_ERROR when .well-known has an invalid base_url for " +
         "m.homeserver (verification failure: 500)", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/_matrix/client/versions").respond(500, {});
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {
@@ -313,6 +324,7 @@ describe("AutoDiscovery", function() {
 
     it("should return FAIL_ERROR when .well-known has an invalid base_url for " +
         "m.homeserver (verification failure: 200 but wrong content)", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
             not_matrix_versions: ["r0.0.1"],
         });
@@ -344,8 +356,9 @@ describe("AutoDiscovery", function() {
 
     it("should return SUCCESS when .well-known has a verifiably accurate base_url for " +
         "m.homeserver", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
-            expect(req.opts.uri).toEqual("https://example.org/_matrix/client/versions");
+            expect(req.path).toEqual("https://example.org/_matrix/client/versions");
         }).respond(200, {
             versions: ["r0.0.1"],
         });
@@ -376,8 +389,9 @@ describe("AutoDiscovery", function() {
     });
 
     it("should return SUCCESS with the right homeserver URL", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
-            expect(req.opts.uri)
+            expect(req.path)
                 .toEqual("https://chat.example.org/_matrix/client/versions");
         }).respond(200, {
             versions: ["r0.0.1"],
@@ -411,8 +425,9 @@ describe("AutoDiscovery", function() {
 
     it("should return SUCCESS / FAIL_PROMPT when the identity server configuration " +
         "is wrong (missing base_url)", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
-            expect(req.opts.uri)
+            expect(req.path)
                 .toEqual("https://chat.example.org/_matrix/client/versions");
         }).respond(200, {
             versions: ["r0.0.1"],
@@ -451,8 +466,9 @@ describe("AutoDiscovery", function() {
 
     it("should return SUCCESS / FAIL_PROMPT when the identity server configuration " +
         "is wrong (empty base_url)", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
-            expect(req.opts.uri)
+            expect(req.path)
                 .toEqual("https://chat.example.org/_matrix/client/versions");
         }).respond(200, {
             versions: ["r0.0.1"],
@@ -491,8 +507,9 @@ describe("AutoDiscovery", function() {
 
     it("should return SUCCESS / FAIL_PROMPT when the identity server configuration " +
         "is wrong (validation error: 404)", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
-            expect(req.opts.uri)
+            expect(req.path)
                 .toEqual("https://chat.example.org/_matrix/client/versions");
         }).respond(200, {
             versions: ["r0.0.1"],
@@ -532,8 +549,9 @@ describe("AutoDiscovery", function() {
 
     it("should return SUCCESS / FAIL_PROMPT when the identity server configuration " +
         "is wrong (validation error: 500)", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
-            expect(req.opts.uri)
+            expect(req.path)
                 .toEqual("https://chat.example.org/_matrix/client/versions");
         }).respond(200, {
             versions: ["r0.0.1"],
@@ -573,14 +591,15 @@ describe("AutoDiscovery", function() {
 
     it("should return SUCCESS when the identity server configuration is " +
         "verifiably accurate", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
-            expect(req.opts.uri)
+            expect(req.path)
                 .toEqual("https://chat.example.org/_matrix/client/versions");
         }).respond(200, {
             versions: ["r0.0.1"],
         });
         httpBackend.when("GET", "/_matrix/identity/api/v1").check((req) => {
-            expect(req.opts.uri)
+            expect(req.path)
                 .toEqual("https://identity.example.org/_matrix/identity/api/v1");
         }).respond(200, {});
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
@@ -615,14 +634,15 @@ describe("AutoDiscovery", function() {
 
     it("should return SUCCESS and preserve non-standard keys from the " +
         ".well-known response", function() {
+        const httpBackend = getHttpBackend();
         httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
-            expect(req.opts.uri)
+            expect(req.path)
                 .toEqual("https://chat.example.org/_matrix/client/versions");
         }).respond(200, {
             versions: ["r0.0.1"],
         });
         httpBackend.when("GET", "/_matrix/identity/api/v1").check((req) => {
-            expect(req.opts.uri)
+            expect(req.path)
                 .toEqual("https://identity.example.org/_matrix/identity/api/v1");
         }).respond(200, {});
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {

spec/unit/content-repo.spec.ts

@@ -19,41 +19,41 @@ describe("ContentRepo", function() {
         });
 
         it("should return a download URL if no width/height/resize are specified",
-        function() {
-            const mxcUri = "mxc://server.name/resourceid";
-            expect(getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
-                baseUrl + "/_matrix/media/r0/download/server.name/resourceid",
-            );
-        });
+            function() {
+                const mxcUri = "mxc://server.name/resourceid";
+                expect(getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
+                    baseUrl + "/_matrix/media/r0/download/server.name/resourceid",
+                );
+            });
 
         it("should return the empty string for null input", function() {
-            expect(getHttpUriForMxc(null)).toEqual("");
+            expect(getHttpUriForMxc(null as any, '')).toEqual("");
         });
 
         it("should return a thumbnail URL if a width/height/resize is specified",
-        function() {
-            const mxcUri = "mxc://server.name/resourceid";
-            expect(getHttpUriForMxc(baseUrl, mxcUri, 32, 64, "crop")).toEqual(
-                baseUrl + "/_matrix/media/r0/thumbnail/server.name/resourceid" +
+            function() {
+                const mxcUri = "mxc://server.name/resourceid";
+                expect(getHttpUriForMxc(baseUrl, mxcUri, 32, 64, "crop")).toEqual(
+                    baseUrl + "/_matrix/media/r0/thumbnail/server.name/resourceid" +
                 "?width=32&height=64&method=crop",
-            );
-        });
+                );
+            });
 
         it("should put fragments from mxc:// URIs after any query parameters",
-        function() {
-            const mxcUri = "mxc://server.name/resourceid#automade";
-            expect(getHttpUriForMxc(baseUrl, mxcUri, 32)).toEqual(
-                baseUrl + "/_matrix/media/r0/thumbnail/server.name/resourceid" +
+            function() {
+                const mxcUri = "mxc://server.name/resourceid#automade";
+                expect(getHttpUriForMxc(baseUrl, mxcUri, 32)).toEqual(
+                    baseUrl + "/_matrix/media/r0/thumbnail/server.name/resourceid" +
                 "?width=32#automade",
-            );
-        });
+                );
+            });
 
         it("should put fragments from mxc:// URIs at the end of the HTTP URI",
-        function() {
-            const mxcUri = "mxc://server.name/resourceid#automade";
-            expect(getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
-                baseUrl + "/_matrix/media/r0/download/server.name/resourceid#automade",
-            );
-        });
+            function() {
+                const mxcUri = "mxc://server.name/resourceid#automade";
+                expect(getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
+                    baseUrl + "/_matrix/media/r0/download/server.name/resourceid#automade",
+                );
+            });
     });
 });

spec/unit/crypto.spec.ts

@@ -2,6 +2,7 @@ import '../olm-loader';
 // eslint-disable-next-line no-restricted-imports
 import { EventEmitter } from "events";
 
+import { MatrixClient } from "../../src/client";
 import { Crypto } from "../../src/crypto";
 import { MemoryCryptoStore } from "../../src/crypto/store/memory-crypto-store";
 import { MockStorageApi } from "../MockStorageApi";
@@ -25,7 +26,7 @@ function awaitEvent(emitter, event) {
     });
 }
 
-async function keyshareEventForEvent(client, event, index) {
+async function keyshareEventForEvent(client, event, index): Promise<MatrixEvent> {
     const roomId = event.getRoomId();
     const eventContent = event.getWireContent();
     const key = await client.crypto.olmDevice.getInboundGroupSessionKey(
@@ -50,6 +51,7 @@ async function keyshareEventForEvent(client, event, index) {
         },
     });
     // make onRoomKeyEvent think this was an encrypted event
+    // @ts-ignore private property
     ksEvent.senderCurve25519Key = "akey";
     return ksEvent;
 }
@@ -63,6 +65,10 @@ describe("Crypto", function() {
         return Olm.init();
     });
 
+    afterEach(() => {
+        jest.useRealTimers();
+    });
+
     it("Crypto exposes the correct olm library version", function() {
         expect(Crypto.getOlmVersion()[0]).toEqual(3);
     });
@@ -79,7 +85,7 @@ describe("Crypto", function() {
                 getId: () => "$event_id",
                 getSenderKey: () => null,
                 getWireContent: () => {return {};},
-            };
+            } as unknown as MatrixEvent;
 
             let encryptionInfo = client.getEventEncryptionInfo(event);
             expect(encryptionInfo.encrypted).toBeFalsy();
@@ -154,12 +160,15 @@ describe("Crypto", function() {
         beforeEach(async function() {
             const mockStorage = new MockStorageApi();
             const clientStore = new MemoryStore({ localStorage: mockStorage });
-            const cryptoStore = new MemoryCryptoStore(mockStorage);
+            const cryptoStore = new MemoryCryptoStore();
 
             cryptoStore.storeEndToEndDeviceData({
                 devices: {
                     '@bob:home.server': {
                         'BOBDEVICE': {
+                            algorithms: [],
+                            verified: 1,
+                            known: false,
                             keys: {
                                 'curve25519:BOBDEVICE': 'this is a key',
                             },
@@ -167,7 +176,7 @@ describe("Crypto", function() {
                     },
                 },
                 trackingStatus: {},
-            });
+            }, {});
 
             mockBaseApis = {
                 sendToDevice: jest.fn(),
@@ -185,6 +194,7 @@ describe("Crypto", function() {
                 clientStore,
                 cryptoStore,
                 mockRoomList,
+                [],
             );
             crypto.registerEventHandlers(fakeEmitter);
             await crypto.init();
@@ -195,7 +205,7 @@ describe("Crypto", function() {
         });
 
         it("restarts wedged Olm sessions", async function() {
-            const prom = new Promise((resolve) => {
+            const prom = new Promise<void>((resolve) => {
                 mockBaseApis.claimOneTimeKeys = function() {
                     resolve();
                     return otkResponse;
@@ -220,8 +230,8 @@ describe("Crypto", function() {
     });
 
     describe('Key requests', function() {
-        let aliceClient;
-        let bobClient;
+        let aliceClient: MatrixClient;
+        let bobClient: MatrixClient;
 
         beforeEach(async function() {
             aliceClient = (new TestClient(
@@ -276,8 +286,12 @@ describe("Crypto", function() {
                 // alice encrypts each event, and then bob tries to decrypt
                 // them without any keys, so that they'll be in pending
                 await aliceClient.crypto.encryptEvent(event, aliceRoom);
+                // remove keys from the event
+                // @ts-ignore private properties
                 event.clearEvent = undefined;
+                // @ts-ignore private properties
                 event.senderCurve25519Key = null;
+                // @ts-ignore private properties
                 event.claimedEd25519Key = null;
                 try {
                     await bobClient.crypto.decryptEvent(event);
@@ -291,7 +305,7 @@ describe("Crypto", function() {
                 roomId, olmlib.MEGOLM_ALGORITHM,
             );
 
-            let eventPromise = Promise.all(events.map((ev) => {
+            const decryptEventsPromise = Promise.all(events.map((ev) => {
                 return awaitEvent(ev, "Event.decrypted");
             }));
 
@@ -300,11 +314,11 @@ describe("Crypto", function() {
             // can
             let ksEvent = await keyshareEventForEvent(aliceClient, events[1], 1);
             await bobDecryptor.onRoomKeyEvent(ksEvent);
-            await eventPromise;
+            await decryptEventsPromise;
             expect(events[0].getContent().msgtype).toBe("m.bad.encrypted");
             expect(events[1].getContent().msgtype).not.toBe("m.bad.encrypted");
 
-            const cryptoStore = bobClient.cryptoStore;
+            const cryptoStore = bobClient.crypto.cryptoStore;
             const eventContent = events[0].getWireContent();
             const senderKey = eventContent.sender_key;
             const sessionId = eventContent.session_id;
@@ -320,10 +334,10 @@ describe("Crypto", function() {
 
             // keyshare the session key starting at the first message, so
             // that it can now be decrypted
-            eventPromise = awaitEvent(events[0], "Event.decrypted");
+            const decryptEventPromise = awaitEvent(events[0], "Event.decrypted");
             ksEvent = await keyshareEventForEvent(aliceClient, events[0], 0);
             await bobDecryptor.onRoomKeyEvent(ksEvent);
-            await eventPromise;
+            await decryptEventPromise;
             expect(events[0].getContent().msgtype).not.toBe("m.bad.encrypted");
             await sleep(1);
             // the room key request should be gone since we've now decrypted everything
@@ -354,8 +368,12 @@ describe("Crypto", function() {
             // alice encrypts each event, and then bob tries to decrypt
             // them without any keys, so that they'll be in pending
             await aliceClient.crypto.encryptEvent(event, aliceRoom);
+            // remove keys from the event
+            // @ts-ignore private property
             event.clearEvent = undefined;
+            // @ts-ignore private property
             event.senderCurve25519Key = null;
+            // @ts-ignore private property
             event.claimedEd25519Key = null;
             try {
                 await bobClient.crypto.decryptEvent(event);
@@ -370,9 +388,9 @@ describe("Crypto", function() {
 
             const ksEvent = await keyshareEventForEvent(aliceClient, event, 1);
             ksEvent.getContent().sender_key = undefined; // test
-            bobClient.crypto.addInboundGroupSession = jest.fn();
+            bobClient.crypto.olmDevice.addInboundGroupSession = jest.fn();
             await bobDecryptor.onRoomKeyEvent(ksEvent);
-            expect(bobClient.crypto.addInboundGroupSession).not.toHaveBeenCalled();
+            expect(bobClient.crypto.olmDevice.addInboundGroupSession).not.toHaveBeenCalled();
         });
 
         it("creates a new keyshare request if we request a keyshare", async function() {
@@ -388,7 +406,7 @@ describe("Crypto", function() {
                 },
             });
             await aliceClient.cancelAndResendEventRoomKeyRequest(event);
-            const cryptoStore = aliceClient.cryptoStore;
+            const cryptoStore = aliceClient.crypto.cryptoStore;
             const roomKeyRequestBody = {
                 algorithm: olmlib.MEGOLM_ALGORITHM,
                 room_id: "!someroom",
@@ -412,7 +430,8 @@ describe("Crypto", function() {
                 },
             });
             // replace Alice's sendToDevice function with a mock
-            aliceClient.sendToDevice = jest.fn().mockResolvedValue(undefined);
+            const aliceSendToDevice = jest.fn().mockResolvedValue(undefined);
+            aliceClient.sendToDevice = aliceSendToDevice;
             aliceClient.startClient();
 
             // make a room key request, and record the transaction ID for the
@@ -421,11 +440,12 @@ describe("Crypto", function() {
             // key requests get queued until the sync has finished, but we don't
             // let the client set up enough for that to happen, so gut-wrench a bit
             // to force it to send now.
+            // @ts-ignore
             aliceClient.crypto.outgoingRoomKeyRequestManager.sendQueuedRequests();
             jest.runAllTimers();
             await Promise.resolve();
-            expect(aliceClient.sendToDevice).toBeCalledTimes(1);
-            const txnId = aliceClient.sendToDevice.mock.calls[0][2];
+            expect(aliceSendToDevice).toBeCalledTimes(1);
+            const txnId = aliceSendToDevice.mock.calls[0][2];
 
             // give the room key request manager time to update the state
             // of the request
@@ -438,8 +458,8 @@ describe("Crypto", function() {
             // cancelAndResend will call sendToDevice twice:
             // the first call to sendToDevice will be the cancellation
             // the second call to sendToDevice will be the key request
-            expect(aliceClient.sendToDevice).toBeCalledTimes(3);
-            expect(aliceClient.sendToDevice.mock.calls[2][2]).not.toBe(txnId);
+            expect(aliceSendToDevice).toBeCalledTimes(3);
+            expect(aliceSendToDevice.mock.calls[2][2]).not.toBe(txnId);
         });
     });
 
@@ -451,7 +471,7 @@ describe("Crypto", function() {
             await client.initCrypto();
             client.crypto.getSecretStorageKey = async () => null;
             client.crypto.isCrossSigningReady = async () => false;
-            client.crypto.baseApis.uploadDeviceSigningKeys = () => null;
+            client.crypto.baseApis.uploadDeviceSigningKeys = jest.fn().mockResolvedValue(null);
             client.crypto.baseApis.setAccountData = () => null;
             client.crypto.baseApis.uploadKeySignatures = () => null;
             client.crypto.baseApis.http.authedRequest = () => null;
@@ -467,4 +487,105 @@ describe("Crypto", function() {
             client.stopClient();
         });
     });
+
+    describe("encryptAndSendToDevices", () => {
+        let client: TestClient;
+        let ensureOlmSessionsForDevices: jest.SpiedFunction<typeof olmlib.ensureOlmSessionsForDevices>;
+        let encryptMessageForDevice: jest.SpiedFunction<typeof olmlib.encryptMessageForDevice>;
+        const payload = { hello: "world" };
+        let encryptedPayload: object;
+
+        beforeEach(async () => {
+            ensureOlmSessionsForDevices = jest.spyOn(olmlib, "ensureOlmSessionsForDevices");
+            ensureOlmSessionsForDevices.mockResolvedValue({});
+            encryptMessageForDevice = jest.spyOn(olmlib, "encryptMessageForDevice");
+            encryptMessageForDevice.mockImplementation(async (...[result,,,,,, payload]) => {
+                result.plaintext = JSON.stringify(payload);
+            });
+
+            client = new TestClient("@alice:example.org", "aliceweb");
+            await client.client.initCrypto();
+
+            encryptedPayload = {
+                algorithm: "m.olm.v1.curve25519-aes-sha2",
+                sender_key: client.client.crypto.olmDevice.deviceCurve25519Key,
+                ciphertext: { plaintext: JSON.stringify(payload) },
+            };
+        });
+
+        afterEach(async () => {
+            ensureOlmSessionsForDevices.mockRestore();
+            encryptMessageForDevice.mockRestore();
+            await client.stop();
+        });
+
+        it("encrypts and sends to devices", async () => {
+            client.httpBackend
+                .when("PUT", "/sendToDevice/m.room.encrypted", {
+                    messages: {
+                        "@bob:example.org": {
+                            bobweb: encryptedPayload,
+                            bobmobile: encryptedPayload,
+                        },
+                        "@carol:example.org": {
+                            caroldesktop: encryptedPayload,
+                        },
+                    },
+                })
+                .respond(200, {});
+
+            await Promise.all([
+                client.client.encryptAndSendToDevices(
+                    [
+                        { userId: "@bob:example.org", deviceInfo: new DeviceInfo("bobweb") },
+                        { userId: "@bob:example.org", deviceInfo: new DeviceInfo("bobmobile") },
+                        { userId: "@carol:example.org", deviceInfo: new DeviceInfo("caroldesktop") },
+                    ],
+                    payload,
+                ),
+                client.httpBackend.flushAllExpected(),
+            ]);
+        });
+
+        it("sends nothing to devices that couldn't be encrypted to", async () => {
+            encryptMessageForDevice.mockImplementation(async (...[result,,,, userId, device, payload]) => {
+                // Refuse to encrypt to Carol's desktop device
+                if (userId === "@carol:example.org" && device.deviceId === "caroldesktop") return;
+                result.plaintext = JSON.stringify(payload);
+            });
+
+            client.httpBackend
+                .when("PUT", "/sendToDevice/m.room.encrypted", {
+                    // Carol is nowhere to be seen
+                    messages: { "@bob:example.org": { bobweb: encryptedPayload } },
+                })
+                .respond(200, {});
+
+            await Promise.all([
+                client.client.encryptAndSendToDevices(
+                    [
+                        { userId: "@bob:example.org", deviceInfo: new DeviceInfo("bobweb") },
+                        { userId: "@carol:example.org", deviceInfo: new DeviceInfo("caroldesktop") },
+                    ],
+                    payload,
+                ),
+                client.httpBackend.flushAllExpected(),
+            ]);
+        });
+
+        it("no-ops if no devices can be encrypted to", async () => {
+            // Refuse to encrypt to anybody
+            encryptMessageForDevice.mockResolvedValue(undefined);
+
+            // Get the room keys version request out of the way
+            client.httpBackend.when("GET", "/room_keys/version").respond(404, {});
+            await client.httpBackend.flush("/room_keys/version", 1);
+
+            await client.client.encryptAndSendToDevices(
+                [{ userId: "@bob:example.org", deviceInfo: new DeviceInfo("bobweb") }],
+                payload,
+            );
+            client.httpBackend.verifyNoOutstandingRequests();
+        });
+    });
 });

spec/unit/crypto/algorithms/megolm.spec.ts

@@ -1,15 +1,36 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { mocked, MockedObject } from 'jest-mock';
+
 import '../../../olm-loader';
 import * as algorithms from "../../../../src/crypto/algorithms";
 import { MemoryCryptoStore } from "../../../../src/crypto/store/memory-crypto-store";
-import { MockStorageApi } from "../../../MockStorageApi";
 import * as testUtils from "../../../test-utils/test-utils";
 import { OlmDevice } from "../../../../src/crypto/OlmDevice";
-import { Crypto } from "../../../../src/crypto";
+import { Crypto, IncomingRoomKeyRequest } from "../../../../src/crypto";
 import { logger } from "../../../../src/logger";
 import { MatrixEvent } from "../../../../src/models/event";
 import { TestClient } from "../../../TestClient";
 import { Room } from "../../../../src/models/room";
 import * as olmlib from "../../../../src/crypto/olmlib";
+import { TypedEventEmitter } from '../../../../src/models/typed-event-emitter';
+import { ClientEvent, MatrixClient, RoomMember } from '../../../../src';
+import { DeviceInfo, IDevice } from '../../../../src/crypto/deviceinfo';
+import { DeviceTrustLevel } from '../../../../src/crypto/CrossSigning';
 
 const MegolmDecryption = algorithms.DECRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
 const MegolmEncryption = algorithms.ENCRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
@@ -28,17 +49,20 @@ describe("MegolmDecryption", function() {
         return Olm.init();
     });
 
-    let megolmDecryption;
-    let mockOlmLib;
-    let mockCrypto;
-    let mockBaseApis;
+    let megolmDecryption: algorithms.DecryptionAlgorithm;
+    let mockOlmLib: MockedObject<typeof olmlib>;
+    let mockCrypto: MockedObject<Crypto>;
+    let mockBaseApis: MockedObject<MatrixClient>;
 
     beforeEach(async function() {
-        mockCrypto = testUtils.mock(Crypto, 'Crypto');
-        mockBaseApis = {};
+        mockCrypto = testUtils.mock(Crypto, 'Crypto') as MockedObject<Crypto>;
+        mockBaseApis = {
+            claimOneTimeKeys: jest.fn(),
+            sendToDevice: jest.fn(),
+            queueToDevice: jest.fn(),
+        } as unknown as MockedObject<MatrixClient>;
 
-        const mockStorage = new MockStorageApi();
-        const cryptoStore = new MemoryCryptoStore(mockStorage);
+        const cryptoStore = new MemoryCryptoStore();
 
         const olmDevice = new OlmDevice(cryptoStore);
 
@@ -51,11 +75,15 @@ describe("MegolmDecryption", function() {
         });
 
         // we stub out the olm encryption bits
-        mockOlmLib = {};
-        mockOlmLib.ensureOlmSessionsForDevices = jest.fn();
-        mockOlmLib.encryptMessageForDevice =
-            jest.fn().mockResolvedValue(undefined);
+        mockOlmLib = {
+            encryptMessageForDevice: jest.fn().mockResolvedValue(undefined),
+            ensureOlmSessionsForDevices: jest.fn(),
+        } as unknown as MockedObject<typeof olmlib>;
+
+        // @ts-ignore illegal assignment that makes these tests work :/
         megolmDecryption.olmlib = mockOlmLib;
+
+        jest.clearAllMocks();
     });
 
     describe('receives some keys:', function() {
@@ -87,7 +115,7 @@ describe("MegolmDecryption", function() {
                 decryptEvent: function() {
                     return Promise.resolve(decryptedData);
                 },
-            };
+            } as unknown as Crypto;
 
             await event.attemptDecryption(mockCrypto).then(() => {
                 megolmDecryption.onRoomKeyEvent(event);
@@ -115,10 +143,13 @@ describe("MegolmDecryption", function() {
         });
 
         it('can respond to a key request event', function() {
-            const keyRequest = {
+            const keyRequest: IncomingRoomKeyRequest = {
+                requestId: '123',
+                share: jest.fn(),
                 userId: '@alice:foo',
                 deviceId: 'alidevice',
                 requestBody: {
+                    algorithm: '',
                     room_id: ROOM_ID,
                     sender_key: "SENDER_CURVE25519",
                     session_id: groupSession.session_id(),
@@ -131,23 +162,25 @@ describe("MegolmDecryption", function() {
                 expect(hasKeys).toBe(true);
 
                 // set up some pre-conditions for the share call
-                const deviceInfo = {};
+                const deviceInfo = {} as DeviceInfo;
                 mockCrypto.getStoredDevice.mockReturnValue(deviceInfo);
 
                 mockOlmLib.ensureOlmSessionsForDevices.mockResolvedValue({
                     '@alice:foo': { 'alidevice': {
                         sessionId: 'alisession',
+                        device: new DeviceInfo('alidevice'),
                     } },
                 });
 
-                const awaitEncryptForDevice = new Promise((res, rej) => {
+                const awaitEncryptForDevice = new Promise<void>((res, rej) => {
                     mockOlmLib.encryptMessageForDevice.mockImplementation(() => {
                         res();
                         return Promise.resolve();
                     });
                 });
 
-                mockBaseApis.sendToDevice = jest.fn();
+                mockBaseApis.sendToDevice.mockReset();
+                mockBaseApis.queueToDevice.mockReset();
 
                 // do the share
                 megolmDecryption.shareKeysWithDevice(keyRequest);
@@ -265,17 +298,18 @@ describe("MegolmDecryption", function() {
             let olmDevice;
 
             beforeEach(async () => {
+                // @ts-ignore assigning to readonly prop
                 mockCrypto.backupManager = {
                     backupGroupSession: () => {},
                 };
-                const mockStorage = new MockStorageApi();
-                const cryptoStore = new MemoryCryptoStore(mockStorage);
+                const cryptoStore = new MemoryCryptoStore();
 
                 olmDevice = new OlmDevice(cryptoStore);
                 olmDevice.verifySignature = jest.fn();
                 await olmDevice.init();
 
-                mockBaseApis.claimOneTimeKeys = jest.fn().mockReturnValue(Promise.resolve({
+                mockBaseApis.claimOneTimeKeys.mockResolvedValue({
+                    failures: {},
                     one_time_keys: {
                         '@alice:home.server': {
                             aliceDevice: {
@@ -290,8 +324,9 @@ describe("MegolmDecryption", function() {
                             },
                         },
                     },
-                }));
-                mockBaseApis.sendToDevice = jest.fn().mockResolvedValue(undefined);
+                });
+                mockBaseApis.sendToDevice.mockResolvedValue(undefined);
+                mockBaseApis.queueToDevice.mockResolvedValue(undefined);
 
                 aliceDeviceInfo = {
                     deviceId: 'aliceDevice',
@@ -311,18 +346,30 @@ describe("MegolmDecryption", function() {
 
                 mockCrypto.checkDeviceTrust.mockReturnValue({
                     isVerified: () => false,
-                });
+                } as DeviceTrustLevel);
 
                 megolmEncryption = new MegolmEncryption({
                     userId: '@user:id',
+                    deviceId: '12345',
                     crypto: mockCrypto,
                     olmDevice: olmDevice,
                     baseApis: mockBaseApis,
                     roomId: ROOM_ID,
                     config: {
+                        algorithm: 'm.megolm.v1.aes-sha2',
                         rotation_period_ms: rotationPeriodMs,
                     },
                 });
+
+                // Splice the real method onto the mock object as megolm uses this method
+                // on the crypto class in order to encrypt / start sessions
+                // @ts-ignore Mock
+                mockCrypto.encryptAndSendToDevices = Crypto.prototype.encryptAndSendToDevices;
+                // @ts-ignore Mock
+                mockCrypto.olmDevice = olmDevice;
+                // @ts-ignore Mock
+                mockCrypto.baseApis = mockBaseApis;
+
                 mockRoom = {
                     getEncryptionTargetMembers: jest.fn().mockReturnValue(
                         [{ userId: "@alice:home.server" }],
@@ -369,7 +416,7 @@ describe("MegolmDecryption", function() {
                 expect(mockCrypto.downloadKeys).toHaveBeenCalledWith(
                     ['@alice:home.server'], false,
                 );
-                expect(mockBaseApis.sendToDevice).toHaveBeenCalled();
+                expect(mockBaseApis.queueToDevice).toHaveBeenCalled();
                 expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalledWith(
                     [['@alice:home.server', 'aliceDevice']], 'signed_curve25519', 2000,
                 );
@@ -412,7 +459,7 @@ describe("MegolmDecryption", function() {
                     'YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWI',
                 );
 
-                mockBaseApis.sendToDevice.mockClear();
+                mockBaseApis.queueToDevice.mockClear();
                 await megolmEncryption.reshareKeyWithDevice(
                     olmDevice.deviceCurve25519Key,
                     ct1.session_id,
@@ -420,7 +467,7 @@ describe("MegolmDecryption", function() {
                     aliceDeviceInfo,
                 );
 
-                expect(mockBaseApis.sendToDevice).not.toHaveBeenCalled();
+                expect(mockBaseApis.queueToDevice).not.toHaveBeenCalled();
             });
         });
     });
@@ -449,33 +496,33 @@ describe("MegolmDecryption", function() {
         };
         const roomId = "!someroom";
         const room = new Room(roomId, aliceClient, "@alice:example.com", {});
+
+        const bobMember = new RoomMember(roomId, "@bob:example.com");
         room.getEncryptionTargetMembers = async function() {
-            return [{ userId: "@bob:example.com" }];
+            return [bobMember];
         };
         room.setBlacklistUnverifiedDevices(true);
         aliceClient.store.storeRoom(room);
         await aliceClient.setRoomEncryption(roomId, encryptionCfg);
 
-        const BOB_DEVICES = {
+        const BOB_DEVICES: Record<string, IDevice> = {
             bobdevice1: {
-                user_id: "@bob:example.com",
-                device_id: "bobdevice1",
                 algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
                 keys: {
                     "ed25519:Dynabook": bobDevice1.deviceEd25519Key,
                     "curve25519:Dynabook": bobDevice1.deviceCurve25519Key,
                 },
                 verified: 0,
+                known: false,
             },
             bobdevice2: {
-                user_id: "@bob:example.com",
-                device_id: "bobdevice2",
                 algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
                 keys: {
                     "ed25519:Dynabook": bobDevice2.deviceEd25519Key,
                     "curve25519:Dynabook": bobDevice2.deviceCurve25519Key,
                 },
                 verified: -1,
+                known: false,
             },
         };
 
@@ -486,32 +533,7 @@ describe("MegolmDecryption", function() {
             return this.getDevicesFromStore(userIds);
         };
 
-        let run = false;
-        aliceClient.sendToDevice = async (msgtype, contentMap) => {
-            run = true;
-            expect(msgtype).toMatch(/^(org.matrix|m).room_key.withheld$/);
-            delete contentMap["@bob:example.com"].bobdevice1.session_id;
-            delete contentMap["@bob:example.com"].bobdevice2.session_id;
-            expect(contentMap).toStrictEqual({
-                '@bob:example.com': {
-                    bobdevice1: {
-                        algorithm: "m.megolm.v1.aes-sha2",
-                        room_id: roomId,
-                        code: 'm.unverified',
-                        reason:
-                        'The sender has disabled encrypting to unverified devices.',
-                        sender_key: aliceDevice.deviceCurve25519Key,
-                    },
-                    bobdevice2: {
-                        algorithm: "m.megolm.v1.aes-sha2",
-                        room_id: roomId,
-                        code: 'm.blacklisted',
-                        reason: 'The sender has blocked you.',
-                        sender_key: aliceDevice.deviceCurve25519Key,
-                    },
-                },
-            });
-        };
+        aliceClient.sendToDevice = jest.fn().mockResolvedValue({});
 
         const event = new MatrixEvent({
             type: "m.room.message",
@@ -525,7 +547,30 @@ describe("MegolmDecryption", function() {
         });
         await aliceClient.crypto.encryptEvent(event, room);
 
-        expect(run).toBe(true);
+        expect(aliceClient.sendToDevice).toHaveBeenCalled();
+        const [msgtype, contentMap] = mocked(aliceClient.sendToDevice).mock.calls[0];
+        expect(msgtype).toMatch(/^(org.matrix|m).room_key.withheld$/);
+        delete contentMap["@bob:example.com"].bobdevice1.session_id;
+        delete contentMap["@bob:example.com"].bobdevice2.session_id;
+        expect(contentMap).toStrictEqual({
+            '@bob:example.com': {
+                bobdevice1: {
+                    algorithm: "m.megolm.v1.aes-sha2",
+                    room_id: roomId,
+                    code: 'm.unverified',
+                    reason:
+                    'The sender has disabled encrypting to unverified devices.',
+                    sender_key: aliceDevice.deviceCurve25519Key,
+                },
+                bobdevice2: {
+                    algorithm: "m.megolm.v1.aes-sha2",
+                    room_id: roomId,
+                    code: 'm.blacklisted',
+                    reason: 'The sender has blocked you.',
+                    sender_key: aliceDevice.deviceCurve25519Key,
+                },
+            },
+        });
 
         aliceClient.stopClient();
         bobClient1.stopClient();
@@ -557,18 +602,16 @@ describe("MegolmDecryption", function() {
         await aliceClient.setRoomEncryption(roomId, encryptionCfg);
         await bobClient.setRoomEncryption(roomId, encryptionCfg);
 
-        aliceRoom.getEncryptionTargetMembers = async () => {
-            return [
-                {
-                    userId: "@alice:example.com",
-                    membership: "join",
-                },
-                {
-                    userId: "@bob:example.com",
-                    membership: "join",
-                },
-            ];
-        };
+        aliceRoom.getEncryptionTargetMembers = jest.fn().mockResolvedValue([
+            {
+                userId: "@alice:example.com",
+                membership: "join",
+            },
+            {
+                userId: "@bob:example.com",
+                membership: "join",
+            },
+        ]);
         const BOB_DEVICES = {
             bobdevice: {
                 user_id: "@bob:example.com",
@@ -590,30 +633,14 @@ describe("MegolmDecryption", function() {
             return this.getDevicesFromStore(userIds);
         };
 
-        aliceClient.claimOneTimeKeys = async () => {
+        aliceClient.claimOneTimeKeys = jest.fn().mockResolvedValue({
             // Bob has no one-time keys
-            return {
-                one_time_keys: {},
-            };
-        };
-
-        const sendPromise = new Promise((resolve, reject) => {
-            aliceClient.sendToDevice = async (msgtype, contentMap) => {
-                expect(msgtype).toMatch(/^(org.matrix|m).room_key.withheld$/);
-                expect(contentMap).toStrictEqual({
-                    '@bob:example.com': {
-                        bobdevice: {
-                            algorithm: "m.megolm.v1.aes-sha2",
-                            code: 'm.no_olm',
-                            reason: 'Unable to establish a secure channel.',
-                            sender_key: aliceDevice.deviceCurve25519Key,
-                        },
-                    },
-                });
-                resolve();
-            };
+            one_time_keys: {},
+            failures: {},
         });
 
+        aliceClient.sendToDevice = jest.fn().mockResolvedValue({});
+
         const event = new MatrixEvent({
             type: "m.room.message",
             sender: "@alice:example.com",
@@ -622,7 +649,21 @@ describe("MegolmDecryption", function() {
             content: {},
         });
         await aliceClient.crypto.encryptEvent(event, aliceRoom);
-        await sendPromise;
+
+        expect(aliceClient.sendToDevice).toHaveBeenCalled();
+        const [msgtype, contentMap] = mocked(aliceClient.sendToDevice).mock.calls[0];
+        expect(msgtype).toMatch(/^(org.matrix|m).room_key.withheld$/);
+        expect(contentMap).toStrictEqual({
+            '@bob:example.com': {
+                bobdevice: {
+                    algorithm: "m.megolm.v1.aes-sha2",
+                    code: 'm.no_olm',
+                    reason: 'Unable to establish a secure channel.',
+                    sender_key: aliceDevice.deviceCurve25519Key,
+                },
+            },
+        });
+
         aliceClient.stopClient();
         bobClient.stopClient();
     });
@@ -640,10 +681,13 @@ describe("MegolmDecryption", function() {
         ]);
         const bobDevice = bobClient.crypto.olmDevice;
 
+        const aliceEventEmitter = new TypedEventEmitter<ClientEvent.ToDeviceEvent, any>();
+        aliceClient.crypto.registerEventHandlers(aliceEventEmitter);
+
         const roomId = "!someroom";
 
-        aliceClient.crypto.onToDeviceEvent(new MatrixEvent({
-            type: "org.matrix.room_key.withheld",
+        aliceEventEmitter.emit(ClientEvent.ToDeviceEvent, new MatrixEvent({
+            type: "m.room_key.withheld",
             sender: "@bob:example.com",
             content: {
                 algorithm: "m.megolm.v1.aes-sha2",
@@ -669,7 +713,7 @@ describe("MegolmDecryption", function() {
             },
         }))).rejects.toThrow("The sender has blocked you.");
 
-        aliceClient.crypto.onToDeviceEvent(new MatrixEvent({
+        aliceEventEmitter.emit(ClientEvent.ToDeviceEvent, new MatrixEvent({
             type: "m.room_key.withheld",
             sender: "@bob:example.com",
             content: {
@@ -710,15 +754,19 @@ describe("MegolmDecryption", function() {
             aliceClient.initCrypto(),
             bobClient.initCrypto(),
         ]);
-        aliceClient.crypto.downloadKeys = async () => {};
+
+        const aliceEventEmitter = new TypedEventEmitter<ClientEvent.ToDeviceEvent, any>();
+        aliceClient.crypto.registerEventHandlers(aliceEventEmitter);
+
+        aliceClient.crypto.downloadKeys = jest.fn();
         const bobDevice = bobClient.crypto.olmDevice;
 
         const roomId = "!someroom";
 
         const now = Date.now();
 
-        aliceClient.crypto.onToDeviceEvent(new MatrixEvent({
-            type: "org.matrix.room_key.withheld",
+        aliceEventEmitter.emit(ClientEvent.ToDeviceEvent, new MatrixEvent({
+            type: "m.room_key.withheld",
             sender: "@bob:example.com",
             content: {
                 algorithm: "m.megolm.v1.aes-sha2",
@@ -749,7 +797,7 @@ describe("MegolmDecryption", function() {
             origin_server_ts: now,
         }))).rejects.toThrow("The sender was unable to establish a secure channel.");
 
-        aliceClient.crypto.onToDeviceEvent(new MatrixEvent({
+        aliceEventEmitter.emit(ClientEvent.ToDeviceEvent, new MatrixEvent({
             type: "m.room_key.withheld",
             sender: "@bob:example.com",
             content: {
@@ -795,15 +843,18 @@ describe("MegolmDecryption", function() {
             aliceClient.initCrypto(),
             bobClient.initCrypto(),
         ]);
+        const aliceEventEmitter = new TypedEventEmitter<ClientEvent.ToDeviceEvent, any>();
+        aliceClient.crypto.registerEventHandlers(aliceEventEmitter);
+
         const bobDevice = bobClient.crypto.olmDevice;
-        aliceClient.crypto.downloadKeys = async () => {};
+        aliceClient.crypto.downloadKeys = jest.fn();
 
         const roomId = "!someroom";
 
         const now = Date.now();
 
         // pretend we got an event that we can't decrypt
-        aliceClient.crypto.onToDeviceEvent(new MatrixEvent({
+        aliceEventEmitter.emit(ClientEvent.ToDeviceEvent, new MatrixEvent({
             type: "m.room.encrypted",
             sender: "@bob:example.com",
             content: {

spec/unit/crypto/algorithms/olm.spec.ts

@@ -1,6 +1,6 @@
 /*
 Copyright 2018,2019 New Vector Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2022 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,17 +15,18 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { MockedObject } from 'jest-mock';
+
 import '../../../olm-loader';
 import { MemoryCryptoStore } from "../../../../src/crypto/store/memory-crypto-store";
-import { MockStorageApi } from "../../../MockStorageApi";
 import { logger } from "../../../../src/logger";
 import { OlmDevice } from "../../../../src/crypto/OlmDevice";
 import * as olmlib from "../../../../src/crypto/olmlib";
 import { DeviceInfo } from "../../../../src/crypto/deviceinfo";
+import { MatrixClient } from '../../../../src';
 
 function makeOlmDevice() {
-    const mockStorage = new MockStorageApi();
-    const cryptoStore = new MemoryCryptoStore(mockStorage);
+    const cryptoStore = new MemoryCryptoStore();
     const olmDevice = new OlmDevice(cryptoStore);
     return olmDevice;
 }
@@ -51,8 +52,8 @@ describe("OlmDevice", function() {
         return global.Olm.init();
     });
 
-    let aliceOlmDevice;
-    let bobOlmDevice;
+    let aliceOlmDevice: OlmDevice;
+    let bobOlmDevice: OlmDevice;
 
     beforeEach(async function() {
         aliceOlmDevice = makeOlmDevice();
@@ -69,7 +70,7 @@ describe("OlmDevice", function() {
                 bobOlmDevice.deviceCurve25519Key,
                 sid,
                 "The olm or proteus is an aquatic salamander in the family Proteidae",
-            );
+            ) as any; // OlmDevice.encryptMessage has incorrect return type
 
             const result = await bobOlmDevice.createInboundSession(
                 aliceOlmDevice.deviceCurve25519Key,
@@ -96,7 +97,7 @@ describe("OlmDevice", function() {
                 bobOlmDevice.deviceCurve25519Key,
                 sessionId,
                 MESSAGE,
-            );
+            ) as any; // OlmDevice.encryptMessage has incorrect return type
 
             const bobRecreatedOlmDevice = makeOlmDevice();
             bobRecreatedOlmDevice.init({ fromExportedDevice: exported });
@@ -120,7 +121,7 @@ describe("OlmDevice", function() {
                 bobOlmDevice.deviceCurve25519Key,
                 sessionId,
                 MESSAGE_2,
-            );
+            ) as any; // OlmDevice.encryptMessage has incorrect return type
 
             const bobRecreatedAgainOlmDevice = makeOlmDevice();
             bobRecreatedAgainOlmDevice.init({ fromExportedDevice: exportedAgain });
@@ -148,7 +149,7 @@ describe("OlmDevice", function() {
                         setTimeout(reject, 500);
                     });
                 },
-            };
+            } as unknown as MockedObject<MatrixClient>;
             const devicesByUser = {
                 "@bob:example.com": [
                     DeviceInfo.fromStorage({
@@ -205,7 +206,7 @@ describe("OlmDevice", function() {
                         setTimeout(reject, 500);
                     });
                 },
-            };
+            } as unknown as MockedObject<MatrixClient>;
 
             const deviceBobA = DeviceInfo.fromStorage({
                 keys: {

spec/unit/crypto/verification/sas.spec.js

@@ -218,7 +218,7 @@ describe("SAS verification", function() {
             ]);
 
             // make sure that it uses the preferred method
-            expect(macMethod).toBe("hkdf-hmac-sha256");
+            expect(macMethod).toBe("org.matrix.msc3783.hkdf-hmac-sha256");
             expect(keyAgreement).toBe("curve25519-hkdf-sha256");
 
             // make sure Alice and Bob verified each other
@@ -230,6 +230,62 @@ describe("SAS verification", function() {
             expect(aliceDevice.isVerified()).toBeTruthy();
         });
 
+        it("should be able to verify using the old base64", async () => {
+            // pretend that Alice can only understand the old (incorrect) base64
+            // encoding, and make sure that she can still verify with Bob
+            let macMethod;
+            const aliceOrigSendToDevice = alice.client.sendToDevice.bind(alice.client);
+            alice.client.sendToDevice = (type, map) => {
+                if (type === "m.key.verification.start") {
+                    // Note: this modifies not only the message that Bob
+                    // receives, but also the copy of the message that Alice
+                    // has, since it is the same object.  If this does not
+                    // happen, the verification will fail due to a hash
+                    // commitment mismatch.
+                    map[bob.client.getUserId()][bob.client.deviceId]
+                        .message_authentication_codes = ['hkdf-hmac-sha256'];
+                }
+                return aliceOrigSendToDevice(type, map);
+            };
+            const bobOrigSendToDevice = bob.client.sendToDevice.bind(bob.client);
+            bob.client.sendToDevice = (type, map) => {
+                if (type === "m.key.verification.accept") {
+                    macMethod = map[alice.client.getUserId()][alice.client.deviceId]
+                        .message_authentication_code;
+                }
+                return bobOrigSendToDevice(type, map);
+            };
+
+            alice.httpBackend.when('POST', '/keys/query').respond(200, {
+                failures: {},
+                device_keys: {
+                    "@bob:example.com": BOB_DEVICES,
+                },
+            });
+            bob.httpBackend.when('POST', '/keys/query').respond(200, {
+                failures: {},
+                device_keys: {
+                    "@alice:example.com": ALICE_DEVICES,
+                },
+            });
+
+            await Promise.all([
+                aliceVerifier.verify(),
+                bobPromise.then((verifier) => verifier.verify()),
+                alice.httpBackend.flush(),
+                bob.httpBackend.flush(),
+            ]);
+
+            expect(macMethod).toBe("hkdf-hmac-sha256");
+
+            const bobDevice
+                  = await alice.client.getStoredDevice("@bob:example.com", "Dynabook");
+            expect(bobDevice.isVerified()).toBeTruthy();
+            const aliceDevice
+                  = await bob.client.getStoredDevice("@alice:example.com", "Osborne2");
+            expect(aliceDevice.isVerified()).toBeTruthy();
+        });
+
         it("should be able to verify using the old MAC", async () => {
             // pretend that Alice can only understand the old (incorrect) MAC,
             // and make sure that she can still verify with Bob

spec/unit/event-timeline.spec.ts

@@ -1,26 +1,36 @@
+import { mocked } from 'jest-mock';
+
 import * as utils from "../test-utils/test-utils";
 import { EventTimeline } from "../../src/models/event-timeline";
 import { RoomState } from "../../src/models/room-state";
+import { MatrixClient } from "../../src/matrix";
+import { Room } from "../../src/models/room";
+import { RoomMember } from "../../src/models/room-member";
+import { EventTimelineSet } from "../../src/models/event-timeline-set";
 
-function mockRoomStates(timeline) {
-    timeline.startState = utils.mock(RoomState, "startState");
-    timeline.endState = utils.mock(RoomState, "endState");
-}
+jest.mock("../../src/models/room-state");
 
 describe("EventTimeline", function() {
     const roomId = "!foo:bar";
     const userA = "@alice:bar";
     const userB = "@bertha:bar";
-    let timeline;
+    let timeline: EventTimeline;
+
+    const mockClient = {} as unknown as MatrixClient;
+
+    const getTimeline = (): EventTimeline => {
+        const room = new Room(roomId, mockClient, userA);
+        const timelineSet = new EventTimelineSet(room);
+        jest.spyOn(timelineSet.room, 'getUnfilteredTimelineSet').mockReturnValue(timelineSet);
+
+        return new EventTimeline(timelineSet);
+    };
 
     beforeEach(function() {
-        // XXX: this is a horrid hack; should use sinon or something instead to mock
-        const timelineSet = { room: { roomId: roomId } };
-        timelineSet.room.getUnfilteredTimelineSet = function() {
-            return timelineSet;
-        };
+        // reset any RoomState mocks
+        jest.resetAllMocks();
 
-        timeline = new EventTimeline(timelineSet);
+        timeline = getTimeline();
     });
 
     describe("construction", function() {
@@ -31,10 +41,6 @@ describe("EventTimeline", function() {
     });
 
     describe("initialiseState", function() {
-        beforeEach(function() {
-            mockRoomStates(timeline);
-        });
-
         it("should copy state events to start and end state", function() {
             const events = [
                 utils.mkMembership({
@@ -48,11 +54,15 @@ describe("EventTimeline", function() {
                 }),
             ];
             timeline.initialiseState(events);
-            expect(timeline.startState.setStateEvents).toHaveBeenCalledWith(
+            // @ts-ignore private prop
+            const timelineStartState = timeline.startState;
+            expect(mocked(timelineStartState).setStateEvents).toHaveBeenCalledWith(
                 events,
                 { timelineWasEmpty: undefined },
             );
-            expect(timeline.endState.setStateEvents).toHaveBeenCalledWith(
+            // @ts-ignore private prop
+            const timelineEndState = timeline.endState;
+            expect(mocked(timelineEndState).setStateEvents).toHaveBeenCalledWith(
                 events,
                 { timelineWasEmpty: undefined },
             );
@@ -103,8 +113,8 @@ describe("EventTimeline", function() {
         });
 
         it("setNeighbouringTimeline should set neighbour", function() {
-            const prev = { a: "a" };
-            const next = { b: "b" };
+            const prev = getTimeline();
+            const next = getTimeline();
             timeline.setNeighbouringTimeline(prev, EventTimeline.BACKWARDS);
             timeline.setNeighbouringTimeline(next, EventTimeline.FORWARDS);
             expect(timeline.getNeighbouringTimeline(EventTimeline.BACKWARDS)).toBe(prev);
@@ -112,8 +122,8 @@ describe("EventTimeline", function() {
         });
 
         it("setNeighbouringTimeline should throw if called twice", function() {
-            const prev = { a: "a" };
-            const next = { b: "b" };
+            const prev = getTimeline();
+            const next = getTimeline();
             expect(function() {
                 timeline.setNeighbouringTimeline(prev, EventTimeline.BACKWARDS);
             }).not.toThrow();
@@ -135,10 +145,6 @@ describe("EventTimeline", function() {
     });
 
     describe("addEvent", function() {
-        beforeEach(function() {
-            mockRoomStates(timeline);
-        });
-
         const events = [
             utils.mkMessage({
                 room: roomId, user: userA, msg: "hungry hungry hungry",
@@ -171,24 +177,22 @@ describe("EventTimeline", function() {
         });
 
         it("should set event.sender for new and old events", function() {
-            const sentinel = {
-                userId: userA,
-                membership: "join",
-                name: "Alice",
-            };
-            const oldSentinel = {
-                userId: userA,
-                membership: "join",
-                name: "Old Alice",
-            };
-            timeline.getState(EventTimeline.FORWARDS).getSentinelMember
+            const sentinel = new RoomMember(roomId, userA);
+            sentinel.name = "Alice";
+            sentinel.membership = "join";
+
+            const oldSentinel = new RoomMember(roomId, userA);
+            sentinel.name = "Old Alice";
+            sentinel.membership = "join";
+
+            mocked(timeline.getState(EventTimeline.FORWARDS)).getSentinelMember
                 .mockImplementation(function(uid) {
                     if (uid === userA) {
                         return sentinel;
                     }
                     return null;
                 });
-            timeline.getState(EventTimeline.BACKWARDS).getSentinelMember
+            mocked(timeline.getState(EventTimeline.BACKWARDS)).getSentinelMember
                 .mockImplementation(function(uid) {
                     if (uid === userA) {
                         return oldSentinel;
@@ -212,43 +216,41 @@ describe("EventTimeline", function() {
         });
 
         it("should set event.target for new and old m.room.member events",
-        function() {
-            const sentinel = {
-                userId: userA,
-                membership: "join",
-                name: "Alice",
-            };
-            const oldSentinel = {
-                userId: userA,
-                membership: "join",
-                name: "Old Alice",
-            };
-            timeline.getState(EventTimeline.FORWARDS).getSentinelMember
-                .mockImplementation(function(uid) {
-                    if (uid === userA) {
-                        return sentinel;
-                    }
-                    return null;
-                });
-            timeline.getState(EventTimeline.BACKWARDS).getSentinelMember
-                .mockImplementation(function(uid) {
-                    if (uid === userA) {
-                        return oldSentinel;
-                    }
-                    return null;
-                });
+            function() {
+                const sentinel = new RoomMember(roomId, userA);
+                sentinel.name = "Alice";
+                sentinel.membership = "join";
 
-            const newEv = utils.mkMembership({
-                room: roomId, mship: "invite", user: userB, skey: userA, event: true,
+                const oldSentinel = new RoomMember(roomId, userA);
+                sentinel.name = "Old Alice";
+                sentinel.membership = "join";
+
+                mocked(timeline.getState(EventTimeline.FORWARDS)).getSentinelMember
+                    .mockImplementation(function(uid) {
+                        if (uid === userA) {
+                            return sentinel;
+                        }
+                        return null;
+                    });
+                mocked(timeline.getState(EventTimeline.BACKWARDS)).getSentinelMember
+                    .mockImplementation(function(uid) {
+                        if (uid === userA) {
+                            return oldSentinel;
+                        }
+                        return null;
+                    });
+
+                const newEv = utils.mkMembership({
+                    room: roomId, mship: "invite", user: userB, skey: userA, event: true,
+                });
+                const oldEv = utils.mkMembership({
+                    room: roomId, mship: "ban", user: userB, skey: userA, event: true,
+                });
+                timeline.addEvent(newEv, { toStartOfTimeline: false });
+                expect(newEv.target).toEqual(sentinel);
+                timeline.addEvent(oldEv, { toStartOfTimeline: true });
+                expect(oldEv.target).toEqual(oldSentinel);
             });
-            const oldEv = utils.mkMembership({
-                room: roomId, mship: "ban", user: userB, skey: userA, event: true,
-            });
-            timeline.addEvent(newEv, { toStartOfTimeline: false });
-            expect(newEv.target).toEqual(sentinel);
-            timeline.addEvent(oldEv, { toStartOfTimeline: true });
-            expect(oldEv.target).toEqual(oldSentinel);
-        });
 
         it("should call setStateEvents on the right RoomState with the right " +
            "forwardLooking value for new events", function() {
@@ -310,7 +312,11 @@ describe("EventTimeline", function() {
 
         it("Make sure legacy overload passing options directly as parameters still works", () => {
             expect(() => timeline.addEvent(events[0], { toStartOfTimeline: true })).not.toThrow();
-            expect(() => timeline.addEvent(events[0], { stateContext: new RoomState() })).not.toThrow();
+            // @ts-ignore stateContext is not a valid param
+            expect(() => timeline.addEvent(events[0], { stateContext: new RoomState(roomId) })).not.toThrow();
+            expect(() => timeline.addEvent(events[0],
+                { toStartOfTimeline: false, roomState: new RoomState(roomId) },
+            )).not.toThrow();
         });
     });
 
@@ -364,14 +370,14 @@ describe("EventTimeline", function() {
         // - removing the last event got baseIndex into such a state that
         // further addEvent(ev, false) calls made the index increase.
         it("should not make baseIndex assplode when removing the last event",
-           function() {
-               timeline.addEvent(events[0], { toStartOfTimeline: true });
-               timeline.removeEvent(events[0].getId());
-               const initialIndex = timeline.getBaseIndex();
-               timeline.addEvent(events[1], { toStartOfTimeline: false });
-               timeline.addEvent(events[2], { toStartOfTimeline: false });
-               expect(timeline.getBaseIndex()).toEqual(initialIndex);
-               expect(timeline.getEvents().length).toEqual(2);
-           });
+            function() {
+                timeline.addEvent(events[0], { toStartOfTimeline: true });
+                timeline.removeEvent(events[0].getId());
+                const initialIndex = timeline.getBaseIndex();
+                timeline.addEvent(events[1], { toStartOfTimeline: false });
+                timeline.addEvent(events[2], { toStartOfTimeline: false });
+                expect(timeline.getBaseIndex()).toEqual(initialIndex);
+                expect(timeline.getEvents().length).toEqual(2);
+            });
     });
 });

spec/unit/event.spec.ts

@@ -1,6 +1,6 @@
 /*
 Copyright 2017 New Vector Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2022 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,16 +15,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { logger } from "../../src/logger";
 import { MatrixEvent } from "../../src/models/event";
 
 describe("MatrixEvent", () => {
     describe(".attemptDecryption", () => {
         let encryptedEvent;
+        const eventId = 'test_encrypted_event';
 
         beforeEach(() => {
             encryptedEvent = new MatrixEvent({
-                id: 'test_encrypted_event',
+                event_id: eventId,
                 type: 'm.room.encrypted',
                 content: {
                     ciphertext: 'secrets',
@@ -32,45 +32,34 @@ describe("MatrixEvent", () => {
             });
         });
 
-        it('should retry decryption if a retry is queued', () => {
-            let callCount = 0;
-
-            let prom2;
-            let prom2Fulfilled = false;
+        it('should retry decryption if a retry is queued', async () => {
+            const eventAttemptDecryptionSpy = jest.spyOn(encryptedEvent, 'attemptDecryption');
 
             const crypto = {
-                decryptEvent: function() {
-                    ++callCount;
-                    logger.log(`decrypt: ${callCount}`);
-                    if (callCount == 1) {
+                decryptEvent: jest.fn()
+                    .mockImplementationOnce(() => {
                         // schedule a second decryption attempt while
                         // the first one is still running.
-                        prom2 = encryptedEvent.attemptDecryption(crypto);
-                        prom2.then(() => prom2Fulfilled = true);
+                        encryptedEvent.attemptDecryption(crypto);
 
                         const error = new Error("nope");
                         error.name = 'DecryptionError';
                         return Promise.reject(error);
-                    } else {
-                        expect(prom2Fulfilled).toBe(
-                            false, 'second attemptDecryption resolved too soon');
-
+                    })
+                    .mockImplementationOnce(() => {
                         return Promise.resolve({
                             clearEvent: {
                                 type: 'm.room.message',
                             },
                         });
-                    }
-                },
+                    }),
             };
 
-            return encryptedEvent.attemptDecryption(crypto).then(() => {
-                expect(callCount).toEqual(2);
-                expect(encryptedEvent.getType()).toEqual('m.room.message');
+            await encryptedEvent.attemptDecryption(crypto);
 
-                // make sure the second attemptDecryption resolves
-                return prom2;
-            });
+            expect(eventAttemptDecryptionSpy).toHaveBeenCalledTimes(2);
+            expect(crypto.decryptEvent).toHaveBeenCalledTimes(2);
+            expect(encryptedEvent.getType()).toEqual('m.room.message');
         });
     });
 });

spec/unit/filter.spec.ts

@@ -1,9 +1,9 @@
-import { Filter } from "../../src/filter";
+import { Filter, IFilterDefinition } from "../../src/filter";
 
 describe("Filter", function() {
     const filterId = "f1lt3ring15g00d4ursoul";
     const userId = "@sir_arthur_david:humming.tiger";
-    let filter;
+    let filter: Filter;
 
     beforeEach(function() {
         filter = new Filter(userId);
@@ -37,7 +37,7 @@ describe("Filter", function() {
     describe("setDefinition/getDefinition", function() {
         it("should set and get the filter body", function() {
             const definition = {
-                event_format: "client",
+                event_format: "client" as IFilterDefinition['event_format'],
             };
             filter.setDefinition(definition);
             expect(filter.getDefinition()).toEqual(definition);

spec/unit/interactive-auth.spec.ts

@@ -15,8 +15,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { MatrixClient } from "../../src/client";
 import { logger } from "../../src/logger";
-import { InteractiveAuth } from "../../src/interactive-auth";
+import { InteractiveAuth, AuthType } from "../../src/interactive-auth";
 import { MatrixError } from "../../src/http-api";
 import { sleep } from "../../src/utils";
 import { randomString } from "../../src/randomstring";
@@ -29,38 +30,40 @@ class FakeClient {
     }
 }
 
+const getFakeClient = (): MatrixClient => new FakeClient() as unknown as MatrixClient;
+
 describe("InteractiveAuth", function() {
     it("should start an auth stage and complete it", function() {
         const doRequest = jest.fn();
         const stateUpdated = jest.fn();
 
         const ia = new InteractiveAuth({
-            matrixClient: new FakeClient(),
+            matrixClient: getFakeClient(),
             doRequest: doRequest,
             stateUpdated: stateUpdated,
+            requestEmailToken: jest.fn(),
             authData: {
                 session: "sessionId",
                 flows: [
-                    { stages: ["logintype"] },
+                    { stages: [AuthType.Password] },
                 ],
                 params: {
-                    "logintype": { param: "aa" },
+                    [AuthType.Password]: { param: "aa" },
                 },
             },
         });
 
         expect(ia.getSessionId()).toEqual("sessionId");
-        expect(ia.getStageParams("logintype")).toEqual({
+        expect(ia.getStageParams(AuthType.Password)).toEqual({
             param: "aa",
         });
 
         // first we expect a call here
         stateUpdated.mockImplementation(function(stage) {
             logger.log('aaaa');
-            expect(stage).toEqual("logintype");
+            expect(stage).toEqual(AuthType.Password);
             ia.submitAuthDict({
-                type: "logintype",
-                foo: "bar",
+                type: AuthType.Password,
             });
         });
 
@@ -70,8 +73,7 @@ describe("InteractiveAuth", function() {
             logger.log('cccc');
             expect(authData).toEqual({
                 session: "sessionId",
-                type: "logintype",
-                foo: "bar",
+                type: AuthType.Password,
             });
             return Promise.resolve(requestRes);
         });
@@ -86,15 +88,17 @@ describe("InteractiveAuth", function() {
     it("should make a request if no authdata is provided", function() {
         const doRequest = jest.fn();
         const stateUpdated = jest.fn();
+        const requestEmailToken = jest.fn();
 
         const ia = new InteractiveAuth({
-            matrixClient: new FakeClient(),
-            stateUpdated: stateUpdated,
-            doRequest: doRequest,
+            matrixClient: getFakeClient(),
+            stateUpdated,
+            doRequest,
+            requestEmailToken,
         });
 
         expect(ia.getSessionId()).toBe(undefined);
-        expect(ia.getStageParams("logintype")).toBe(undefined);
+        expect(ia.getStageParams(AuthType.Password)).toBe(undefined);
 
         // first we expect a call to doRequest
         doRequest.mockImplementation(function(authData) {
@@ -103,10 +107,10 @@ describe("InteractiveAuth", function() {
             const err = new MatrixError({
                 session: "sessionId",
                 flows: [
-                    { stages: ["logintype"] },
+                    { stages: [AuthType.Password] },
                 ],
                 params: {
-                    "logintype": { param: "aa" },
+                    [AuthType.Password]: { param: "aa" },
                 },
             });
             err.httpStatus = 401;
@@ -116,9 +120,9 @@ describe("InteractiveAuth", function() {
         // .. which should be followed by a call to stateUpdated
         const requestRes = { "a": "b" };
         stateUpdated.mockImplementation(function(stage) {
-            expect(stage).toEqual("logintype");
+            expect(stage).toEqual(AuthType.Password);
             expect(ia.getSessionId()).toEqual("sessionId");
-            expect(ia.getStageParams("logintype")).toEqual({
+            expect(ia.getStageParams(AuthType.Password)).toEqual({
                 param: "aa",
             });
 
@@ -127,15 +131,13 @@ describe("InteractiveAuth", function() {
                 logger.log("request2", authData);
                 expect(authData).toEqual({
                     session: "sessionId",
-                    type: "logintype",
-                    foo: "bar",
+                    type: AuthType.Password,
                 });
                 return Promise.resolve(requestRes);
             });
 
             ia.submitAuthDict({
-                type: "logintype",
-                foo: "bar",
+                type: AuthType.Password,
             });
         });
 
@@ -149,11 +151,13 @@ describe("InteractiveAuth", function() {
     it("should start an auth stage and reject if no auth flow", function() {
         const doRequest = jest.fn();
         const stateUpdated = jest.fn();
+        const requestEmailToken = jest.fn();
 
         const ia = new InteractiveAuth({
-            matrixClient: new FakeClient(),
-            doRequest: doRequest,
-            stateUpdated: stateUpdated,
+            matrixClient: getFakeClient(),
+            doRequest,
+            stateUpdated,
+            requestEmailToken,
         });
 
         doRequest.mockImplementation(function(authData) {
@@ -163,7 +167,7 @@ describe("InteractiveAuth", function() {
                 session: "sessionId",
                 flows: [],
                 params: {
-                    "logintype": { param: "aa" },
+                    [AuthType.Password]: { param: "aa" },
                 },
             });
             err.httpStatus = 401;
@@ -183,7 +187,7 @@ describe("InteractiveAuth", function() {
             requestEmailToken.mockImplementation(async () => ({ sid: "" }));
 
             const ia = new InteractiveAuth({
-                matrixClient: new FakeClient(),
+                matrixClient: getFakeClient(),
                 doRequest, stateUpdated, requestEmailToken,
             });
 
@@ -210,7 +214,7 @@ describe("InteractiveAuth", function() {
             requestEmailToken.mockImplementation(async () => ({ sid: "" }));
 
             const ia = new InteractiveAuth({
-                matrixClient: new FakeClient(),
+                matrixClient: getFakeClient(),
                 doRequest, stateUpdated, requestEmailToken,
             });
 
@@ -239,7 +243,7 @@ describe("InteractiveAuth", function() {
             });
 
             const ia = new InteractiveAuth({
-                matrixClient: new FakeClient(),
+                matrixClient: getFakeClient(),
                 doRequest, stateUpdated, requestEmailToken,
             });
 
@@ -253,7 +257,7 @@ describe("InteractiveAuth", function() {
             requestEmailToken.mockImplementation(() => sleep(500, { sid: "" }));
 
             const ia = new InteractiveAuth({
-                matrixClient: new FakeClient(),
+                matrixClient: getFakeClient(),
                 doRequest, stateUpdated, requestEmailToken,
             });
 
@@ -269,7 +273,7 @@ describe("InteractiveAuth", function() {
             requestEmailToken.mockImplementation(() => sleep(500, { sid }));
 
             const ia = new InteractiveAuth({
-                matrixClient: new FakeClient(),
+                matrixClient: getFakeClient(),
                 doRequest, stateUpdated, requestEmailToken,
             });
 

spec/unit/login.spec.ts

@@ -1,7 +1,7 @@
 import { TestClient } from '../TestClient';
 
 describe('Login request', function() {
-    let client;
+    let client: TestClient;
 
     beforeEach(function() {
         client = new TestClient();

spec/unit/matrix-client.spec.ts

@@ -14,8 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { mocked } from "jest-mock";
+
 import { logger } from "../../src/logger";
-import { MatrixClient } from "../../src/client";
+import { MatrixClient, ClientEvent } from "../../src/client";
 import { Filter } from "../../src/filter";
 import { DEFAULT_TREE_POWER_LEVELS_TEMPLATE } from "../../src/models/MSC3089TreeSpace";
 import {
@@ -27,6 +29,7 @@ import {
     UNSTABLE_MSC3089_TREE_SUBTYPE,
 } from "../../src/@types/event";
 import { MEGOLM_ALGORITHM } from "../../src/crypto/olmlib";
+import { Crypto } from "../../src/crypto";
 import { EventStatus, MatrixEvent } from "../../src/models/event";
 import { Preset } from "../../src/@types/partials";
 import { ReceiptType } from "../../src/@types/read_receipts";
@@ -34,10 +37,16 @@ import * as testUtils from "../test-utils/test-utils";
 import { makeBeaconInfoContent } from "../../src/content-helpers";
 import { M_BEACON_INFO } from "../../src/@types/beacon";
 import { ContentHelpers, Room } from "../../src";
+import { supportsMatrixCall } from "../../src/webrtc/call";
 import { makeBeaconEvent } from "../test-utils/beacon";
 
 jest.useFakeTimers();
 
+jest.mock("../../src/webrtc/call", () => ({
+    ...jest.requireActual("../../src/webrtc/call"),
+    supportsMatrixCall: jest.fn(() => false),
+}));
+
 describe("MatrixClient", function() {
     const userId = "@alice:bar";
     const identityServerUrl = "https://identity.server";
@@ -159,6 +168,24 @@ describe("MatrixClient", function() {
         return new Promise(() => {});
     }
 
+    function makeClient() {
+        client = new MatrixClient({
+            baseUrl: "https://my.home.server",
+            idBaseUrl: identityServerUrl,
+            accessToken: "my.access.token",
+            request: function() {} as any, // NOP
+            store: store,
+            scheduler: scheduler,
+            userId: userId,
+        });
+        // FIXME: We shouldn't be yanking http like this.
+        client.http = [
+            "authedRequest", "getContentUri", "request", "uploadContent",
+        ].reduce((r, k) => { r[k] = jest.fn(); return r; }, {});
+        client.http.authedRequest.mockImplementation(httpReq);
+        client.http.request.mockImplementation(httpReq);
+    }
+
     beforeEach(function() {
         scheduler = [
             "getQueueForEvent", "queueEvent", "removeEventFromQueue",
@@ -176,21 +203,7 @@ describe("MatrixClient", function() {
         store.getClientOptions = jest.fn().mockReturnValue(Promise.resolve(null));
         store.storeClientOptions = jest.fn().mockReturnValue(Promise.resolve(null));
         store.isNewlyCreated = jest.fn().mockReturnValue(Promise.resolve(true));
-        client = new MatrixClient({
-            baseUrl: "https://my.home.server",
-            idBaseUrl: identityServerUrl,
-            accessToken: "my.access.token",
-            request: function() {} as any, // NOP
-            store: store,
-            scheduler: scheduler,
-            userId: userId,
-        });
-        // FIXME: We shouldn't be yanking http like this.
-        client.http = [
-            "authedRequest", "getContentUri", "request", "uploadContent",
-        ].reduce((r, k) => { r[k] = jest.fn(); return r; }, {});
-        client.http.authedRequest.mockImplementation(httpReq);
-        client.http.request.mockImplementation(httpReq);
+        makeClient();
 
         // set reasonable working defaults
         acceptKeepalives = true;
@@ -1297,4 +1310,106 @@ describe("MatrixClient", function() {
             expect(result!.aliases).toEqual(response.aliases);
         });
     });
+
+    describe("pollingTurnServers", () => {
+        afterEach(() => {
+            mocked(supportsMatrixCall).mockReset();
+        });
+
+        it("is false if the client isn't started", () => {
+            expect(client.clientRunning).toBe(false);
+            expect(client.pollingTurnServers).toBe(false);
+        });
+
+        it("is false if VoIP is not supported", async () => {
+            mocked(supportsMatrixCall).mockReturnValue(false);
+            makeClient(); // create the client a second time so it picks up the supportsMatrixCall mock
+            await client.startClient();
+            expect(client.pollingTurnServers).toBe(false);
+        });
+
+        it("is true if VoIP is supported", async () => {
+            mocked(supportsMatrixCall).mockReturnValue(true);
+            makeClient(); // create the client a second time so it picks up the supportsMatrixCall mock
+            await client.startClient();
+            expect(client.pollingTurnServers).toBe(true);
+        });
+    });
+
+    describe("checkTurnServers", () => {
+        beforeAll(() => {
+            mocked(supportsMatrixCall).mockReturnValue(true);
+        });
+
+        beforeEach(() => {
+            makeClient(); // create the client a second time so it picks up the supportsMatrixCall mock
+        });
+
+        afterAll(() => {
+            mocked(supportsMatrixCall).mockReset();
+        });
+
+        it("emits an event when new TURN creds are found", async () => {
+            const turnServer = {
+                uris: [
+                    "turn:turn.example.com:3478?transport=udp",
+                    "turn:10.20.30.40:3478?transport=tcp",
+                    "turns:10.20.30.40:443?transport=tcp",
+                ],
+                username: "1443779631:@user:example.com",
+                password: "JlKfBy1QwLrO20385QyAtEyIv0=",
+            };
+            jest.spyOn(client, "turnServer").mockResolvedValue(turnServer);
+
+            const events: any[][] = [];
+            const onTurnServers = (...args) => events.push(args);
+            client.on(ClientEvent.TurnServers, onTurnServers);
+            expect(await client.checkTurnServers()).toBe(true);
+            client.off(ClientEvent.TurnServers, onTurnServers);
+            expect(events).toEqual([[[{
+                urls: turnServer.uris,
+                username: turnServer.username,
+                credential: turnServer.password,
+            }]]]);
+        });
+
+        it("emits an event when an error occurs", async () => {
+            const error = new Error(":(");
+            jest.spyOn(client, "turnServer").mockRejectedValue(error);
+
+            const events: any[][] = [];
+            const onTurnServersError = (...args) => events.push(args);
+            client.on(ClientEvent.TurnServersError, onTurnServersError);
+            expect(await client.checkTurnServers()).toBe(false);
+            client.off(ClientEvent.TurnServersError, onTurnServersError);
+            expect(events).toEqual([[error, false]]); // non-fatal
+        });
+
+        it("considers 403 errors fatal", async () => {
+            const error = { httpStatus: 403 };
+            jest.spyOn(client, "turnServer").mockRejectedValue(error);
+
+            const events: any[][] = [];
+            const onTurnServersError = (...args) => events.push(args);
+            client.on(ClientEvent.TurnServersError, onTurnServersError);
+            expect(await client.checkTurnServers()).toBe(false);
+            client.off(ClientEvent.TurnServersError, onTurnServersError);
+            expect(events).toEqual([[error, true]]); // fatal
+        });
+    });
+
+    describe("encryptAndSendToDevices", () => {
+        it("throws an error if crypto is unavailable", () => {
+            client.crypto = undefined;
+            expect(() => client.encryptAndSendToDevices([], {})).toThrow();
+        });
+
+        it("is an alias for the crypto method", async () => {
+            client.crypto = testUtils.mock(Crypto, "Crypto");
+            const deviceInfos = [];
+            const payload = {};
+            await client.encryptAndSendToDevices(deviceInfos, payload);
+            expect(client.crypto.encryptAndSendToDevices).toHaveBeenLastCalledWith(deviceInfos, payload);
+        });
+    });
 });

spec/unit/pushprocessor.spec.ts

@@ -1,15 +1,15 @@
 import * as utils from "../test-utils/test-utils";
 import { PushProcessor } from "../../src/pushprocessor";
-import { EventType } from "../../src";
+import { EventType, MatrixClient, MatrixEvent } from "../../src";
 
 describe('NotificationService', function() {
     const testUserId = "@ali:matrix.org";
     const testDisplayName = "Alice M";
     const testRoomId = "!fl1bb13:localhost";
 
-    let testEvent;
+    let testEvent: MatrixEvent;
 
-    let pushProcessor;
+    let pushProcessor: PushProcessor;
 
     // These would be better if individual rules were configured in the tests themselves.
     const matrixClient = {
@@ -196,7 +196,7 @@ describe('NotificationService', function() {
                 ],
             },
         },
-    };
+    } as unknown as MatrixClient;
 
     beforeEach(function() {
         testEvent = utils.mkEvent({

spec/unit/queueToDevice.spec.ts

@@ -0,0 +1,349 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import MockHttpBackend from 'matrix-mock-request';
+import { indexedDB as fakeIndexedDB } from 'fake-indexeddb';
+
+import { IHttpOpts, IndexedDBStore, MatrixEvent, MemoryStore, Room } from "../../src";
+import { MatrixClient } from "../../src/client";
+import { ToDeviceBatch } from '../../src/models/ToDeviceMessage';
+import { logger } from '../../src/logger';
+import { IStore } from '../../src/store';
+
+const FAKE_USER = "@alice:example.org";
+const FAKE_DEVICE_ID = "AAAAAAAA";
+const FAKE_PAYLOAD = {
+    "foo": 42,
+};
+const EXPECTED_BODY = {
+    messages: {
+        [FAKE_USER]: {
+            [FAKE_DEVICE_ID]: FAKE_PAYLOAD,
+        },
+    },
+};
+
+const FAKE_MSG = {
+    userId: FAKE_USER,
+    deviceId: FAKE_DEVICE_ID,
+    payload: FAKE_PAYLOAD,
+};
+
+enum StoreType {
+    Memory = 'Memory',
+    IndexedDB = 'IndexedDB',
+}
+
+// Jest now uses @sinonjs/fake-timers which exposes tickAsync() and a number of
+// other async methods which break the event loop, letting scheduled promise
+// callbacks run. Unfortunately, Jest doesn't expose these, so we have to do
+// it manually (this is what sinon does under the hood). We do both in a loop
+// until the thing we expect happens: hopefully this is the least flakey way
+// and avoids assuming anything about the app's behaviour.
+const realSetTimeout = setTimeout;
+function flushPromises() {
+    return new Promise(r => {
+        realSetTimeout(r, 1);
+    });
+}
+
+async function flushAndRunTimersUntil(cond: () => boolean) {
+    while (!cond()) {
+        await flushPromises();
+        if (cond()) break;
+        jest.advanceTimersToNextTimer();
+    }
+}
+
+describe.each([
+    [StoreType.Memory], [StoreType.IndexedDB],
+])("queueToDevice (%s store)", function(storeType) {
+    let httpBackend: MockHttpBackend;
+    let client: MatrixClient;
+
+    beforeEach(async function() {
+        httpBackend = new MockHttpBackend();
+
+        let store: IStore;
+        if (storeType === StoreType.IndexedDB) {
+            const idbStore = new IndexedDBStore({ indexedDB: fakeIndexedDB });
+            await idbStore.startup();
+            store = idbStore;
+        } else {
+            store = new MemoryStore();
+        }
+
+        client = new MatrixClient({
+            baseUrl: "https://my.home.server",
+            accessToken: "my.access.token",
+            request: httpBackend.requestFn as IHttpOpts["request"],
+            store,
+        });
+    });
+
+    afterEach(function() {
+        jest.useRealTimers();
+        client.stopClient();
+    });
+
+    it("sends a to-device message", async function() {
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).check((request) => {
+            expect(request.data).toEqual(EXPECTED_BODY);
+        }).respond(200, {});
+
+        await client.queueToDevice({
+            eventType: "org.example.foo",
+            batch: [
+                FAKE_MSG,
+            ],
+        });
+
+        await httpBackend.flushAllExpected();
+        // let the code handle the response to the request so we don't get
+        // log output after the test has finished (apparently stopping the
+        // client in aftereach is not sufficient.)
+        await flushPromises();
+    });
+
+    it("retries on error", async function() {
+        jest.useFakeTimers();
+
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).respond(500);
+
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).check((request) => {
+            expect(request.data).toEqual(EXPECTED_BODY);
+        }).respond(200, {});
+
+        await client.queueToDevice({
+            eventType: "org.example.foo",
+            batch: [
+                FAKE_MSG,
+            ],
+        });
+        await flushAndRunTimersUntil(() => httpBackend.requests.length > 0);
+        expect(httpBackend.flushSync(null, 1)).toEqual(1);
+
+        await flushAndRunTimersUntil(() => httpBackend.requests.length > 0);
+
+        expect(httpBackend.flushSync(null, 1)).toEqual(1);
+
+        // flush, as per comment in first test
+        await flushPromises();
+    });
+
+    it("stops retrying on 4xx errors", async function() {
+        jest.useFakeTimers();
+
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).respond(400);
+
+        await client.queueToDevice({
+            eventType: "org.example.foo",
+            batch: [
+                FAKE_MSG,
+            ],
+        });
+        await flushAndRunTimersUntil(() => httpBackend.requests.length > 0);
+        expect(httpBackend.flushSync(null, 1)).toEqual(1);
+
+        // Asserting that another request is never made is obviously
+        // a bit tricky - we just flush the queue what should hopefully
+        // be plenty of times and assert that nothing comes through.
+        let tries = 0;
+        await flushAndRunTimersUntil(() => ++tries === 10);
+
+        expect(httpBackend.requests.length).toEqual(0);
+    });
+
+    it("honours ratelimiting", async function() {
+        jest.useFakeTimers();
+
+        // pick something obscure enough it's unlikley to clash with a
+        // retry delay the algorithm uses anyway
+        const retryDelay = 279 * 1000;
+
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).respond(429, {
+            errcode: "M_LIMIT_EXCEEDED",
+            retry_after_ms: retryDelay,
+        });
+
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).respond(200, {});
+
+        await client.queueToDevice({
+            eventType: "org.example.foo",
+            batch: [
+                FAKE_MSG,
+            ],
+        });
+        await flushAndRunTimersUntil(() => httpBackend.requests.length > 0);
+        expect(httpBackend.flushSync(null, 1)).toEqual(1);
+        await flushPromises();
+
+        logger.info("Advancing clock to just before expected retry time...");
+
+        jest.advanceTimersByTime(retryDelay - 1000);
+        await flushPromises();
+
+        expect(httpBackend.requests.length).toEqual(0);
+
+        logger.info("Advancing clock past expected retry time...");
+
+        jest.advanceTimersByTime(2000);
+        await flushPromises();
+
+        expect(httpBackend.flushSync(null, 1)).toEqual(1);
+    });
+
+    it("retries on retryImmediately()", async function() {
+        httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
+            versions: ["r0.0.1"],
+        });
+
+        await Promise.all([client.startClient(), httpBackend.flush(null, 1, 20)]);
+
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).respond(500);
+
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).respond(200, {});
+
+        await client.queueToDevice({
+            eventType: "org.example.foo",
+            batch: [
+                FAKE_MSG,
+            ],
+        });
+        expect(await httpBackend.flush(null, 1, 1)).toEqual(1);
+        await flushPromises();
+
+        client.retryImmediately();
+
+        // longer timeout here to try & avoid flakiness
+        expect(await httpBackend.flush(null, 1, 3000)).toEqual(1);
+    });
+
+    it("retries on when client is started", async function() {
+        httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
+            versions: ["r0.0.1"],
+        });
+
+        await Promise.all([client.startClient(), httpBackend.flush("/_matrix/client/versions", 1, 20)]);
+
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).respond(500);
+
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).respond(200, {});
+
+        await client.queueToDevice({
+            eventType: "org.example.foo",
+            batch: [
+                FAKE_MSG,
+            ],
+        });
+        expect(await httpBackend.flush(null, 1, 1)).toEqual(1);
+        await flushPromises();
+
+        client.stopClient();
+        await Promise.all([client.startClient(), httpBackend.flush("/_matrix/client/versions", 1, 20)]);
+
+        expect(await httpBackend.flush(null, 1, 20)).toEqual(1);
+    });
+
+    it("retries when a message is retried", async function() {
+        httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
+            versions: ["r0.0.1"],
+        });
+
+        await Promise.all([client.startClient(), httpBackend.flush(null, 1, 20)]);
+
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).respond(500);
+
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).respond(200, {});
+
+        await client.queueToDevice({
+            eventType: "org.example.foo",
+            batch: [
+                FAKE_MSG,
+            ],
+        });
+
+        expect(await httpBackend.flush(null, 1, 1)).toEqual(1);
+        await flushPromises();
+
+        const dummyEvent = new MatrixEvent({
+            event_id: "!fake:example.org",
+        });
+        const mockRoom = {
+            updatePendingEvent: jest.fn(),
+        } as unknown as Room;
+        client.resendEvent(dummyEvent, mockRoom);
+
+        expect(await httpBackend.flush(null, 1, 20)).toEqual(1);
+    });
+
+    it("splits many messages into multiple HTTP requests", async function() {
+        const batch: ToDeviceBatch = {
+            eventType: "org.example.foo",
+            batch: [],
+        };
+
+        for (let i = 0; i <= 20; ++i) {
+            batch.batch.push({
+                userId: `@user${i}:example.org`,
+                deviceId: FAKE_DEVICE_ID,
+                payload: FAKE_PAYLOAD,
+            });
+        }
+
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).check((request) => {
+            expect(Object.keys(request.data.messages).length).toEqual(20);
+        }).respond(200, {});
+
+        httpBackend.when(
+            "PUT", "/sendToDevice/org.example.foo/",
+        ).check((request) => {
+            expect(Object.keys(request.data.messages).length).toEqual(1);
+        }).respond(200, {});
+
+        await client.queueToDevice(batch);
+        await httpBackend.flushAllExpected();
+
+        // flush, as per comment in first test
+        await flushPromises();
+    });
+});

spec/unit/realtime-callbacks.spec.ts

@@ -1,7 +1,7 @@
 import * as callbacks from "../../src/realtime-callbacks";
 
 let wallTime = 1234567890;
-jest.useFakeTimers();
+jest.useFakeTimers().setSystemTime(wallTime);
 
 describe("realtime-callbacks", function() {
     function tick(millis) {
@@ -9,14 +9,6 @@ describe("realtime-callbacks", function() {
         jest.advanceTimersByTime(millis);
     }
 
-    beforeEach(function() {
-        callbacks.setNow(() => wallTime);
-    });
-
-    afterEach(function() {
-        callbacks.setNow();
-    });
-
     describe("setTimeout", function() {
         it("should call the callback after the timeout", function() {
             const callback = jest.fn();
@@ -29,7 +21,7 @@ describe("realtime-callbacks", function() {
 
         it("should default to a zero timeout", function() {
             const callback = jest.fn();
-            callbacks.setTimeout(callback);
+            callbacks.setTimeout(callback, 0);
 
             expect(callback).not.toHaveBeenCalled();
             tick(0);
@@ -46,11 +38,11 @@ describe("realtime-callbacks", function() {
         it("should set 'this' to the global object", function() {
             let passed = false;
             const callback = function() {
-                expect(this).toBe(global); // eslint-disable-line @babel/no-invalid-this
-                expect(this.console).toBeTruthy(); // eslint-disable-line @babel/no-invalid-this
+                expect(this).toBe(global); // eslint-disable-line @typescript-eslint/no-invalid-this
+                expect(this.console).toBeTruthy(); // eslint-disable-line @typescript-eslint/no-invalid-this
                 passed = true;
             };
-            callbacks.setTimeout(callback);
+            callbacks.setTimeout(callback, 0);
             tick(0);
             expect(passed).toBe(true);
         });
@@ -100,7 +92,7 @@ describe("realtime-callbacks", function() {
                 expect(callback2).not.toHaveBeenCalled();
             });
 
-            callbacks.setTimeout(callback1);
+            callbacks.setTimeout(callback1, 0);
             callbacks.setTimeout(callback2, -100);
 
             expect(callback1).not.toHaveBeenCalled();
@@ -117,14 +109,14 @@ describe("realtime-callbacks", function() {
                 expect(callback2).not.toHaveBeenCalled();
             });
 
-            callbacks.setTimeout(callback1);
+            callbacks.setTimeout(callback1, 1);
             expect(callback1).not.toHaveBeenCalled();
             expect(callback2).not.toHaveBeenCalled();
-            tick(0);
+            tick(1);
             expect(callback1).toHaveBeenCalled();
             // the fake timer won't actually run callbacks registered during
             // one tick until the next tick.
-            tick(1);
+            tick(2);
             expect(callback2).toHaveBeenCalled();
         });
 
@@ -147,9 +139,9 @@ describe("realtime-callbacks", function() {
     describe("cancelTimeout", function() {
         it("should cancel a pending timeout", function() {
             const callback = jest.fn();
-            const k = callbacks.setTimeout(callback);
+            const k = callbacks.setTimeout(callback, 10);
             callbacks.clearTimeout(k);
-            tick(0);
+            tick(11);
             expect(callback).not.toHaveBeenCalled();
         });
 

spec/unit/room.spec.ts

@@ -361,6 +361,71 @@ describe("Room", function() {
 
             expect(callCount).toEqual(2);
         });
+
+        it("should be able to update local echo without a txn ID (/send then /sync)", function() {
+            const eventJson = utils.mkMessage({
+                room: roomId, user: userA, event: false,
+            }) as object;
+            delete eventJson["txn_id"];
+            delete eventJson["event_id"];
+            const localEvent = new MatrixEvent(Object.assign({ event_id: "$temp" }, eventJson));
+            localEvent.status = EventStatus.SENDING;
+            expect(localEvent.getTxnId()).toBeNull();
+            expect(room.timeline.length).toEqual(0);
+
+            // first add the local echo. This is done before the /send request is even sent.
+            const txnId = "My_txn_id";
+            room.addPendingEvent(localEvent, txnId);
+            expect(room.getEventForTxnId(txnId)).toEqual(localEvent);
+            expect(room.timeline.length).toEqual(1);
+
+            // now the /send request returns the true event ID.
+            const realEventId = "$real-event-id";
+            room.updatePendingEvent(localEvent, EventStatus.SENT, realEventId);
+
+            // then /sync returns the remoteEvent, it should de-dupe based on the event ID.
+            const remoteEvent = new MatrixEvent(Object.assign({ event_id: realEventId }, eventJson));
+            expect(remoteEvent.getTxnId()).toBeNull();
+            room.addLiveEvents([remoteEvent]);
+            // the duplicate strategy code should ensure we don't add a 2nd event to the live timeline
+            expect(room.timeline.length).toEqual(1);
+            // but without the event ID matching we will still have the local event in pending events
+            expect(room.getEventForTxnId(txnId)).toBeUndefined();
+        });
+
+        it("should be able to update local echo without a txn ID (/sync then /send)", function() {
+            const eventJson = utils.mkMessage({
+                room: roomId, user: userA, event: false,
+            }) as object;
+            delete eventJson["txn_id"];
+            delete eventJson["event_id"];
+            const txnId = "My_txn_id";
+            const localEvent = new MatrixEvent(Object.assign({ event_id: "$temp", txn_id: txnId }, eventJson));
+            localEvent.status = EventStatus.SENDING;
+            expect(localEvent.getTxnId()).toEqual(txnId);
+            expect(room.timeline.length).toEqual(0);
+
+            // first add the local echo. This is done before the /send request is even sent.
+            room.addPendingEvent(localEvent, txnId);
+            expect(room.getEventForTxnId(txnId)).toEqual(localEvent);
+            expect(room.timeline.length).toEqual(1);
+
+            // now the /sync returns the remoteEvent, it is impossible for the JS SDK to de-dupe this.
+            const realEventId = "$real-event-id";
+            const remoteEvent = new MatrixEvent(Object.assign({ event_id: realEventId }, eventJson));
+            expect(remoteEvent.getUnsigned().transaction_id).toBeUndefined();
+            room.addLiveEvents([remoteEvent]);
+            expect(room.timeline.length).toEqual(2); // impossible to de-dupe as no txn ID or matching event ID
+
+            // then the /send request returns the real event ID.
+            // Now it is possible for the JS SDK to de-dupe this.
+            room.updatePendingEvent(localEvent, EventStatus.SENT, realEventId);
+
+            // the 2nd event should be removed from the timeline.
+            expect(room.timeline.length).toEqual(1);
+            // but without the event ID matching we will still have the local event in pending events
+            expect(room.getEventForTxnId(txnId)).toBeUndefined();
+        });
     });
 
     describe('addEphemeralEvents', () => {
@@ -2370,16 +2435,96 @@ describe("Room", function() {
             expect(room.getEventReadUpTo(userA)).toEqual("eventId");
         });
 
-        it("prefers older receipt", () => {
-            room.getReadReceiptForUserId = (userId, ignore, receiptType) => {
-                return (receiptType === ReceiptType.Read
-                    ? { eventId: "eventId1" }
-                    : { eventId: "eventId2" }
-                    ) as IWrappedReceipt;
-            };
-            room.getUnfilteredTimelineSet = () => ({ compareEventOrdering: (event1, event2) => 1 } as EventTimelineSet);
+        describe("prefers newer receipt", () => {
+            it("should compare correctly using timelines", () => {
+                room.getReadReceiptForUserId = (userId, ignore, receiptType) => {
+                    if (receiptType === ReceiptType.ReadPrivate) {
+                        return { eventId: "eventId1" } as IWrappedReceipt;
+                    }
+                    if (receiptType === ReceiptType.UnstableReadPrivate) {
+                        return { eventId: "eventId2" } as IWrappedReceipt;
+                    }
+                    if (receiptType === ReceiptType.Read) {
+                        return { eventId: "eventId3" } as IWrappedReceipt;
+                    }
+                };
 
-            expect(room.getEventReadUpTo(userA)).toEqual("eventId1");
+                for (let i = 1; i <= 3; i++) {
+                    room.getUnfilteredTimelineSet = () => ({ compareEventOrdering: (event1, event2) => {
+                        return (event1 === `eventId${i}`) ? 1 : -1;
+                    } } as EventTimelineSet);
+
+                    expect(room.getEventReadUpTo(userA)).toEqual(`eventId${i}`);
+                }
+            });
+
+            it("should compare correctly by timestamp", () => {
+                for (let i = 1; i <= 3; i++) {
+                    room.getUnfilteredTimelineSet = () => ({
+                        compareEventOrdering: (_1, _2) => null,
+                    } as EventTimelineSet);
+                    room.getReadReceiptForUserId = (userId, ignore, receiptType) => {
+                        if (receiptType === ReceiptType.ReadPrivate) {
+                            return { eventId: "eventId1", data: { ts: i === 1 ? 1 : 0 } } as IWrappedReceipt;
+                        }
+                        if (receiptType === ReceiptType.UnstableReadPrivate) {
+                            return { eventId: "eventId2", data: { ts: i === 2 ? 1 : 0 } } as IWrappedReceipt;
+                        }
+                        if (receiptType === ReceiptType.Read) {
+                            return { eventId: "eventId3", data: { ts: i === 3 ? 1 : 0 } } as IWrappedReceipt;
+                        }
+                    };
+
+                    expect(room.getEventReadUpTo(userA)).toEqual(`eventId${i}`);
+                }
+            });
+
+            describe("fallback precedence", () => {
+                beforeAll(() => {
+                    room.getUnfilteredTimelineSet = () => ({
+                        compareEventOrdering: (_1, _2) => null,
+                    } as EventTimelineSet);
+                });
+
+                it("should give precedence to m.read.private", () => {
+                    room.getReadReceiptForUserId = (userId, ignore, receiptType) => {
+                        if (receiptType === ReceiptType.ReadPrivate) {
+                            return { eventId: "eventId1" } as IWrappedReceipt;
+                        }
+                        if (receiptType === ReceiptType.UnstableReadPrivate) {
+                            return { eventId: "eventId2" } as IWrappedReceipt;
+                        }
+                        if (receiptType === ReceiptType.Read) {
+                            return { eventId: "eventId3" } as IWrappedReceipt;
+                        }
+                    };
+
+                    expect(room.getEventReadUpTo(userA)).toEqual(`eventId1`);
+                });
+
+                it("should give precedence to org.matrix.msc2285.read.private", () => {
+                    room.getReadReceiptForUserId = (userId, ignore, receiptType) => {
+                        if (receiptType === ReceiptType.UnstableReadPrivate) {
+                            return { eventId: "eventId2" } as IWrappedReceipt;
+                        }
+                        if (receiptType === ReceiptType.Read) {
+                            return { eventId: "eventId2" } as IWrappedReceipt;
+                        }
+                    };
+
+                    expect(room.getEventReadUpTo(userA)).toEqual(`eventId2`);
+                });
+
+                it("should give precedence to m.read", () => {
+                    room.getReadReceiptForUserId = (userId, ignore, receiptType) => {
+                        if (receiptType === ReceiptType.Read) {
+                            return { eventId: "eventId3" } as IWrappedReceipt;
+                        }
+                    };
+
+                    expect(room.getEventReadUpTo(userA)).toEqual(`eventId3`);
+                });
+            });
         });
     });
 });

spec/unit/scheduler.spec.ts

@@ -45,8 +45,8 @@ describe("MatrixScheduler", function() {
         queueFn = function() {
             return "one_big_queue";
         };
-        const deferA = defer();
-        const deferB = defer();
+        const deferA = defer<Record<string, boolean>>();
+        const deferB = defer<Record<string, boolean>>();
         let yieldedA = false;
         scheduler.setProcessFunction(function(event) {
             if (yieldedA) {
@@ -70,84 +70,84 @@ describe("MatrixScheduler", function() {
     });
 
     it("should invoke the retryFn on failure and wait the amount of time specified",
-    async function() {
-        const waitTimeMs = 1500;
-        const retryDefer = defer();
-        retryFn = function() {
-            retryDefer.resolve();
-            return waitTimeMs;
-        };
-        queueFn = function() {
-            return "yep";
-        };
+        async function() {
+            const waitTimeMs = 1500;
+            const retryDefer = defer();
+            retryFn = function() {
+                retryDefer.resolve();
+                return waitTimeMs;
+            };
+            queueFn = function() {
+                return "yep";
+            };
 
-        let procCount = 0;
-        scheduler.setProcessFunction(function(ev) {
-            procCount += 1;
-            if (procCount === 1) {
-                expect(ev).toEqual(eventA);
-                return deferred.promise;
-            } else if (procCount === 2) {
+            let procCount = 0;
+            scheduler.setProcessFunction(function(ev) {
+                procCount += 1;
+                if (procCount === 1) {
+                    expect(ev).toEqual(eventA);
+                    return deferred.promise;
+                } else if (procCount === 2) {
                 // don't care about this deferred
-                return new Promise();
-            }
-            expect(procCount).toBeLessThan(3);
+                    return new Promise(() => {});
+                }
+                expect(procCount).toBeLessThan(3);
+            });
+
+            scheduler.queueEvent(eventA);
+            // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
+            // wait just long enough before it does
+            await Promise.resolve();
+            expect(procCount).toEqual(1);
+            deferred.reject({});
+            await retryDefer.promise;
+            expect(procCount).toEqual(1);
+            jest.advanceTimersByTime(waitTimeMs);
+            await Promise.resolve();
+            expect(procCount).toEqual(2);
         });
 
-        scheduler.queueEvent(eventA);
-        // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
-        // wait just long enough before it does
-        await Promise.resolve();
-        expect(procCount).toEqual(1);
-        deferred.reject({});
-        await retryDefer.promise;
-        expect(procCount).toEqual(1);
-        jest.advanceTimersByTime(waitTimeMs);
-        await Promise.resolve();
-        expect(procCount).toEqual(2);
-    });
-
     it("should give up if the retryFn on failure returns -1 and try the next event",
-    async function() {
+        async function() {
         // Queue A & B.
         // Reject A and return -1 on retry.
         // Expect B to be tried next and the promise for A to be rejected.
-        retryFn = function() {
-            return -1;
-        };
-        queueFn = function() {
-            return "yep";
-        };
+            retryFn = function() {
+                return -1;
+            };
+            queueFn = function() {
+                return "yep";
+            };
 
-        const deferA = defer();
-        const deferB = defer();
-        let procCount = 0;
-        scheduler.setProcessFunction(function(ev) {
-            procCount += 1;
-            if (procCount === 1) {
-                expect(ev).toEqual(eventA);
-                return deferA.promise;
-            } else if (procCount === 2) {
-                expect(ev).toEqual(eventB);
-                return deferB.promise;
-            }
-            expect(procCount).toBeLessThan(3);
-        });
+            const deferA = defer();
+            const deferB = defer();
+            let procCount = 0;
+            scheduler.setProcessFunction(function(ev) {
+                procCount += 1;
+                if (procCount === 1) {
+                    expect(ev).toEqual(eventA);
+                    return deferA.promise;
+                } else if (procCount === 2) {
+                    expect(ev).toEqual(eventB);
+                    return deferB.promise;
+                }
+                expect(procCount).toBeLessThan(3);
+            });
 
-        const globalA = scheduler.queueEvent(eventA);
-        scheduler.queueEvent(eventB);
-        // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
-        // wait just long enough before it does
-        await Promise.resolve();
-        expect(procCount).toEqual(1);
-        deferA.reject({});
-        try {
-            await globalA;
-        } catch (err) {
+            const globalA = scheduler.queueEvent(eventA);
+            scheduler.queueEvent(eventB);
+            // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
+            // wait just long enough before it does
             await Promise.resolve();
-            expect(procCount).toEqual(2);
-        }
-    });
+            expect(procCount).toEqual(1);
+            deferA.reject({});
+            try {
+                await globalA;
+            } catch (err) {
+                await Promise.resolve();
+                expect(procCount).toEqual(2);
+            }
+        });
 
     it("should treat each queue separately", function(done) {
         // Queue messages A B C D.
@@ -175,7 +175,7 @@ describe("MatrixScheduler", function() {
         const expectOrder = [
             eventA.getId(), eventB.getId(), eventD.getId(),
         ];
-        const deferA = defer();
+        const deferA = defer<void>();
         scheduler.setProcessFunction(function(event) {
             const id = expectOrder.shift();
             expect(id).toEqual(event.getId());
@@ -191,7 +191,7 @@ describe("MatrixScheduler", function() {
 
         // wait a bit then resolve A and we should get D (not C) next.
         setTimeout(function() {
-            deferA.resolve({});
+            deferA.resolve();
         }, 1000);
         jest.advanceTimersByTime(1000);
     });
@@ -336,28 +336,29 @@ describe("MatrixScheduler", function() {
                     errcode: "M_LIMIT_EXCEEDED", retry_after_ms: 5000,
                 }),
             );
-            expect(res >= 500).toBe(true, "Didn't wait long enough.");
+            expect(res >= 500).toBe(true);
         });
 
         it("should give up after 5 attempts", function() {
             const res = MatrixScheduler.RETRY_BACKOFF_RATELIMIT(
-                eventA, 5, {},
+                eventA, 5, new MatrixError({}),
             );
-            expect(res).toBe(-1, "Didn't give up.");
+            expect(res).toBe(-1);
         });
 
         it("should do exponential backoff", function() {
+            const error = new MatrixError({});
             expect(MatrixScheduler.RETRY_BACKOFF_RATELIMIT(
-                eventA, 1, {},
+                eventA, 1, error,
             )).toEqual(2000);
             expect(MatrixScheduler.RETRY_BACKOFF_RATELIMIT(
-                eventA, 2, {},
+                eventA, 2, error,
             )).toEqual(4000);
             expect(MatrixScheduler.RETRY_BACKOFF_RATELIMIT(
-                eventA, 3, {},
+                eventA, 3, error,
             )).toEqual(8000);
             expect(MatrixScheduler.RETRY_BACKOFF_RATELIMIT(
-                eventA, 4, {},
+                eventA, 4, error,
             )).toEqual(16000);
         });
     });

spec/unit/sync-accumulator.spec.ts

@@ -1,6 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2022 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -302,6 +302,9 @@ describe("SyncAccumulator", function() {
                     [ReceiptType.ReadPrivate]: {
                         "@dan:localhost": { ts: 4 },
                     },
+                    [ReceiptType.UnstableReadPrivate]: {
+                        "@matthew:localhost": { ts: 5 },
+                    },
                     "some.other.receipt.type": {
                         "@should_be_ignored:localhost": { key: "val" },
                     },
@@ -347,6 +350,9 @@ describe("SyncAccumulator", function() {
                     [ReceiptType.ReadPrivate]: {
                         "@dan:localhost": { ts: 4 },
                     },
+                    [ReceiptType.UnstableReadPrivate]: {
+                        "@matthew:localhost": { ts: 5 },
+                    },
                 },
                 "$event2:localhost": {
                     [ReceiptType.Read]: {
@@ -384,6 +390,10 @@ describe("SyncAccumulator", function() {
             };
         }
 
+        afterEach(() => {
+            jest.spyOn(global.Date, 'now').mockRestore();
+        });
+
         it("should copy summary properties", function() {
             sa.accumulate(createSyncResponseWithSummary({
                 "m.heroes": ["@alice:bar"],
@@ -413,25 +423,19 @@ describe("SyncAccumulator", function() {
             const delta = 1000;
             const startingTs = 1000;
 
-            const oldDateNow = Date.now;
-            try {
-                Date.now = jest.fn();
-                Date.now.mockReturnValue(startingTs);
+            jest.spyOn(global.Date, 'now').mockReturnValue(startingTs);
 
-                sa.accumulate(RES_WITH_AGE);
+            sa.accumulate(RES_WITH_AGE);
 
-                Date.now.mockReturnValue(startingTs + delta);
+            jest.spyOn(global.Date, 'now').mockReturnValue(startingTs + delta);
 
-                const output = sa.getJSON();
-                expect(output.roomsData.join["!foo:bar"].timeline.events[0].unsigned.age).toEqual(
-                    RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0].unsigned.age + delta,
-                );
-                expect(Object.keys(output.roomsData.join["!foo:bar"].timeline.events[0])).toEqual(
-                    Object.keys(RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0]),
-                );
-            } finally {
-                Date.now = oldDateNow;
-            }
+            const output = sa.getJSON();
+            expect(output.roomsData.join["!foo:bar"].timeline.events[0].unsigned.age).toEqual(
+                RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0].unsigned.age + delta,
+            );
+            expect(Object.keys(output.roomsData.join["!foo:bar"].timeline.events[0])).toEqual(
+                Object.keys(RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0]),
+            );
         });
 
         it("should mangle age without adding extra keys", () => {

spec/unit/timeline-window.spec.js

@@ -1,456 +0,0 @@
-import { EventTimeline } from "../../src/models/event-timeline";
-import { TimelineIndex, TimelineWindow } from "../../src/timeline-window";
-import * as utils from "../test-utils/test-utils";
-
-const ROOM_ID = "roomId";
-const USER_ID = "userId";
-
-/*
- * create a timeline with a bunch (default 3) events.
- * baseIndex is 1 by default.
- */
-function createTimeline(numEvents, baseIndex) {
-    if (numEvents === undefined) {
-        numEvents = 3;
-    }
-    if (baseIndex === undefined) {
-        baseIndex = 1;
-    }
-
-    // XXX: this is a horrid hack
-    const timelineSet = { room: { roomId: ROOM_ID } };
-    timelineSet.room.getUnfilteredTimelineSet = function() {
-        return timelineSet;
-    };
-
-    const timeline = new EventTimeline(timelineSet);
-
-    // add the events after the baseIndex first
-    addEventsToTimeline(timeline, numEvents - baseIndex, false);
-
-    // then add those before the baseIndex
-    addEventsToTimeline(timeline, baseIndex, true);
-
-    expect(timeline.getBaseIndex()).toEqual(baseIndex);
-    return timeline;
-}
-
-function addEventsToTimeline(timeline, numEvents, toStartOfTimeline) {
-    for (let i = 0; i < numEvents; i++) {
-        timeline.addEvent(
-            utils.mkMessage({
-                room: ROOM_ID, user: USER_ID,
-                event: true,
-            }),
-            { toStartOfTimeline },
-        );
-    }
-}
-
-/*
- * create a pair of linked timelines
- */
-function createLinkedTimelines() {
-    const tl1 = createTimeline();
-    const tl2 = createTimeline();
-    tl1.setNeighbouringTimeline(tl2, EventTimeline.FORWARDS);
-    tl2.setNeighbouringTimeline(tl1, EventTimeline.BACKWARDS);
-    return [tl1, tl2];
-}
-
-describe("TimelineIndex", function() {
-    describe("minIndex", function() {
-        it("should return the min index relative to BaseIndex", function() {
-            const timelineIndex = new TimelineIndex(createTimeline(), 0);
-            expect(timelineIndex.minIndex()).toEqual(-1);
-        });
-    });
-
-    describe("maxIndex", function() {
-        it("should return the max index relative to BaseIndex", function() {
-            const timelineIndex = new TimelineIndex(createTimeline(), 0);
-            expect(timelineIndex.maxIndex()).toEqual(2);
-        });
-    });
-
-    describe("advance", function() {
-        it("should advance up to the end of the timeline", function() {
-            const timelineIndex = new TimelineIndex(createTimeline(), 0);
-            const result = timelineIndex.advance(3);
-            expect(result).toEqual(2);
-            expect(timelineIndex.index).toEqual(2);
-        });
-
-        it("should retreat back to the start of the timeline", function() {
-            const timelineIndex = new TimelineIndex(createTimeline(), 0);
-            const result = timelineIndex.advance(-2);
-            expect(result).toEqual(-1);
-            expect(timelineIndex.index).toEqual(-1);
-        });
-
-        it("should advance into the next timeline", function() {
-            const timelines = createLinkedTimelines();
-            const tl1 = timelines[0];
-            const tl2 = timelines[1];
-
-            // initialise the index pointing at the end of the first timeline
-            const timelineIndex = new TimelineIndex(tl1, 2);
-
-            const result = timelineIndex.advance(1);
-            expect(result).toEqual(1);
-            expect(timelineIndex.timeline).toBe(tl2);
-
-            // we expect the index to be the zero (ie, the same as the
-            // BaseIndex), because the BaseIndex points at the second event,
-            // and we've advanced past the first.
-            expect(timelineIndex.index).toEqual(0);
-        });
-
-        it("should retreat into the previous timeline", function() {
-            const timelines = createLinkedTimelines();
-            const tl1 = timelines[0];
-            const tl2 = timelines[1];
-
-            // initialise the index pointing at the start of the second
-            // timeline
-            const timelineIndex = new TimelineIndex(tl2, -1);
-
-            const result = timelineIndex.advance(-1);
-            expect(result).toEqual(-1);
-            expect(timelineIndex.timeline).toBe(tl1);
-            expect(timelineIndex.index).toEqual(1);
-        });
-    });
-
-    describe("retreat", function() {
-        it("should retreat up to the start of the timeline", function() {
-            const timelineIndex = new TimelineIndex(createTimeline(), 0);
-            const result = timelineIndex.retreat(2);
-            expect(result).toEqual(1);
-            expect(timelineIndex.index).toEqual(-1);
-        });
-    });
-});
-
-describe("TimelineWindow", function() {
-    /**
-     * create a dummy eventTimelineSet and client, and a TimelineWindow
-     * attached to them.
-     */
-    let timelineSet;
-    let client;
-    function createWindow(timeline, opts) {
-        timelineSet = { getTimelineForEvent: () => null };
-        client = {};
-        client.getEventTimeline = function(timelineSet0, eventId0) {
-            expect(timelineSet0).toBe(timelineSet);
-            return Promise.resolve(timeline);
-        };
-
-        return new TimelineWindow(client, timelineSet, opts);
-    }
-
-    describe("load", function() {
-        it("should initialise from the live timeline", function() {
-            const liveTimeline = createTimeline();
-            const room = {};
-            room.getLiveTimeline = function() {
-                return liveTimeline;
-            };
-
-            const timelineWindow = new TimelineWindow(undefined, room);
-            return timelineWindow.load(undefined, 2).then(function() {
-                const expectedEvents = liveTimeline.getEvents().slice(1);
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            });
-        });
-
-        it("should initialise from a specific event", function() {
-            const timeline = createTimeline();
-            const eventId = timeline.getEvents()[1].getId();
-
-            const timelineSet = { getTimelineForEvent: () => null };
-            const client = {};
-            client.getEventTimeline = function(timelineSet0, eventId0) {
-                expect(timelineSet0).toBe(timelineSet);
-                expect(eventId0).toEqual(eventId);
-                return Promise.resolve(timeline);
-            };
-
-            const timelineWindow = new TimelineWindow(client, timelineSet);
-            return timelineWindow.load(eventId, 3).then(function() {
-                const expectedEvents = timeline.getEvents();
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            });
-        });
-
-        it("canPaginate should return false until load has returned", function() {
-            const timeline = createTimeline();
-            timeline.setPaginationToken("toktok1", EventTimeline.BACKWARDS);
-            timeline.setPaginationToken("toktok2", EventTimeline.FORWARDS);
-
-            const eventId = timeline.getEvents()[1].getId();
-
-            const timelineSet = { getTimelineForEvent: () => null };
-            const client = {};
-
-            const timelineWindow = new TimelineWindow(client, timelineSet);
-
-            client.getEventTimeline = function(timelineSet0, eventId0) {
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(false);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(false);
-                return Promise.resolve(timeline);
-            };
-
-            return timelineWindow.load(eventId, 3).then(function() {
-                const expectedEvents = timeline.getEvents();
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(true);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(true);
-            });
-        });
-    });
-
-    describe("pagination", function() {
-        it("should be able to advance across the initial timeline", function() {
-            const timeline = createTimeline();
-            const eventId = timeline.getEvents()[1].getId();
-            const timelineWindow = createWindow(timeline);
-
-            return timelineWindow.load(eventId, 1).then(function() {
-                const expectedEvents = [timeline.getEvents()[1]];
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(true);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(true);
-
-                return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
-            }).then(function(success) {
-                expect(success).toBe(true);
-                const expectedEvents = timeline.getEvents().slice(1);
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(true);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(false);
-
-                return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
-            }).then(function(success) {
-                expect(success).toBe(false);
-
-                return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
-            }).then(function(success) {
-                expect(success).toBe(true);
-                const expectedEvents = timeline.getEvents();
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(false);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(false);
-                return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
-            }).then(function(success) {
-                expect(success).toBe(false);
-            });
-        });
-
-        it("should advance into next timeline", function() {
-            const tls = createLinkedTimelines();
-            const eventId = tls[0].getEvents()[1].getId();
-            const timelineWindow = createWindow(tls[0], { windowLimit: 5 });
-
-            return timelineWindow.load(eventId, 3).then(function() {
-                const expectedEvents = tls[0].getEvents();
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(false);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(true);
-
-                return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
-            }).then(function(success) {
-                expect(success).toBe(true);
-                const expectedEvents = tls[0].getEvents()
-                    .concat(tls[1].getEvents().slice(0, 2));
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(false);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(true);
-
-                return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
-            }).then(function(success) {
-                expect(success).toBe(true);
-                // the windowLimit should have made us drop an event from
-                // tls[0]
-                const expectedEvents = tls[0].getEvents().slice(1)
-                    .concat(tls[1].getEvents());
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(true);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(false);
-                return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
-            }).then(function(success) {
-                expect(success).toBe(false);
-            });
-        });
-
-        it("should retreat into previous timeline", function() {
-            const tls = createLinkedTimelines();
-            const eventId = tls[1].getEvents()[1].getId();
-            const timelineWindow = createWindow(tls[1], { windowLimit: 5 });
-
-            return timelineWindow.load(eventId, 3).then(function() {
-                const expectedEvents = tls[1].getEvents();
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(true);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(false);
-
-                return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
-            }).then(function(success) {
-                expect(success).toBe(true);
-                const expectedEvents = tls[0].getEvents().slice(1, 3)
-                    .concat(tls[1].getEvents());
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(true);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(false);
-
-                return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
-            }).then(function(success) {
-                expect(success).toBe(true);
-                // the windowLimit should have made us drop an event from
-                // tls[1]
-                const expectedEvents = tls[0].getEvents()
-                    .concat(tls[1].getEvents().slice(0, 2));
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(false);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(true);
-                return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
-            }).then(function(success) {
-                expect(success).toBe(false);
-            });
-        });
-
-        it("should make forward pagination requests", function() {
-            const timeline = createTimeline();
-            timeline.setPaginationToken("toktok", EventTimeline.FORWARDS);
-
-            const timelineWindow = createWindow(timeline, { windowLimit: 5 });
-            const eventId = timeline.getEvents()[1].getId();
-
-            client.paginateEventTimeline = function(timeline0, opts) {
-                expect(timeline0).toBe(timeline);
-                expect(opts.backwards).toBe(false);
-                expect(opts.limit).toEqual(2);
-
-                addEventsToTimeline(timeline, 3, false);
-                return Promise.resolve(true);
-            };
-
-            return timelineWindow.load(eventId, 3).then(function() {
-                const expectedEvents = timeline.getEvents();
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(false);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(true);
-                return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
-            }).then(function(success) {
-                expect(success).toBe(true);
-                const expectedEvents = timeline.getEvents().slice(0, 5);
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            });
-        });
-
-        it("should make backward pagination requests", function() {
-            const timeline = createTimeline();
-            timeline.setPaginationToken("toktok", EventTimeline.BACKWARDS);
-
-            const timelineWindow = createWindow(timeline, { windowLimit: 5 });
-            const eventId = timeline.getEvents()[1].getId();
-
-            client.paginateEventTimeline = function(timeline0, opts) {
-                expect(timeline0).toBe(timeline);
-                expect(opts.backwards).toBe(true);
-                expect(opts.limit).toEqual(2);
-
-                addEventsToTimeline(timeline, 3, true);
-                return Promise.resolve(true);
-            };
-
-            return timelineWindow.load(eventId, 3).then(function() {
-                const expectedEvents = timeline.getEvents();
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(true);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(false);
-                return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
-            }).then(function(success) {
-                expect(success).toBe(true);
-                const expectedEvents = timeline.getEvents().slice(1, 6);
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            });
-        });
-
-        it("should limit the number of unsuccessful pagination requests", function() {
-            const timeline = createTimeline();
-            timeline.setPaginationToken("toktok", EventTimeline.FORWARDS);
-
-            const timelineWindow = createWindow(timeline, { windowLimit: 5 });
-            const eventId = timeline.getEvents()[1].getId();
-
-            let paginateCount = 0;
-            client.paginateEventTimeline = function(timeline0, opts) {
-                expect(timeline0).toBe(timeline);
-                expect(opts.backwards).toBe(false);
-                expect(opts.limit).toEqual(2);
-                paginateCount += 1;
-                return Promise.resolve(true);
-            };
-
-            return timelineWindow.load(eventId, 3).then(function() {
-                const expectedEvents = timeline.getEvents();
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(false);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(true);
-                return timelineWindow.paginate(EventTimeline.FORWARDS, 2, true, 3);
-            }).then(function(success) {
-                expect(success).toBe(false);
-                expect(paginateCount).toEqual(3);
-                const expectedEvents = timeline.getEvents().slice(0, 3);
-                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-
-                expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
-                    .toBe(false);
-                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
-                    .toBe(true);
-            });
-        });
-    });
-});

spec/unit/timeline-window.spec.ts

@@ -0,0 +1,439 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { MockedObject } from 'jest-mock';
+
+import { MatrixClient } from "../../src/client";
+import { EventTimelineSet } from "../../src/models/event-timeline-set";
+import { Room } from "../../src/models/room";
+import { EventTimeline } from "../../src/models/event-timeline";
+import { TimelineIndex, TimelineWindow } from "../../src/timeline-window";
+import { mkMessage } from "../test-utils/test-utils";
+
+const ROOM_ID = "roomId";
+const USER_ID = "userId";
+const mockClient = {
+    getEventTimeline: jest.fn(),
+    paginateEventTimeline: jest.fn(),
+} as unknown as MockedObject<MatrixClient>;
+
+/*
+ * create a timeline with a bunch (default 3) events.
+ * baseIndex is 1 by default.
+ */
+function createTimeline(numEvents = 3, baseIndex = 1): EventTimeline {
+    const room = new Room(ROOM_ID, mockClient, USER_ID);
+    const timelineSet = new EventTimelineSet(room);
+    jest.spyOn(timelineSet.room, 'getUnfilteredTimelineSet').mockReturnValue(timelineSet);
+
+    const timeline = new EventTimeline(timelineSet);
+
+    // add the events after the baseIndex first
+    addEventsToTimeline(timeline, numEvents - baseIndex, false);
+
+    // then add those before the baseIndex
+    addEventsToTimeline(timeline, baseIndex, true);
+
+    expect(timeline.getBaseIndex()).toEqual(baseIndex);
+    return timeline;
+}
+
+function addEventsToTimeline(timeline: EventTimeline, numEvents: number, toStartOfTimeline: boolean) {
+    for (let i = 0; i < numEvents; i++) {
+        timeline.addEvent(
+            mkMessage({
+                room: ROOM_ID, user: USER_ID,
+                event: true,
+            }),
+            { toStartOfTimeline },
+        );
+    }
+}
+
+/*
+ * create a pair of linked timelines
+ */
+function createLinkedTimelines(): [EventTimeline, EventTimeline] {
+    const tl1 = createTimeline();
+    const tl2 = createTimeline();
+    tl1.setNeighbouringTimeline(tl2, EventTimeline.FORWARDS);
+    tl2.setNeighbouringTimeline(tl1, EventTimeline.BACKWARDS);
+    return [tl1, tl2];
+}
+
+describe("TimelineIndex", function() {
+    beforeEach(() => {
+        jest.clearAllMocks();
+        mockClient.getEventTimeline.mockResolvedValue(undefined);
+    });
+
+    describe("minIndex", function() {
+        it("should return the min index relative to BaseIndex", function() {
+            const timelineIndex = new TimelineIndex(createTimeline(), 0);
+            expect(timelineIndex.minIndex()).toEqual(-1);
+        });
+    });
+
+    describe("maxIndex", function() {
+        it("should return the max index relative to BaseIndex", function() {
+            const timelineIndex = new TimelineIndex(createTimeline(), 0);
+            expect(timelineIndex.maxIndex()).toEqual(2);
+        });
+    });
+
+    describe("advance", function() {
+        it("should advance up to the end of the timeline", function() {
+            const timelineIndex = new TimelineIndex(createTimeline(), 0);
+            const result = timelineIndex.advance(3);
+            expect(result).toEqual(2);
+            expect(timelineIndex.index).toEqual(2);
+        });
+
+        it("should retreat back to the start of the timeline", function() {
+            const timelineIndex = new TimelineIndex(createTimeline(), 0);
+            const result = timelineIndex.advance(-2);
+            expect(result).toEqual(-1);
+            expect(timelineIndex.index).toEqual(-1);
+        });
+
+        it("should advance into the next timeline", function() {
+            const timelines = createLinkedTimelines();
+            const tl1 = timelines[0];
+            const tl2 = timelines[1];
+
+            // initialise the index pointing at the end of the first timeline
+            const timelineIndex = new TimelineIndex(tl1, 2);
+
+            const result = timelineIndex.advance(1);
+            expect(result).toEqual(1);
+            expect(timelineIndex.timeline).toBe(tl2);
+
+            // we expect the index to be the zero (ie, the same as the
+            // BaseIndex), because the BaseIndex points at the second event,
+            // and we've advanced past the first.
+            expect(timelineIndex.index).toEqual(0);
+        });
+
+        it("should retreat into the previous timeline", function() {
+            const timelines = createLinkedTimelines();
+            const tl1 = timelines[0];
+            const tl2 = timelines[1];
+
+            // initialise the index pointing at the start of the second
+            // timeline
+            const timelineIndex = new TimelineIndex(tl2, -1);
+
+            const result = timelineIndex.advance(-1);
+            expect(result).toEqual(-1);
+            expect(timelineIndex.timeline).toBe(tl1);
+            expect(timelineIndex.index).toEqual(1);
+        });
+    });
+
+    describe("retreat", function() {
+        it("should retreat up to the start of the timeline", function() {
+            const timelineIndex = new TimelineIndex(createTimeline(), 0);
+            const result = timelineIndex.retreat(2);
+            expect(result).toEqual(1);
+            expect(timelineIndex.index).toEqual(-1);
+        });
+    });
+});
+
+describe("TimelineWindow", function() {
+    /**
+     * create a dummy eventTimelineSet and client, and a TimelineWindow
+     * attached to them.
+     */
+    function createWindow(timeline: EventTimeline, opts?: {
+        windowLimit?: number;
+    }): [TimelineWindow, EventTimelineSet] {
+        const timelineSet = { getTimelineForEvent: () => null } as unknown as EventTimelineSet;
+        mockClient.getEventTimeline.mockResolvedValue(timeline);
+
+        return [new TimelineWindow(mockClient, timelineSet, opts), timelineSet];
+    }
+
+    beforeEach(() => {
+        jest.clearAllMocks();
+        mockClient.getEventTimeline.mockResolvedValue(undefined);
+        mockClient.paginateEventTimeline.mockReturnValue(undefined);
+    });
+
+    describe("load", function() {
+        it("should initialise from the live timeline", async function() {
+            const liveTimeline = createTimeline();
+            const room = new Room(ROOM_ID, mockClient, USER_ID);
+            const timelineSet = new EventTimelineSet(room);
+            jest.spyOn(timelineSet, 'getLiveTimeline').mockReturnValue(liveTimeline);
+
+            const timelineWindow = new TimelineWindow(mockClient, timelineSet);
+            await timelineWindow.load(undefined, 2);
+
+            expect(timelineSet.getLiveTimeline).toHaveBeenCalled();
+
+            const expectedEvents = liveTimeline.getEvents().slice(1);
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents);
+        });
+
+        it("should initialise from a specific event", async function() {
+            const timeline = createTimeline();
+            const eventId = timeline.getEvents()[1].getId();
+
+            const timelineSet = { getTimelineForEvent: () => null } as unknown as EventTimelineSet;
+            mockClient.getEventTimeline.mockResolvedValue(timeline);
+
+            const timelineWindow = new TimelineWindow(mockClient, timelineSet);
+            await timelineWindow.load(eventId, 3);
+            expect(mockClient.getEventTimeline).toHaveBeenCalledWith(timelineSet, eventId);
+            const expectedEvents = timeline.getEvents();
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents);
+        });
+
+        it("canPaginate should return false until load has returned", async function() {
+            const timeline = createTimeline();
+            timeline.setPaginationToken("toktok1", EventTimeline.BACKWARDS);
+            timeline.setPaginationToken("toktok2", EventTimeline.FORWARDS);
+
+            const eventId = timeline.getEvents()[1].getId();
+
+            const timelineSet = { getTimelineForEvent: () => null } as unknown as EventTimelineSet;
+            mockClient.getEventTimeline.mockResolvedValue(timeline);
+
+            const timelineWindow = new TimelineWindow(mockClient, timelineSet);
+
+            const timelineWindowLoadPromise = timelineWindow.load(eventId, 3);
+
+            // cannot paginate before load is complete
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS)).toBe(false);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS)).toBe(false);
+
+            // wait for load
+            await timelineWindowLoadPromise;
+            const expectedEvents = timeline.getEvents();
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents);
+
+            // can paginate now
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
+                .toBe(true);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
+                .toBe(true);
+        });
+    });
+
+    describe("pagination", function() {
+        it("should be able to advance across the initial timeline", async function() {
+            const timeline = createTimeline();
+            const eventId = timeline.getEvents()[1].getId();
+            const [timelineWindow] = createWindow(timeline);
+
+            await timelineWindow.load(eventId, 1);
+
+            const expectedEvents = [timeline.getEvents()[1]];
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents);
+
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
+                .toBe(true);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
+                .toBe(true);
+
+            expect(await timelineWindow.paginate(EventTimeline.FORWARDS, 2)).toBe(true);
+            const expectedEventsAfterPagination = timeline.getEvents().slice(1);
+            expect(timelineWindow.getEvents()).toEqual(expectedEventsAfterPagination);
+
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
+                .toBe(true);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
+                .toBe(false);
+
+            // cant paginate forward anymore
+            expect(await timelineWindow.paginate(EventTimeline.FORWARDS, 2)).toBe(false);
+
+            // paginate back again
+            expect(await timelineWindow.paginate(EventTimeline.BACKWARDS, 2)).toBe(true);
+
+            const expectedEvents3 = timeline.getEvents();
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents3);
+
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
+                .toBe(false);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
+                .toBe(false);
+            expect(await timelineWindow.paginate(EventTimeline.BACKWARDS, 2)).toBe(false);
+        });
+
+        it("should advance into next timeline", async function() {
+            const tls = createLinkedTimelines();
+            const eventId = tls[0].getEvents()[1].getId();
+            const [timelineWindow] = createWindow(tls[0], { windowLimit: 5 });
+
+            await timelineWindow.load(eventId, 3);
+            const expectedEvents = tls[0].getEvents();
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents);
+
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
+                .toBe(false);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
+                .toBe(true);
+
+            expect(await timelineWindow.paginate(EventTimeline.FORWARDS, 2)).toBe(true);
+            const expectedEvents2 = tls[0].getEvents()
+                .concat(tls[1].getEvents().slice(0, 2));
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents2);
+
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
+                .toBe(false);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
+                .toBe(true);
+
+            expect(await timelineWindow.paginate(EventTimeline.FORWARDS, 2)).toBe(true);
+
+            // the windowLimit should have made us drop an event from
+            // tls[0]
+            const expectedEvents3 = tls[0].getEvents().slice(1)
+                .concat(tls[1].getEvents());
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents3);
+
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
+                .toBe(true);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
+                .toBe(false);
+            expect(await timelineWindow.paginate(EventTimeline.FORWARDS, 2)).toBe(false);
+        });
+
+        it("should retreat into previous timeline", async function() {
+            const tls = createLinkedTimelines();
+            const eventId = tls[1].getEvents()[1].getId();
+            const [timelineWindow] = createWindow(tls[1], { windowLimit: 5 });
+
+            await timelineWindow.load(eventId, 3);
+
+            const expectedEvents = tls[1].getEvents();
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents);
+
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
+                .toBe(true);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
+                .toBe(false);
+
+            expect(await timelineWindow.paginate(EventTimeline.BACKWARDS, 2)).toBe(true);
+            const expectedEvents2 = tls[0].getEvents().slice(1, 3)
+                .concat(tls[1].getEvents());
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents2);
+
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
+                .toBe(true);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
+                .toBe(false);
+
+            expect(await timelineWindow.paginate(EventTimeline.BACKWARDS, 2)).toBe(true);
+            // the windowLimit should have made us drop an event from
+            // tls[1]
+            const expectedEvents3 = tls[0].getEvents()
+                .concat(tls[1].getEvents().slice(0, 2));
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents3);
+
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
+                .toBe(false);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
+                .toBe(true);
+            expect(await timelineWindow.paginate(EventTimeline.BACKWARDS, 2)).toBe(false);
+        });
+
+        it("should make forward pagination requests", async function() {
+            const timeline = createTimeline();
+            timeline.setPaginationToken("toktok", EventTimeline.FORWARDS);
+
+            const [timelineWindow] = createWindow(timeline, { windowLimit: 5 });
+            const eventId = timeline.getEvents()[1].getId();
+
+            mockClient.paginateEventTimeline.mockImplementation(async (_t, _opts) => {
+                addEventsToTimeline(timeline, 3, false);
+                return true;
+            });
+
+            await timelineWindow.load(eventId, 3);
+            const expectedEvents = timeline.getEvents();
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents);
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS)).toBe(false);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS)).toBe(true);
+
+            expect(await timelineWindow.paginate(EventTimeline.FORWARDS, 2)).toBe(true);
+            expect(mockClient.paginateEventTimeline).toHaveBeenCalledWith(timeline, { backwards: false, limit: 2 });
+            const expectedEvents2 = timeline.getEvents().slice(0, 5);
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents2);
+        });
+
+        it("should make backward pagination requests", async function() {
+            const timeline = createTimeline();
+            timeline.setPaginationToken("toktok", EventTimeline.BACKWARDS);
+
+            const [timelineWindow] = createWindow(timeline, { windowLimit: 5 });
+            const eventId = timeline.getEvents()[1].getId();
+
+            mockClient.paginateEventTimeline.mockImplementation(async (_t, _opts) => {
+                addEventsToTimeline(timeline, 3, true);
+                return true;
+            });
+
+            await timelineWindow.load(eventId, 3);
+            const expectedEvents = timeline.getEvents();
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents);
+
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS)).toBe(true);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS)).toBe(false);
+
+            expect(await timelineWindow.paginate(EventTimeline.BACKWARDS, 2)).toBe(true);
+            expect(mockClient.paginateEventTimeline).toHaveBeenCalledWith(timeline, { backwards: true, limit: 2 });
+
+            const expectedEvents2 = timeline.getEvents().slice(1, 6);
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents2);
+        });
+
+        it("should limit the number of unsuccessful pagination requests", async function() {
+            const timeline = createTimeline();
+            timeline.setPaginationToken("toktok", EventTimeline.FORWARDS);
+
+            const [timelineWindow] = createWindow(timeline, { windowLimit: 5 });
+            const eventId = timeline.getEvents()[1].getId();
+
+            mockClient.paginateEventTimeline.mockImplementation(async (_t, _opts) => {
+                return true;
+            });
+
+            await timelineWindow.load(eventId, 3);
+            const expectedEvents = timeline.getEvents();
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents);
+
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
+                .toBe(false);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
+                .toBe(true);
+            expect(await timelineWindow.paginate(EventTimeline.FORWARDS, 2, true, 3)).toBe(false);
+
+            expect(mockClient.paginateEventTimeline).toHaveBeenCalledWith(timeline, { backwards: false, limit: 2 });
+
+            expect(mockClient.paginateEventTimeline).toHaveBeenCalledTimes(3);
+            const expectedEvents2 = timeline.getEvents().slice(0, 3);
+            expect(timelineWindow.getEvents()).toEqual(expectedEvents2);
+
+            expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
+                .toBe(false);
+            expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
+                .toBe(true);
+        });
+    });
+});

spec/unit/user.spec.ts

@@ -1,16 +1,32 @@
-import { User } from "../../src/models/user";
-import * as utils from "../test-utils/test-utils";
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { User, UserEvent } from "../../src/models/user";
+import { mkEvent } from "../test-utils/test-utils";
 
 describe("User", function() {
     const userId = "@alice:bar";
-    let user;
+    let user: User;
 
     beforeEach(function() {
         user = new User(userId);
     });
 
     describe("setPresenceEvent", function() {
-        const event = utils.mkEvent({
+        const event = mkEvent({
             type: "m.presence", content: {
                 presence: "online",
                 user_id: userId,
@@ -22,7 +38,7 @@ describe("User", function() {
 
         it("should emit 'User.displayName' if the display name changes", function() {
             let emitCount = 0;
-            user.on("User.displayName", function(ev, usr) {
+            user.on(UserEvent.DisplayName, function(ev, usr) {
                 emitCount += 1;
             });
             user.setPresenceEvent(event);
@@ -33,7 +49,7 @@ describe("User", function() {
 
         it("should emit 'User.avatarUrl' if the avatar URL changes", function() {
             let emitCount = 0;
-            user.on("User.avatarUrl", function(ev, usr) {
+            user.on(UserEvent.AvatarUrl, function(ev, usr) {
                 emitCount += 1;
             });
             user.setPresenceEvent(event);
@@ -44,7 +60,7 @@ describe("User", function() {
 
         it("should emit 'User.presence' if the presence changes", function() {
             let emitCount = 0;
-            user.on("User.presence", function(ev, usr) {
+            user.on(UserEvent.Presence, function(ev, usr) {
                 emitCount += 1;
             });
             user.setPresenceEvent(event);

spec/unit/utils.spec.ts

@@ -15,6 +15,7 @@ import {
 import { logger } from "../../src/logger";
 import { mkMessage } from "../test-utils/test-utils";
 import { makeBeaconEvent } from "../test-utils/beacon";
+import { ReceiptType } from "../../src/@types/read_receipts";
 
 // TODO: Fix types throughout
 
@@ -109,18 +110,6 @@ describe("utils", function() {
         });
     });
 
-    describe("checkObjectHasNoAdditionalKeys", function() {
-        it("should throw for extra keys", function() {
-            expect(function() {
-                utils.checkObjectHasNoAdditionalKeys({ foo: "bar", baz: 4 }, ["foo"]);
-            }).toThrow();
-
-            expect(function() {
-                utils.checkObjectHasNoAdditionalKeys({ foo: "bar" }, ["foo"]);
-            }).not.toThrow();
-        });
-    });
-
     describe("deepCompare", function() {
         const assert = {
             isTrue: function(x: any) {
@@ -535,4 +524,54 @@ describe("utils", function() {
             ).toEqual([beaconEvent2, beaconEvent1, beaconEvent3]);
         });
     });
+
+    describe('getPrivateReadReceiptField', () => {
+        it('should return m.read.private if server supports stable', async () => {
+            expect(await utils.getPrivateReadReceiptField({
+                doesServerSupportUnstableFeature: jest.fn().mockImplementation((feature) => {
+                    return feature === "org.matrix.msc2285.stable";
+                }),
+            } as any)).toBe(ReceiptType.ReadPrivate);
+        });
+
+        it('should return m.read.private if server supports stable and unstable', async () => {
+            expect(await utils.getPrivateReadReceiptField({
+                doesServerSupportUnstableFeature: jest.fn().mockImplementation((feature) => {
+                    return ["org.matrix.msc2285.stable", "org.matrix.msc2285"].includes(feature);
+                }),
+            } as any)).toBe(ReceiptType.ReadPrivate);
+        });
+
+        it('should return org.matrix.msc2285.read.private if server supports  unstable', async () => {
+            expect(await utils.getPrivateReadReceiptField({
+                doesServerSupportUnstableFeature: jest.fn().mockImplementation((feature) => {
+                    return feature === "org.matrix.msc2285";
+                }),
+            } as any)).toBe(ReceiptType.UnstableReadPrivate);
+        });
+
+        it('should return none if server does not support either', async () => {
+            expect(await utils.getPrivateReadReceiptField({
+                doesServerSupportUnstableFeature: jest.fn().mockResolvedValue(false),
+            } as any)).toBeFalsy();
+        });
+    });
+
+    describe('isSupportedReceiptType', () => {
+        it('should support m.read', () => {
+            expect(utils.isSupportedReceiptType(ReceiptType.Read)).toBeTruthy();
+        });
+
+        it('should support m.read.private', () => {
+            expect(utils.isSupportedReceiptType(ReceiptType.ReadPrivate)).toBeTruthy();
+        });
+
+        it('should support org.matrix.msc2285.read.private', () => {
+            expect(utils.isSupportedReceiptType(ReceiptType.UnstableReadPrivate)).toBeTruthy();
+        });
+
+        it('should not support other receipt types', () => {
+            expect(utils.isSupportedReceiptType("this is a receipt type")).toBeFalsy();
+        });
+    });
 });

spec/unit/webrtc/call.spec.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Matrix.org Foundation C.I.C.
+Copyright 2020 - 2022 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,119 +15,25 @@ limitations under the License.
 */
 
 import { TestClient } from '../../TestClient';
-import { MatrixCall, CallErrorCode, CallEvent, supportsMatrixCall } from '../../../src/webrtc/call';
+import { MatrixCall, CallErrorCode, CallEvent, supportsMatrixCall, CallType } from '../../../src/webrtc/call';
 import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes';
-import { RoomMember } from "../../../src";
+import {
+    DUMMY_SDP,
+    MockMediaHandler,
+    MockMediaStream,
+    MockMediaStreamTrack,
+    MockMediaDeviceInfo,
+    MockRTCPeerConnection,
+} from "../../test-utils/webrtc";
+import { CallFeed } from "../../../src/webrtc/callFeed";
 
-const DUMMY_SDP = (
-    "v=0\r\n" +
-    "o=- 5022425983810148698 2 IN IP4 127.0.0.1\r\n" +
-    "s=-\r\nt=0 0\r\na=group:BUNDLE 0\r\n" +
-    "a=msid-semantic: WMS h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA\r\n" +
-    "m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126\r\n" +
-    "c=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:hLDR\r\n" +
-    "a=ice-pwd:bMGD9aOldHWiI+6nAq/IIlRw\r\n" +
-    "a=ice-options:trickle\r\n" +
-    "a=fingerprint:sha-256 E4:94:84:F9:4A:98:8A:56:F5:5F:FD:AF:72:B9:32:89:49:5C:4B:9A:" +
-        "4A:15:8E:41:8A:F3:69:E4:39:52:DC:D6\r\n" +
-    "a=setup:active\r\n" +
-    "a=mid:0\r\n" +
-    "a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n" +
-    "a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n" +
-    "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n" +
-    "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n" +
-    "a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n" +
-    "a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n" +
-    "a=sendrecv\r\n" +
-    "a=msid:h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA 4357098f-3795-4131-bff4-9ba9c0348c49\r\n" +
-    "a=rtcp-mux\r\n" +
-    "a=rtpmap:111 opus/48000/2\r\n" +
-    "a=rtcp-fb:111 transport-cc\r\n" +
-    "a=fmtp:111 minptime=10;useinbandfec=1\r\n" +
-    "a=rtpmap:103 ISAC/16000\r\n" +
-    "a=rtpmap:104 ISAC/32000\r\n" +
-    "a=rtpmap:9 G722/8000\r\n" +
-    "a=rtpmap:0 PCMU/8000\r\n" +
-    "a=rtpmap:8 PCMA/8000\r\n" +
-    "a=rtpmap:106 CN/32000\r\n" +
-    "a=rtpmap:105 CN/16000\r\n" +
-    "a=rtpmap:13 CN/8000\r\n" +
-    "a=rtpmap:110 telephone-event/48000\r\n" +
-    "a=rtpmap:112 telephone-event/32000\r\n" +
-    "a=rtpmap:113 telephone-event/16000\r\n" +
-    "a=rtpmap:126 telephone-event/8000\r\n" +
-    "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n"
-);
+const startVoiceCall = async (client: TestClient, call: MatrixCall): Promise<void> => {
+    const callPromise = call.placeVoiceCall();
+    await client.httpBackend.flush("");
+    await callPromise;
 
-class MockRTCPeerConnection {
-    localDescription: RTCSessionDescription;
-
-    constructor() {
-        this.localDescription = {
-            sdp: DUMMY_SDP,
-            type: 'offer',
-            toJSON: function() {},
-        };
-    }
-
-    addEventListener() {}
-    createOffer() {
-        return Promise.resolve({});
-    }
-    setRemoteDescription() {
-        return Promise.resolve();
-    }
-    setLocalDescription() {
-        return Promise.resolve();
-    }
-    close() {}
-    getStats() { return []; }
-    addTrack(track: MockMediaStreamTrack) {return new MockRTCRtpSender(track);}
-}
-
-class MockRTCRtpSender {
-    constructor(public track: MockMediaStreamTrack) {}
-
-    replaceTrack(track: MockMediaStreamTrack) {this.track = track;}
-}
-
-class MockMediaStreamTrack {
-    constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) {}
-
-    stop() {}
-}
-
-class MockMediaStream {
-    constructor(
-        public id: string,
-        private tracks: MockMediaStreamTrack[] = [],
-    ) {}
-
-    getTracks() { return this.tracks; }
-    getAudioTracks() { return this.tracks.filter((track) => track.kind === "audio"); }
-    getVideoTracks() { return this.tracks.filter((track) => track.kind === "video"); }
-    addEventListener() {}
-    removeEventListener() { }
-    addTrack(track: MockMediaStreamTrack) {this.tracks.push(track);}
-    removeTrack(track: MockMediaStreamTrack) {this.tracks.splice(this.tracks.indexOf(track), 1);}
-}
-
-class MockMediaDeviceInfo {
-    constructor(
-        public kind: "audio" | "video",
-    ) {}
-}
-
-class MockMediaHandler {
-    getUserMediaStream(audio: boolean, video: boolean) {
-        const tracks = [];
-        if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio"));
-        if (video) tracks.push(new MockMediaStreamTrack("video_track", "video"));
-
-        return new MockMediaStream("mock_stream_from_media_handler", tracks);
-    }
-    stopUserMediaStream() {}
-}
+    call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
+};
 
 describe('Call', function() {
     let client;
@@ -185,9 +91,8 @@ describe('Call', function() {
     });
 
     it('should ignore candidate events from non-matching party ID', async function() {
-        const callPromise = call.placeVoiceCall();
-        await client.httpBackend.flush();
-        await callPromise;
+        await startVoiceCall(client, call);
+
         await call.onAnswerReceived({
             getContent: () => {
                 return {
@@ -241,9 +146,7 @@ describe('Call', function() {
     });
 
     it('should add candidates received before answer if party ID is correct', async function() {
-        const callPromise = call.placeVoiceCall();
-        await client.httpBackend.flush();
-        await callPromise;
+        await startVoiceCall(client, call);
         call.peerConn.addIceCandidate = jest.fn();
 
         call.onRemoteIceCandidatesReceived({
@@ -301,9 +204,7 @@ describe('Call', function() {
     });
 
     it('should map asserted identity messages to remoteAssertedIdentity', async function() {
-        const callPromise = call.placeVoiceCall();
-        await client.httpBackend.flush();
-        await callPromise;
+        await startVoiceCall(client, call);
         await call.onAnswerReceived({
             getContent: () => {
                 return {
@@ -345,13 +246,7 @@ describe('Call', function() {
     });
 
     it("should map SDPStreamMetadata to feeds", async () => {
-        const callPromise = call.placeVoiceCall();
-        await client.httpBackend.flush();
-        await callPromise;
-
-        call.getOpponentMember = () => {
-            return { userId: "@bob:bar.uk" };
-        };
+        await startVoiceCall(client, call);
 
         await call.onAnswerReceived({
             getContent: () => {
@@ -389,13 +284,7 @@ describe('Call', function() {
     });
 
     it("should fallback to replaceTrack() if the other side doesn't support SPDStreamMetadata", async () => {
-        const callPromise = call.placeVoiceCall();
-        await client.httpBackend.flush();
-        await callPromise;
-
-        call.getOpponentMember = () => {
-            return { userId: "@bob:bar.uk" } as RoomMember;
-        };
+        await startVoiceCall(client, call);
 
         await call.onAnswerReceived({
             getContent: () => {
@@ -438,9 +327,7 @@ describe('Call', function() {
             ),
         );
 
-        const callPromise = call.placeVideoCall();
-        await client.httpBackend.flush();
-        await callPromise;
+        await startVoiceCall(client, call);
 
         await call.onAnswerReceived({
             getContent: () => {
@@ -476,9 +363,7 @@ describe('Call', function() {
     });
 
     it("should handle upgrade to video call", async () => {
-        const callPromise = call.placeVoiceCall();
-        await client.httpBackend.flush();
-        await callPromise;
+        await startVoiceCall(client, call);
 
         await call.onAnswerReceived({
             getContent: () => {
@@ -506,6 +391,273 @@ describe('Call', function() {
         }).track.id).toBe("video_track");
     });
 
+    it("should handle SDPStreamMetadata changes", async () => {
+        await startVoiceCall(client, call);
+
+        call.updateRemoteSDPStreamMetadata({
+            "remote_stream": {
+                purpose: SDPStreamMetadataPurpose.Usermedia,
+                audio_muted: false,
+                video_muted: false,
+            },
+        });
+        call.pushRemoteFeed(new MockMediaStream("remote_stream", []));
+        const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream");
+
+        call.onSDPStreamMetadataChangedReceived({
+            getContent: () => ({
+                [SDPStreamMetadataKey]: {
+                    "remote_stream": {
+                        purpose: SDPStreamMetadataPurpose.Screenshare,
+                        audio_muted: true,
+                        video_muted: true,
+                        id: "feed_id2",
+                    },
+                },
+            }),
+        });
+
+        expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Screenshare);
+        expect(feed?.audioMuted).toBe(true);
+        expect(feed?.videoMuted).toBe(true);
+    });
+
+    it("should choose opponent member", async () => {
+        const callPromise = call.placeVoiceCall();
+        await client.httpBackend.flush();
+        await callPromise;
+
+        const opponentMember = {
+            roomId: call.roomId,
+            userId: "opponentUserId",
+        };
+        const opponentCaps = {
+            "m.call.transferee": true,
+            "m.call.dtmf": false,
+        };
+        call.chooseOpponent({
+            getContent: () => ({
+                version: 1,
+                party_id: "party_id",
+                capabilities: opponentCaps,
+            }),
+            sender: opponentMember,
+        });
+
+        expect(call.getOpponentMember()).toBe(opponentMember);
+        expect(call.opponentPartyId).toBe("party_id");
+        expect(call.opponentCaps).toBe(opponentCaps);
+        expect(call.opponentCanBeTransferred()).toBe(true);
+        expect(call.opponentSupportsDTMF()).toBe(false);
+    });
+
+    describe("should deduce the call type correctly", () => {
+        it("if no video", async () => {
+            call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
+
+            call.pushRemoteFeed(new MockMediaStream("remote_stream1", []));
+            expect(call.type).toBe(CallType.Voice);
+        });
+
+        it("if remote video", async () => {
+            call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
+
+            call.pushRemoteFeed(new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")]));
+            expect(call.type).toBe(CallType.Video);
+        });
+
+        it("if local video", async () => {
+            call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
+
+            call.pushNewLocalFeed(
+                new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")]),
+                SDPStreamMetadataPurpose.Usermedia,
+                false,
+            );
+            expect(call.type).toBe(CallType.Video);
+        });
+    });
+
+    it("should correctly generate local SDPStreamMetadata", async () => {
+        const callPromise = call.placeCallWithCallFeeds([new CallFeed({
+            client,
+            // @ts-ignore Mock
+            stream: new MockMediaStream("local_stream1", [new MockMediaStreamTrack("track_id", "audio")]),
+            roomId: call.roomId,
+            userId: client.getUserId(),
+            purpose: SDPStreamMetadataPurpose.Usermedia,
+            audioMuted: false,
+            videoMuted: false,
+        })]);
+        await client.httpBackend.flush();
+        await callPromise;
+        call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
+
+        call.pushNewLocalFeed(
+            new MockMediaStream("local_stream2", [new MockMediaStreamTrack("track_id", "video")]),
+            SDPStreamMetadataPurpose.Screenshare, "feed_id2",
+        );
+        await call.setMicrophoneMuted(true);
+
+        expect(call.getLocalSDPStreamMetadata()).toStrictEqual({
+            "local_stream1": {
+                "purpose": SDPStreamMetadataPurpose.Usermedia,
+                "audio_muted": true,
+                "video_muted": true,
+            },
+            "local_stream2": {
+                "purpose": SDPStreamMetadataPurpose.Screenshare,
+                "audio_muted": true,
+                "video_muted": false,
+            },
+        });
+    });
+
+    it("feed and stream getters return correctly", async () => {
+        const localUsermediaStream = new MockMediaStream("local_usermedia_stream_id", []);
+        const localScreensharingStream = new MockMediaStream("local_screensharing_stream_id", []);
+        const remoteUsermediaStream = new MockMediaStream("remote_usermedia_stream_id", []);
+        const remoteScreensharingStream = new MockMediaStream("remote_screensharing_stream_id", []);
+
+        const callPromise = call.placeCallWithCallFeeds([
+            new CallFeed({
+                client,
+                userId: client.getUserId(),
+                // @ts-ignore Mock
+                stream: localUsermediaStream,
+                purpose: SDPStreamMetadataPurpose.Usermedia,
+                id: "local_usermedia_feed_id",
+                audioMuted: false,
+                videoMuted: false,
+            }),
+            new CallFeed({
+                client,
+                userId: client.getUserId(),
+                // @ts-ignore Mock
+                stream: localScreensharingStream,
+                purpose: SDPStreamMetadataPurpose.Screenshare,
+                id: "local_screensharing_feed_id",
+                audioMuted: false,
+                videoMuted: false,
+            }),
+        ]);
+        await client.httpBackend.flush();
+        await callPromise;
+        call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
+
+        call.updateRemoteSDPStreamMetadata({
+            "remote_usermedia_stream_id": {
+                purpose: SDPStreamMetadataPurpose.Usermedia,
+                id: "remote_usermedia_feed_id",
+                audio_muted: false,
+                video_muted: false,
+            },
+            "remote_screensharing_stream_id": {
+                purpose: SDPStreamMetadataPurpose.Screenshare,
+                id: "remote_screensharing_feed_id",
+                audio_muted: false,
+                video_muted: false,
+            },
+        });
+        call.pushRemoteFeed(remoteUsermediaStream);
+        call.pushRemoteFeed(remoteScreensharingStream);
+
+        expect(call.localUsermediaFeed.stream).toBe(localUsermediaStream);
+        expect(call.localUsermediaStream).toBe(localUsermediaStream);
+        expect(call.localScreensharingFeed.stream).toBe(localScreensharingStream);
+        expect(call.localScreensharingStream).toBe(localScreensharingStream);
+        expect(call.remoteUsermediaFeed.stream).toBe(remoteUsermediaStream);
+        expect(call.remoteUsermediaStream).toBe(remoteUsermediaStream);
+        expect(call.remoteScreensharingFeed.stream).toBe(remoteScreensharingStream);
+        expect(call.remoteScreensharingStream).toBe(remoteScreensharingStream);
+        expect(call.hasRemoteUserMediaAudioTrack).toBe(false);
+    });
+
+    it("should end call after receiving a select event with a different party id", async () => {
+        const callPromise = call.initWithInvite({
+            getContent: () => ({
+                version: 1,
+                call_id: "call_id",
+                party_id: "remote_party_id",
+                offer: {
+                    sdp: DUMMY_SDP,
+                },
+            }),
+            getLocalAge: () => null,
+        });
+        call.feeds.push(new CallFeed({
+            client,
+            userId: "remote_user_id",
+            // @ts-ignore Mock
+            stream: new MockMediaStream("remote_stream_id", [new MockMediaStreamTrack("remote_tack_id")]),
+            id: "remote_feed_id",
+            purpose: SDPStreamMetadataPurpose.Usermedia,
+        }));
+        await client.httpBackend.flush();
+        await callPromise;
+
+        const callHangupCallback = jest.fn();
+        call.on(CallEvent.Hangup, callHangupCallback);
+
+        await call.onSelectAnswerReceived({
+            getContent: () => ({
+                version: 1,
+                call_id: call.callId,
+                party_id: 'party_id',
+                selected_party_id: "different_party_id",
+            }),
+        });
+
+        expect(callHangupCallback).toHaveBeenCalled();
+    });
+
+    describe("turn servers", () => {
+        it("should fallback if allowed", async () => {
+            client.client.isFallbackICEServerAllowed = () => true;
+            const localCall = new MatrixCall({
+                client: client.client,
+                roomId: '!room_id',
+            });
+
+            expect((localCall as any).turnServers).toStrictEqual([{ urls: ["stun:turn.matrix.org"] }]);
+        });
+
+        it("should not fallback if not allowed", async () => {
+            client.client.isFallbackICEServerAllowed = () => false;
+            const localCall = new MatrixCall({
+                client: client.client,
+                roomId: '!room_id',
+            });
+
+            expect((localCall as any).turnServers).toStrictEqual([]);
+        });
+
+        it("should not fallback if we supplied turn servers", async () => {
+            client.client.isFallbackICEServerAllowed = () => true;
+            const turnServers = [{ urls: ["turn.server.org"] }];
+            const localCall = new MatrixCall({
+                client: client.client,
+                roomId: '!room_id',
+                turnServers,
+            });
+
+            expect((localCall as any).turnServers).toStrictEqual(turnServers);
+        });
+    });
+
+    it("should handle creating a data channel", async () => {
+        await startVoiceCall(client, call);
+
+        const dataChannelCallback = jest.fn();
+        call.on(CallEvent.DataChannel, dataChannelCallback);
+
+        const dataChannel = call.createDataChannel("data_channel_label", { id: 123 });
+
+        expect(dataChannelCallback).toHaveBeenCalledWith(dataChannel);
+        expect(dataChannel.label).toBe("data_channel_label");
+        expect(dataChannel.id).toBe(123);
+    });
+
     describe("supportsMatrixCall", () => {
         it("should return true when the environment is right", () => {
             expect(supportsMatrixCall()).toBe(true);
@@ -541,4 +693,63 @@ describe('Call', function() {
             expect(supportsMatrixCall()).toBe(false);
         });
     });
+
+    describe("ignoring streams with ids for which we already have a feed", () => {
+        const STREAM_ID = "stream_id";
+        const FEEDS_CHANGED_CALLBACK = jest.fn();
+
+        beforeEach(async () => {
+            await startVoiceCall(client, call);
+            call.on(CallEvent.FeedsChanged, FEEDS_CHANGED_CALLBACK);
+            jest.spyOn(call, "pushLocalFeed");
+        });
+
+        afterEach(() => {
+            FEEDS_CHANGED_CALLBACK.mockReset();
+        });
+
+        it("should ignore stream passed to pushRemoteFeed()", async () => {
+            await call.onAnswerReceived({
+                getContent: () => {
+                    return {
+                        version: 1,
+                        call_id: call.callId,
+                        party_id: 'party_id',
+                        answer: {
+                            sdp: DUMMY_SDP,
+                        },
+                        [SDPStreamMetadataKey]: {
+                            [STREAM_ID]: {
+                                purpose: SDPStreamMetadataPurpose.Usermedia,
+                            },
+                        },
+                    };
+                },
+            });
+
+            call.pushRemoteFeed(new MockMediaStream(STREAM_ID));
+            call.pushRemoteFeed(new MockMediaStream(STREAM_ID));
+
+            expect(call.getRemoteFeeds().length).toBe(1);
+            expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1);
+        });
+
+        it("should ignore stream passed to pushRemoteFeedWithoutMetadata()", async () => {
+            call.pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID));
+            call.pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID));
+
+            expect(call.getRemoteFeeds().length).toBe(1);
+            expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1);
+        });
+
+        it("should ignore stream passed to pushNewLocalFeed()", async () => {
+            call.pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare);
+            call.pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare);
+
+            // We already have one local feed from placeVoiceCall()
+            expect(call.getLocalFeeds().length).toBe(2);
+            expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1);
+            expect(call.pushLocalFeed).toHaveBeenCalled();
+        });
+    });
 });

spec/unit/webrtc/callFeed.spec.ts

@@ -0,0 +1,89 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes";
+import { CallFeed } from "../../../src/webrtc/callFeed";
+import { TestClient } from "../../TestClient";
+import { MockMediaStream, MockMediaStreamTrack } from "../../test-utils/webrtc";
+
+describe("CallFeed", () => {
+    let client;
+
+    beforeEach(() => {
+        client = new TestClient("@alice:foo", "somedevice", "token", undefined, {});
+    });
+
+    afterEach(() => {
+        client.stop();
+    });
+
+    describe("muting", () => {
+        let feed: CallFeed;
+
+        beforeEach(() => {
+            feed = new CallFeed({
+                client,
+                roomId: "room1",
+                userId: "user1",
+                // @ts-ignore Mock
+                stream: new MockMediaStream("stream1"),
+                purpose: SDPStreamMetadataPurpose.Usermedia,
+                audioMuted: false,
+                videoMuted: false,
+            });
+        });
+
+        describe("muting by default", () => {
+            it("should mute audio by default", () => {
+                expect(feed.isAudioMuted()).toBeTruthy();
+            });
+
+            it("should mute video by default", () => {
+                expect(feed.isVideoMuted()).toBeTruthy();
+            });
+        });
+
+        describe("muting after adding a track", () => {
+            it("should un-mute audio", () => {
+                // @ts-ignore Mock
+                feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true));
+                expect(feed.isAudioMuted()).toBeFalsy();
+            });
+
+            it("should un-mute video", () => {
+                // @ts-ignore Mock
+                feed.stream.addTrack(new MockMediaStreamTrack("track", "video", true));
+                expect(feed.isVideoMuted()).toBeFalsy();
+            });
+        });
+
+        describe("muting after calling setAudioVideoMuted()", () => {
+            it("should mute audio by default ", () => {
+                // @ts-ignore Mock
+                feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true));
+                feed.setAudioVideoMuted(true, false);
+                expect(feed.isAudioMuted()).toBeTruthy();
+            });
+
+            it("should mute video by default", () => {
+                // @ts-ignore Mock
+                feed.stream.addTrack(new MockMediaStreamTrack("track", "video", true));
+                feed.setAudioVideoMuted(false, true);
+                expect(feed.isVideoMuted()).toBeTruthy();
+            });
+        });
+    });
+});

src/@types/read_receipts.ts

@@ -17,5 +17,9 @@ limitations under the License.
 export enum ReceiptType {
     Read = "m.read",
     FullyRead = "m.fully_read",
-    ReadPrivate = "org.matrix.msc2285.read.private"
+    ReadPrivate = "m.read.private",
+    /**
+     * @deprecated Please use the ReadPrivate type when possible. This value may be removed at any time without notice.
+     */
+    UnstableReadPrivate = "org.matrix.msc2285.read.private",
 }

src/@types/requests.ts

@@ -113,7 +113,7 @@ export interface IRoomDirectoryOptions {
     since?: string;
     filter?: {
         generic_search_term?: string;
-        "org.matrix.msc3827.room_types"?: Array<RoomType | null>;
+        room_types?: Array<RoomType | null>;
     };
     include_all_networks?: boolean;
     third_party_instance_id?: string;

src/ToDeviceMessageQueue.ts

@@ -0,0 +1,125 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { logger } from "./logger";
+import { MatrixClient } from "./matrix";
+import { IndexedToDeviceBatch, ToDeviceBatch, ToDeviceBatchWithTxnId, ToDevicePayload } from "./models/ToDeviceMessage";
+import { MatrixScheduler } from "./scheduler";
+
+const MAX_BATCH_SIZE = 20;
+
+/**
+ * Maintains a queue of outgoing to-device messages, sending them
+ * as soon as the homeserver is reachable.
+ */
+export class ToDeviceMessageQueue {
+    private sending = false;
+    private running = true;
+    private retryTimeout: number = null;
+    private retryAttempts = 0;
+
+    constructor(private client: MatrixClient) {
+    }
+
+    public start(): void {
+        this.running = true;
+        this.sendQueue();
+    }
+
+    public stop(): void {
+        this.running = false;
+        if (this.retryTimeout !== null) clearTimeout(this.retryTimeout);
+        this.retryTimeout = null;
+    }
+
+    public async queueBatch(batch: ToDeviceBatch): Promise<void> {
+        const batches: ToDeviceBatchWithTxnId[] = [];
+        for (let i = 0; i < batch.batch.length; i += MAX_BATCH_SIZE) {
+            batches.push({
+                eventType: batch.eventType,
+                batch: batch.batch.slice(i, i + MAX_BATCH_SIZE),
+                txnId: this.client.makeTxnId(),
+            });
+        }
+
+        await this.client.store.saveToDeviceBatches(batches);
+        this.sendQueue();
+    }
+
+    public sendQueue = async (): Promise<void> => {
+        if (this.retryTimeout !== null) clearTimeout(this.retryTimeout);
+        this.retryTimeout = null;
+
+        if (this.sending || !this.running) return;
+
+        logger.debug("Attempting to send queued to-device messages");
+
+        this.sending = true;
+        let headBatch;
+        try {
+            while (this.running) {
+                headBatch = await this.client.store.getOldestToDeviceBatch();
+                if (headBatch === null) break;
+                await this.sendBatch(headBatch);
+                await this.client.store.removeToDeviceBatch(headBatch.id);
+                this.retryAttempts = 0;
+            }
+
+            // Make sure we're still running after the async tasks: if not, stop.
+            if (!this.running) return;
+
+            logger.debug("All queued to-device messages sent");
+        } catch (e) {
+            ++this.retryAttempts;
+            // eslint-disable-next-line @typescript-eslint/naming-convention
+            // eslint-disable-next-line new-cap
+            const retryDelay = MatrixScheduler.RETRY_BACKOFF_RATELIMIT(null, this.retryAttempts, e);
+            if (retryDelay === -1) {
+                // the scheduler function doesn't differentiate between fatal errors and just getting
+                // bored and giving up for now
+                if (Math.floor(e.httpStatus / 100) === 4) {
+                    logger.error("Fatal error when sending to-device message - dropping to-device batch!", e);
+                    await this.client.store.removeToDeviceBatch(headBatch.id);
+                } else {
+                    logger.info("Automatic retry limit reached for to-device messages.");
+                }
+                return;
+            }
+
+            logger.info(`Failed to send batch of to-device messages. Will retry in ${retryDelay}ms`, e);
+            this.retryTimeout = setTimeout(this.sendQueue, retryDelay);
+        } finally {
+            this.sending = false;
+        }
+    };
+
+    /**
+     * Attempts to send a batch of to-device messages.
+     */
+    private async sendBatch(batch: IndexedToDeviceBatch): Promise<void> {
+        const contentMap: Record<string, Record<string, ToDevicePayload>> = {};
+        for (const item of batch.batch) {
+            if (!contentMap[item.userId]) {
+                contentMap[item.userId] = {};
+            }
+            contentMap[item.userId][item.deviceId] = item.payload;
+        }
+
+        logger.info(`Sending batch of ${batch.batch.length} to-device messages with ID ${batch.id}`);
+
+        await this.client.sendToDevice(batch.eventType, contentMap, batch.txnId);
+    }
+}

src/client.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2015-2021 The Matrix.org Foundation C.I.C.
+Copyright 2015-2022 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -42,7 +42,8 @@ import { IActionsObject, PushProcessor } from "./pushprocessor";
 import { AutoDiscovery, AutoDiscoveryAction } from "./autodiscovery";
 import * as olmlib from "./crypto/olmlib";
 import { decodeBase64, encodeBase64 } from "./crypto/olmlib";
-import { IExportedDevice as IOlmDevice } from "./crypto/OlmDevice";
+import { IExportedDevice as IExportedOlmDevice } from "./crypto/OlmDevice";
+import { IOlmDevice } from "./crypto/algorithms/megolm";
 import { TypedReEmitter } from './ReEmitter';
 import { IRoomEncryption, RoomList } from './crypto/RoomList';
 import { logger } from './logger';
@@ -190,8 +191,12 @@ import { MediaHandler } from "./webrtc/mediaHandler";
 import { IRefreshTokenResponse } from "./@types/auth";
 import { TypedEventEmitter } from "./models/typed-event-emitter";
 import { ReceiptType } from "./@types/read_receipts";
+import { MSC3575SlidingSyncRequest, MSC3575SlidingSyncResponse, SlidingSync } from "./sliding-sync";
+import { SlidingSyncSdk } from "./sliding-sync-sdk";
 import { Thread, THREAD_RELATION_TYPE } from "./models/thread";
 import { MBeaconInfoEventContent, M_BEACON_INFO } from "./@types/beacon";
+import { ToDeviceMessageQueue } from "./ToDeviceMessageQueue";
+import { ToDeviceBatch } from "./models/ToDeviceMessage";
 
 export type Store = IStore;
 
@@ -204,7 +209,7 @@ const CAPABILITIES_CACHE_MS = 21600000; // 6 hours - an arbitrary value
 const TURN_CHECK_INTERVAL = 10 * 60 * 1000; // poll for turn credentials every 10 minutes
 
 interface IExportedDevice {
-    olmDevice: IOlmDevice;
+    olmDevice: IExportedOlmDevice;
     userId: string;
     deviceId: string;
 }
@@ -411,6 +416,11 @@ export interface IStartClientOpts {
      * @experimental
      */
     experimentalThreadSupport?: boolean;
+
+    /**
+     * @experimental
+     */
+     slidingSync?: SlidingSync;
 }
 
 export interface IStoredClientOpts extends IStartClientOpts {
@@ -423,15 +433,9 @@ export enum RoomVersionStability {
     Unstable = "unstable",
 }
 
-export interface IRoomCapability { // MSC3244
-    preferred: string | null;
-    support: string[];
-}
-
 export interface IRoomVersionsCapability {
     default: string;
     available: Record<string, RoomVersionStability>;
-    "org.matrix.msc3244.room_capabilities"?: Record<string, IRoomCapability>; // MSC3244
 }
 
 export interface ICapability {
@@ -501,7 +505,7 @@ interface ITurnServerResponse {
     ttl: number;
 }
 
-interface ITurnServer {
+export interface ITurnServer {
     urls: string[];
     username: string;
     credential: string;
@@ -786,6 +790,8 @@ export enum ClientEvent {
     DeleteRoom = "deleteRoom",
     SyncUnexpectedError = "sync.unexpectedError",
     ClientWellKnown = "WellKnown.client",
+    TurnServers = "turnServers",
+    TurnServersError = "turnServers.error",
 }
 
 type RoomEvents = RoomEvent.Name
@@ -856,6 +862,8 @@ export type ClientEventHandlerMap = {
     [ClientEvent.DeleteRoom]: (roomId: string) => void;
     [ClientEvent.SyncUnexpectedError]: (error: Error) => void;
     [ClientEvent.ClientWellKnown]: (data: IClientWellKnown) => void;
+    [ClientEvent.TurnServers]: (servers: ITurnServer[]) => void;
+    [ClientEvent.TurnServersError]: (error: Error, fatal: boolean) => void;
 } & RoomEventHandlerMap
     & RoomStateEventHandlerMap
     & CryptoEventHandlerMap
@@ -909,7 +917,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     protected verificationMethods: VerificationMethod[];
     protected fallbackICEServerAllowed = false;
     protected roomList: RoomList;
-    protected syncApi: SyncApi;
+    protected syncApi: SlidingSyncSdk | SyncApi;
     public pushRules: IPushRules;
     protected syncLeftRoomsPromise: Promise<Room[]>;
     protected syncedLeftRooms = false;
@@ -932,12 +940,14 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     protected clientWellKnownPromise: Promise<IClientWellKnown>;
     protected turnServers: ITurnServer[] = [];
     protected turnServersExpiry = 0;
-    protected checkTurnServersIntervalID: ReturnType<typeof setInterval>;
-    protected exportedOlmDeviceToImport: IOlmDevice;
+    protected checkTurnServersIntervalID: ReturnType<typeof setInterval> | null = null;
+    protected exportedOlmDeviceToImport: IExportedOlmDevice;
     protected txnCtr = 0;
     protected mediaHandler = new MediaHandler(this);
     protected pendingEventEncryption = new Map<string, Promise<void>>();
 
+    private toDeviceMessageQueue: ToDeviceMessageQueue;
+
     constructor(opts: IMatrixClientCreateOpts) {
         super();
 
@@ -946,6 +956,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         this.baseUrl = opts.baseUrl;
         this.idBaseUrl = opts.idBaseUrl;
+        this.identityServer = opts.identityServer;
 
         this.usingExternalCrypto = opts.usingExternalCrypto;
         this.store = opts.store || new StubStore();
@@ -1031,6 +1042,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         // we don't want to start sending unencrypted events to them.
         this.roomList = new RoomList(this.cryptoStore);
 
+        this.toDeviceMessageQueue = new ToDeviceMessageQueue(this);
+
         // The SDK doesn't really provide a clean way for events to recalculate the push
         // actions for themselves, so we have to kinda help them out when they are encrypted.
         // We do this so that push rules are correctly executed on events in their decrypted
@@ -1075,11 +1088,12 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
                 // Figure out if we've read something or if it's just informational
                 const content = event.getContent();
                 const isSelf = Object.keys(content).filter(eid => {
-                    const read = content[eid][ReceiptType.Read];
-                    if (read && Object.keys(read).includes(this.getUserId())) return true;
+                    for (const [key, value] of Object.entries(content[eid])) {
+                        if (!utils.isSupportedReceiptType(key)) continue;
+                        if (!value) continue;
 
-                    const readPrivate = content[eid][ReceiptType.ReadPrivate];
-                    if (readPrivate && Object.keys(readPrivate).includes(this.getUserId())) return true;
+                        if (Object.keys(value).includes(this.getUserId())) return true;
+                    }
 
                     return false;
                 }).length > 0;
@@ -1181,7 +1195,11 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             }
             return this.canResetTimelineCallback(roomId);
         };
-        this.syncApi = new SyncApi(this, this.clientOpts);
+        if (this.clientOpts.slidingSync) {
+            this.syncApi = new SlidingSyncSdk(this.clientOpts.slidingSync, this, this.clientOpts);
+        } else {
+            this.syncApi = new SyncApi(this, this.clientOpts);
+        }
         this.syncApi.sync();
 
         if (this.clientOpts.clientWellKnownPollPeriod !== undefined) {
@@ -1190,6 +1208,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             }, 1000 * this.clientOpts.clientWellKnownPollPeriod);
             this.fetchClientWellKnown();
         }
+
+        this.toDeviceMessageQueue.start();
     }
 
     /**
@@ -1214,9 +1234,13 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         this.callEventHandler = null;
 
         global.clearInterval(this.checkTurnServersIntervalID);
+        this.checkTurnServersIntervalID = null;
+
         if (this.clientWellKnownIntervalID !== undefined) {
             global.clearInterval(this.clientWellKnownIntervalID);
         }
+
+        this.toDeviceMessageQueue.stop();
     }
 
     /**
@@ -1403,7 +1427,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      *
      * @return {?string} MXID for the logged-in user, or null if not logged in
      */
-    public getUserId(): string {
+    public getUserId(): string | null {
         if (this.credentials && this.credentials.userId) {
             return this.credentials.userId;
         }
@@ -1425,7 +1449,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * Get the local part of the current user ID e.g. "foo" in "@foo:bar".
      * @return {?string} The user ID localpart or null.
      */
-    public getUserIdLocalpart(): string {
+    public getUserIdLocalpart(): string | null {
         if (this.credentials && this.credentials.userId) {
             return this.credentials.userId.split(":")[0].substring(1);
         }
@@ -1480,7 +1504,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * @param {string} roomId The room the call is to be placed in.
      * @return {MatrixCall} the call or null if the browser doesn't support calling.
      */
-    public createCall(roomId: string): MatrixCall {
+    public createCall(roomId: string): MatrixCall | null {
         return createNewMatrixCall(this, roomId);
     }
 
@@ -1555,9 +1579,13 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     /**
      * Retry a backed off syncing request immediately. This should only be used when
      * the user <b>explicitly</b> attempts to retry their lost connection.
+     * Will also retry any outbound to-device messages currently in the queue to be sent
+     * (retries of regular outgoing events are handled separately, per-event).
      * @return {boolean} True if this resulted in a request being retried.
      */
     public retryImmediately(): boolean {
+        // don't await for this promise: we just want to kick it off
+        this.toDeviceMessageQueue.sendQueue();
         return this.syncApi.retryImmediately();
     }
 
@@ -1788,7 +1816,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      *
      * @return {module:crypto/deviceinfo} device or null
      */
-    public getStoredDevice(userId: string, deviceId: string): DeviceInfo {
+    public getStoredDevice(userId: string, deviceId: string): DeviceInfo | null {
         if (!this.crypto) {
             throw new Error("End-to-end encryption disabled");
         }
@@ -2538,6 +2566,30 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         return this.roomList.isRoomEncrypted(roomId);
     }
 
+    /**
+     * Encrypts and sends a given object via Olm to-device messages to a given
+     * set of devices.
+     *
+     * @param {object[]} userDeviceInfoArr
+     *   mapping from userId to deviceInfo
+     *
+     * @param {object} payload fields to include in the encrypted payload
+     *      *
+     * @return {Promise<{contentMap, deviceInfoByDeviceId}>} Promise which
+     *     resolves once the message has been encrypted and sent to the given
+     *     userDeviceMap, and returns the { contentMap, deviceInfoByDeviceId }
+     *     of the successfully sent messages.
+     */
+    public encryptAndSendToDevices(
+        userDeviceInfoArr: IOlmDevice<DeviceInfo>[],
+        payload: object,
+    ): Promise<void> {
+        if (!this.crypto) {
+            throw new Error("End-to-End encryption disabled");
+        }
+        return this.crypto.encryptAndSendToDevices(userDeviceInfoArr, payload);
+    }
+
     /**
      * Forces the current outbound group session to be discarded such
      * that another one will be created next time an event is sent.
@@ -3313,7 +3365,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * @return {?User} A user or null if there is no data store or the user does
      * not exist.
      */
-    public getUser(userId: string): User {
+    public getUser(userId: string): User | null {
         return this.store.getUser(userId);
     }
 
@@ -3494,7 +3546,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     }
 
     /**
-     * Resend an event.
+     * Resend an event. Will also retry any to-device messages waiting to be sent.
      * @param {MatrixEvent} event The event to resend.
      * @param {Room} room Optional. The room the event is in. Will update the
      * timeline entry if provided.
@@ -3502,6 +3554,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * @return {module:http-api.MatrixError} Rejects: with an error response.
      */
     public resendEvent(event: MatrixEvent, room: Room): Promise<ISendEventResponse> {
+        // also kick the to-device queue to retry
+        this.toDeviceMessageQueue.sendQueue();
+
         this.updatePendingEventStatus(room, event, EventStatus.SENDING);
         return this.encryptAndSendEvent(room, event);
     }
@@ -4606,7 +4661,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             room?.addLocalEchoReceipt(this.credentials.userId, rpEvent, ReceiptType.ReadPrivate);
         }
 
-        return this.setRoomReadMarkersHttpRequest(roomId, rmEventId, rrEventId, rpEventId);
+        return await this.setRoomReadMarkersHttpRequest(roomId, rmEventId, rrEventId, rpEventId);
     }
 
     /**
@@ -4814,8 +4869,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         };
 
         if (
-            this.identityServer &&
-            this.identityServer.getAccessToken &&
+            this.identityServer?.getAccessToken &&
             await this.doesServerAcceptIdentityAccessToken()
         ) {
             const identityAccessToken = await this.identityServer.getAccessToken();
@@ -5880,6 +5934,9 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         // There can be only room-kind push rule per room
         // and its id is the room id.
         if (this.pushRules) {
+            if (!this.pushRules[scope] || !this.pushRules[scope].room) {
+                return;
+            }
             for (let i = 0; i < this.pushRules[scope].room.length; i++) {
                 const rule = this.pushRules[scope].room[i];
                 if (rule.rule_id === roomId) {
@@ -6284,14 +6341,18 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     }
 
     /**
-     * Get the unix timestamp (in seconds) at which the current
+     * Get the unix timestamp (in milliseconds) at which the current
      * TURN credentials (from getTurnServers) expire
-     * @return {number} The expiry timestamp, in seconds, or null if no credentials
+     * @return {number} The expiry timestamp, in milliseconds, or null if no credentials
      */
     public getTurnServersExpiry(): number | null {
         return this.turnServersExpiry;
     }
 
+    public get pollingTurnServers(): boolean {
+        return this.checkTurnServersIntervalID !== null;
+    }
+
     // XXX: Intended private, used in code.
     public async checkTurnServers(): Promise<boolean> {
         if (!this.canSupportVoip) {
@@ -6319,17 +6380,21 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
                     // The TTL is in seconds but we work in ms
                     this.turnServersExpiry = Date.now() + (res.ttl * 1000);
                     credentialsGood = true;
+                    this.emit(ClientEvent.TurnServers, this.turnServers);
                 }
             } catch (err) {
                 logger.error("Failed to get TURN URIs", err);
-                // If we get a 403, there's no point in looping forever.
                 if (err.httpStatus === 403) {
+                    // We got a 403, so there's no point in looping forever.
                     logger.info("TURN access unavailable for this account: stopping credentials checks");
                     if (this.checkTurnServersIntervalID !== null) global.clearInterval(this.checkTurnServersIntervalID);
                     this.checkTurnServersIntervalID = null;
+                    this.emit(ClientEvent.TurnServersError, err, true); // fatal
+                } else {
+                    // otherwise, if we failed for whatever reason, try again the next time we're called.
+                    this.emit(ClientEvent.TurnServersError, err, false); // non-fatal
                 }
             }
-            // otherwise, if we failed for whatever reason, try again the next time we're called.
         }
 
         return credentialsGood;
@@ -6806,7 +6871,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * Get the access token associated with this account.
      * @return {?String} The access_token or null
      */
-    public getAccessToken(): string {
+    public getAccessToken(): string | null {
         return this.http.opts.accessToken || null;
     }
 
@@ -7202,8 +7267,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             .filter(i => !i.id_access_token);
         if (
             invitesNeedingToken.length > 0 &&
-            this.identityServer &&
-            this.identityServer.getAccessToken &&
+            this.identityServer?.getAccessToken &&
             await this.doesServerAcceptIdentityAccessToken()
         ) {
             const identityAccessToken = await this.identityServer.getAccessToken();
@@ -7437,7 +7501,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * don't want other users to see the read receipts. This is experimental. Optional.
      * @return {Promise} Resolves: the empty object, {}.
      */
-    public setRoomReadMarkersHttpRequest(
+    public async setRoomReadMarkersHttpRequest(
         roomId: string,
         rmEventId: string,
         rrEventId: string,
@@ -7450,9 +7514,13 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         const content = {
             [ReceiptType.FullyRead]: rmEventId,
             [ReceiptType.Read]: rrEventId,
-            [ReceiptType.ReadPrivate]: rpEventId,
         };
 
+        const privateField = await utils.getPrivateReadReceiptField(this);
+        if (privateField) {
+            content[privateField] = rpEventId;
+        }
+
         return this.http.authedRequest(undefined, Method.Post, path, undefined, content);
     }
 
@@ -8687,7 +8755,10 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     }
 
     /**
-     * Send an event to a specific list of devices
+     * Send an event to a specific list of devices.
+     * This is a low-level API that simply wraps the HTTP API
+     * call to send to-device messages. We recommend using
+     * queueToDevice() which is a higher level API.
      *
      * @param {string} eventType  type of event to send
      * @param {Object.<string, Object<string, Object>>} contentMap
@@ -8719,6 +8790,17 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         return this.http.authedRequest(undefined, Method.Put, path, undefined, body);
     }
 
+    /**
+     * Sends events directly to specific devices using Matrix's to-device
+     * messaging system. The batch will be split up into appropriately sized
+     * batches for sending and stored in the store so they can be retried
+     * later if they fail to send. Retries will happen automatically.
+     * @param batch The to-device messages to send
+     */
+    public queueToDevice(batch: ToDeviceBatch): Promise<void> {
+        return this.toDeviceMessageQueue.queueBatch(batch);
+    }
+
     /**
      * Get the third party protocols that can be reached using
      * this HS
@@ -8898,7 +8980,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
      * @param {string} roomId The room ID to get a tree space reference for.
      * @returns {MSC3089TreeSpace} The tree space, or null if not a tree space.
      */
-    public unstableGetFileTreeSpace(roomId: string): MSC3089TreeSpace {
+    public unstableGetFileTreeSpace(roomId: string): MSC3089TreeSpace | null {
         const room = this.getRoom(roomId);
         if (room?.getMyMembership() !== 'join') return null;
 
@@ -8915,6 +8997,41 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         return new MSC3089TreeSpace(this, roomId);
     }
 
+    /**
+     * Perform a single MSC3575 sliding sync request.
+     * @param {MSC3575SlidingSyncRequest} req The request to make.
+     * @param {string} proxyBaseUrl The base URL for the sliding sync proxy.
+     * @returns {MSC3575SlidingSyncResponse} The sliding sync response, or a standard error.
+     * @throws on non 2xx status codes with an object with a field "httpStatus":number.
+     */
+    public slidingSync(
+        req: MSC3575SlidingSyncRequest, proxyBaseUrl?: string,
+    ): IAbortablePromise<MSC3575SlidingSyncResponse> {
+        const qps: Record<string, any> = {};
+        if (req.pos) {
+            qps.pos = req.pos;
+            delete req.pos;
+        }
+        if (req.timeout) {
+            qps.timeout = req.timeout;
+            delete req.timeout;
+        }
+        const clientTimeout = req.clientTimeout;
+        delete req.clientTimeout;
+        return this.http.authedRequest<MSC3575SlidingSyncResponse>(
+            undefined,
+            Method.Post,
+            "/sync",
+            qps,
+            req,
+            {
+                prefix: "/_matrix/client/unstable/org.matrix.msc3575",
+                baseUrl: proxyBaseUrl,
+                localTimeoutMs: clientTimeout,
+            },
+        );
+    }
+
     /**
      * @experimental
      */
@@ -8944,12 +9061,12 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     }
 
     public processBeaconEvents(
-        room: Room,
+        room?: Room,
         events?: MatrixEvent[],
     ): void {
-        if (!events?.length) {
-            return;
-        }
+        if (!events?.length) return;
+        if (!room) return;
+
         room.currentState.processBeaconEvents(events, this);
     }
 

src/content-repo.ts

@@ -36,9 +36,9 @@ import * as utils from "./utils";
 export function getHttpUriForMxc(
     baseUrl: string,
     mxc: string,
-    width: number,
-    height: number,
-    resizeMethod: string,
+    width?: number,
+    height?: number,
+    resizeMethod?: string,
     allowDirectLinks = false,
 ): string {
     if (typeof mxc !== "string" || !mxc) {

src/crypto/algorithms/megolm.ts

@@ -606,96 +606,25 @@ class MegolmEncryption extends EncryptionAlgorithm {
     private encryptAndSendKeysToDevices(
         session: OutboundSessionInfo,
         chainIndex: number,
-        userDeviceMap: IOlmDevice[],
+        devices: IOlmDevice[],
         payload: IPayload,
     ): Promise<void> {
-        const contentMap: Record<string, Record<string, IEncryptedContent>> = {};
-        // Map from userId to a map of deviceId to deviceInfo
-        const deviceInfoByUserIdAndDeviceId = new Map<string, Map<string, DeviceInfo>>();
-
-        const promises: Promise<unknown>[] = [];
-        for (let i = 0; i < userDeviceMap.length; i++) {
-            const encryptedContent: IEncryptedContent = {
-                algorithm: olmlib.OLM_ALGORITHM,
-                sender_key: this.olmDevice.deviceCurve25519Key,
-                ciphertext: {},
-            };
-            const val = userDeviceMap[i];
-            const userId = val.userId;
-            const deviceInfo = val.deviceInfo;
-            const deviceId = deviceInfo.deviceId;
-
-            // Assign to temp value to make type-checking happy
-            let userIdDeviceInfo = deviceInfoByUserIdAndDeviceId.get(userId);
-
-            if (userIdDeviceInfo === undefined) {
-                userIdDeviceInfo = new Map<string, DeviceInfo>();
-
-                deviceInfoByUserIdAndDeviceId.set(userId, userIdDeviceInfo);
+        return this.crypto.encryptAndSendToDevices(
+            devices,
+            payload,
+        ).then(() => {
+            // store that we successfully uploaded the keys of the current slice
+            for (const device of devices) {
+                session.markSharedWithDevice(
+                    device.userId,
+                    device.deviceInfo.deviceId,
+                    device.deviceInfo.getIdentityKey(),
+                    chainIndex,
+                );
             }
-
-            // We hold by reference, this updates deviceInfoByUserIdAndDeviceId[userId]
-            userIdDeviceInfo.set(deviceId, deviceInfo);
-
-            if (!contentMap[userId]) {
-                contentMap[userId] = {};
-            }
-            contentMap[userId][deviceId] = encryptedContent;
-
-            promises.push(
-                olmlib.encryptMessageForDevice(
-                    encryptedContent.ciphertext,
-                    this.userId,
-                    this.deviceId,
-                    this.olmDevice,
-                    userId,
-                    deviceInfo,
-                    payload,
-                ),
-            );
-        }
-
-        return Promise.all(promises).then(() => {
-            // prune out any devices that encryptMessageForDevice could not encrypt for,
-            // in which case it will have just not added anything to the ciphertext object.
-            // There's no point sending messages to devices if we couldn't encrypt to them,
-            // since that's effectively a blank message.
-            for (const userId of Object.keys(contentMap)) {
-                for (const deviceId of Object.keys(contentMap[userId])) {
-                    if (Object.keys(contentMap[userId][deviceId].ciphertext).length === 0) {
-                        logger.log(
-                            "No ciphertext for device " +
-                            userId + ":" + deviceId + ": pruning",
-                        );
-                        delete contentMap[userId][deviceId];
-                    }
-                }
-                // No devices left for that user? Strip that too.
-                if (Object.keys(contentMap[userId]).length === 0) {
-                    logger.log("Pruned all devices for user " + userId);
-                    delete contentMap[userId];
-                }
-            }
-
-            // Is there anything left?
-            if (Object.keys(contentMap).length === 0) {
-                logger.log("No users left to send to: aborting");
-                return;
-            }
-
-            return this.baseApis.sendToDevice("m.room.encrypted", contentMap).then(() => {
-                // store that we successfully uploaded the keys of the current slice
-                for (const userId of Object.keys(contentMap)) {
-                    for (const deviceId of Object.keys(contentMap[userId])) {
-                        session.markSharedWithDevice(
-                            userId,
-                            deviceId,
-                            deviceInfoByUserIdAndDeviceId.get(userId).get(deviceId).getIdentityKey(),
-                            chainIndex,
-                        );
-                    }
-                }
-            });
+        }).catch((error) => {
+            logger.error("failed to encryptAndSendToDevices", error);
+            throw error;
         });
     }
 
@@ -738,7 +667,6 @@ class MegolmEncryption extends EncryptionAlgorithm {
             contentMap[userId][deviceId] = message;
         }
 
-        await this.baseApis.sendToDevice("org.matrix.room_key.withheld", contentMap);
         await this.baseApis.sendToDevice("m.room_key.withheld", contentMap);
 
         // record the fact that we notified these blocked devices

src/crypto/api.ts

@@ -109,11 +109,6 @@ export interface ISecretStorageKeyInfo {
     passphrase: IPassphraseInfo;
 }
 
-export interface ISecretStorageKey {
-    keyId: string;
-    keyInfo: ISecretStorageKeyInfo;
-}
-
 export interface IPassphraseInfo {
     algorithm: "m.pbkdf2";
     iterations: number;

src/crypto/index.ts

@@ -23,9 +23,11 @@ limitations under the License.
 
 import anotherjson from "another-json";
 
+import { EventType } from "../@types/event";
 import { TypedReEmitter } from '../ReEmitter';
 import { logger } from '../logger';
 import { IExportedDevice, OlmDevice } from "./OlmDevice";
+import { IOlmDevice } from "./algorithms/megolm";
 import * as olmlib from "./olmlib";
 import { DeviceInfoMap, DeviceList } from "./DeviceList";
 import { DeviceInfo, IDevice } from "./deviceinfo";
@@ -68,6 +70,7 @@ import { IStore } from "../store";
 import { Room, RoomEvent } from "../models/room";
 import { RoomMember, RoomMemberEvent } from "../models/room-member";
 import { EventStatus, IClearEvent, IEvent, MatrixEvent, MatrixEventEvent } from "../models/event";
+import { ToDeviceBatch } from "../models/ToDeviceMessage";
 import {
     ClientEvent,
     ICrossSigningKey,
@@ -201,6 +204,14 @@ export interface IRequestsMap {
     setRequestByChannel(channel: IVerificationChannel, request: VerificationRequest): void;
 }
 
+/* eslint-disable camelcase */
+export interface IEncryptedContent {
+    algorithm: string;
+    sender_key: string;
+    ciphertext: Record<string, string>;
+}
+/* eslint-enable camelcase */
+
 export enum CryptoEvent {
     DeviceVerificationChanged = "deviceVerificationChanged",
     UserTrustStatusChanged = "userTrustStatusChanged",
@@ -2859,7 +2870,10 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
      */
     public async decryptEvent(event: MatrixEvent): Promise<IEventDecryptionResult> {
         if (event.isRedacted()) {
-            const redactionEvent = new MatrixEvent(event.getUnsigned().redacted_because);
+            const redactionEvent = new MatrixEvent({
+                room_id: event.getRoomId(),
+                ...event.getUnsigned().redacted_because,
+            });
             const decryptedEvent = await this.decryptEvent(redactionEvent);
 
             return {
@@ -3097,6 +3111,81 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
         });
     }
 
+    /**
+     * Encrypts and sends a given object via Olm to-device messages to a given
+     * set of devices.
+     * @param {object[]} userDeviceInfoArr the devices to send to
+     * @param {object} payload fields to include in the encrypted payload
+     * @return {Promise<{contentMap, deviceInfoByDeviceId}>} Promise which
+     *     resolves once the message has been encrypted and sent to the given
+     *     userDeviceMap, and returns the { contentMap, deviceInfoByDeviceId }
+     *     of the successfully sent messages.
+     */
+    public async encryptAndSendToDevices(
+        userDeviceInfoArr: IOlmDevice<DeviceInfo>[],
+        payload: object,
+    ): Promise<void> {
+        const toDeviceBatch: ToDeviceBatch = {
+            eventType: EventType.RoomMessageEncrypted,
+            batch: [],
+        };
+
+        try {
+            await Promise.all(userDeviceInfoArr.map(async ({ userId, deviceInfo }) => {
+                const deviceId = deviceInfo.deviceId;
+                const encryptedContent: IEncryptedContent = {
+                    algorithm: olmlib.OLM_ALGORITHM,
+                    sender_key: this.olmDevice.deviceCurve25519Key,
+                    ciphertext: {},
+                };
+
+                toDeviceBatch.batch.push({
+                    userId,
+                    deviceId,
+                    payload: encryptedContent,
+                });
+
+                await olmlib.ensureOlmSessionsForDevices(
+                    this.olmDevice,
+                    this.baseApis,
+                    { [userId]: [deviceInfo] },
+                );
+                await olmlib.encryptMessageForDevice(
+                    encryptedContent.ciphertext,
+                    this.userId,
+                    this.deviceId,
+                    this.olmDevice,
+                    userId,
+                    deviceInfo,
+                    payload,
+                );
+            }));
+
+            // prune out any devices that encryptMessageForDevice could not encrypt for,
+            // in which case it will have just not added anything to the ciphertext object.
+            // There's no point sending messages to devices if we couldn't encrypt to them,
+            // since that's effectively a blank message.
+            toDeviceBatch.batch = toDeviceBatch.batch.filter(msg => {
+                if (Object.keys(msg.payload.ciphertext).length > 0) {
+                    return true;
+                } else {
+                    logger.log(`No ciphertext for device ${msg.userId}:${msg.deviceId}: pruning`);
+                    return false;
+                }
+            });
+
+            try {
+                await this.baseApis.queueToDevice(toDeviceBatch);
+            } catch (e) {
+                logger.error("sendToDevice failed", e);
+                throw e;
+            }
+        } catch (e) {
+            logger.error("encryptAndSendToDevices promises failed", e);
+            throw e;
+        }
+    }
+
     private onMembership = (event: MatrixEvent, member: RoomMember, oldMembership?: string) => {
         try {
             this.onRoomMembership(event, member, oldMembership);
@@ -3119,8 +3208,7 @@ export class Crypto extends TypedEventEmitter<CryptoEvent, CryptoEventHandlerMap
                 this.secretStorage.onRequestReceived(event);
             } else if (event.getType() === "m.secret.send") {
                 this.secretStorage.onSecretReceived(event);
-            } else if (event.getType() === "m.room_key.withheld"
-                || event.getType() === "org.matrix.room_key.withheld") {
+            } else if (event.getType() === "m.room_key.withheld") {
                 this.onRoomKeyWithheldEvent(event);
             } else if (event.getContent().transaction_id) {
                 this.onKeyVerificationMessage(event);

src/crypto/verification/Error.ts

@@ -46,13 +46,6 @@ export const newUserCancelledError = errorFactory("m.user", "Cancelled by user")
  */
 export const newTimeoutError = errorFactory("m.timeout", "Timed out");
 
-/**
- * The transaction is unknown.
- */
-export const newUnknownTransactionError = errorFactory(
-    "m.unknown_transaction", "Unknown transaction",
-);
-
 /**
  * An unknown method was selected.
  */
@@ -72,11 +65,6 @@ export const newKeyMismatchError = errorFactory(
     "m.key_mismatch", "Key mismatch",
 );
 
-/**
- * The user does not match.
- */
-export const newUserMismatchError = errorFactory("m.user_error", "User mismatch");
-
 /**
  * An invalid message was sent.
  */

src/crypto/verification/SAS.ts

@@ -180,6 +180,7 @@ function generateSas(sasBytes: number[], methods: string[]): IGeneratedSas {
 
 const macMethods = {
     "hkdf-hmac-sha256": "calculate_mac",
+    "org.matrix.msc3783.hkdf-hmac-sha256": "calculate_mac_fixed_base64",
     "hmac-sha256": "calculate_mac_long_kdf",
 };
 
@@ -221,7 +222,7 @@ const calculateKeyAgreement = {
  */
 const KEY_AGREEMENT_LIST = ["curve25519-hkdf-sha256", "curve25519"];
 const HASHES_LIST = ["sha256"];
-const MAC_LIST = ["hkdf-hmac-sha256", "hmac-sha256"];
+const MAC_LIST = ["org.matrix.msc3783.hkdf-hmac-sha256", "hkdf-hmac-sha256", "hmac-sha256"];
 const SAS_LIST = Object.keys(sasGenerators);
 
 const KEY_AGREEMENT_SET = new Set(KEY_AGREEMENT_LIST);

src/http-api.ts

@@ -111,6 +111,7 @@ interface IRequest extends _Request {
 
 interface IRequestOpts<T> {
     prefix?: string;
+    baseUrl?: string;
     localTimeoutMs?: number;
     headers?: Record<string, string>;
     json?: boolean; // defaults to true
@@ -576,6 +577,9 @@ export class MatrixHttpApi {
      * @param {string=} opts.prefix The full prefix to use e.g.
      * "/_matrix/client/v2_alpha". If not specified, uses this.opts.prefix.
      *
+     * @param {string=} opts.baseUrl The alternative base url to use.
+     * If not specified, uses this.opts.baseUrl
+     *
      * @param {Object=} opts.headers map of additional request headers
      *
      * @return {Promise} Resolves to <code>{data: {Object},
@@ -671,7 +675,8 @@ export class MatrixHttpApi {
         opts?: O,
     ): IAbortablePromise<ResponseType<T, O>> {
         const prefix = opts?.prefix ?? this.opts.prefix;
-        const fullUri = this.opts.baseUrl + prefix + path;
+        const baseUrl = opts?.baseUrl ?? this.opts.baseUrl;
+        const fullUri = baseUrl + prefix + path;
 
         return this.requestOtherUrl<T, O>(callback, method, fullUri, queryParams, data, opts);
     }

src/models/ToDeviceMessage.ts

@@ -0,0 +1,38 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export type ToDevicePayload = Record<string, any>;
+
+export interface ToDeviceMessage {
+    userId: string;
+    deviceId: string;
+    payload: ToDevicePayload;
+}
+
+export interface ToDeviceBatch {
+    eventType: string;
+    batch: ToDeviceMessage[];
+}
+
+// Only used internally
+export interface ToDeviceBatchWithTxnId extends ToDeviceBatch {
+    txnId: string;
+}
+
+// Only used internally
+export interface IndexedToDeviceBatch extends ToDeviceBatchWithTxnId {
+    id: number;
+}

src/models/room-state.ts

@@ -370,7 +370,6 @@ export class RoomState extends TypedEventEmitter<EmittedEvents, EventHandlerMap>
         });
 
         this.onBeaconLivenessChange();
-
         // update higher level data structures. This needs to be done AFTER the
         // core event dict as these structures may depend on other state events in
         // the given array (e.g. disambiguating display names in one go to do both
@@ -401,7 +400,6 @@ export class RoomState extends TypedEventEmitter<EmittedEvents, EventHandlerMap>
 
                 const member = this.getOrCreateMember(userId, event);
                 member.setMembershipEvent(event, this);
-
                 this.updateMember(member);
                 this.emit(RoomStateEvent.Members, event, this, member);
             } else if (event.getType() === EventType.RoomPowerLevels) {

src/models/room.ts

@@ -57,7 +57,7 @@ import { RelationsContainer } from "./relations-container";
 // room versions which are considered okay for people to run without being asked
 // to upgrade (ie: "stable"). Eventually, we should remove these when all homeservers
 // return an m.room_versions capability.
-const KNOWN_SAFE_ROOM_VERSION = '9';
+export const KNOWN_SAFE_ROOM_VERSION = '9';
 const SAFE_ROOM_VERSIONS = ['1', '2', '3', '4', '5', '6', '7', '8', '9'];
 
 function synthesizeReceipt(userId: string, event: MatrixEvent, receiptType: ReceiptType): MatrixEvent {
@@ -72,7 +72,7 @@ function synthesizeReceipt(userId: string, event: MatrixEvent, receiptType: Rece
                 },
             },
         },
-        type: "m.receipt",
+        type: EventType.Receipt,
         room_id: event.getRoomId(),
     });
 }
@@ -1917,6 +1917,27 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
         // If any pending visibility change is waiting for this (older) event,
         this.applyPendingVisibilityEvents(event);
 
+        // Sliding Sync modifications:
+        // The proxy cannot guarantee every sent event will have a transaction_id field, so we need
+        // to check the event ID against the list of pending events if there is no transaction ID
+        // field. Only do this for events sent by us though as it's potentially expensive to loop
+        // the pending events map.
+        const txnId = event.getUnsigned().transaction_id;
+        if (!txnId && event.getSender() === this.myUserId) {
+            // check the txn map for a matching event ID
+            for (const tid in this.txnToEvent) {
+                const localEvent = this.txnToEvent[tid];
+                if (localEvent.getId() === event.getId()) {
+                    logger.debug("processLiveEvent: found sent event without txn ID: ", tid, event.getId());
+                    // update the unsigned field so we can re-use the same codepaths
+                    const unsigned = event.getUnsigned();
+                    unsigned.transaction_id = tid;
+                    event.setUnsigned(unsigned);
+                    break;
+                }
+            }
+        }
+
         if (event.getUnsigned().transaction_id) {
             const existingEvent = this.txnToEvent[event.getUnsigned().transaction_id];
             if (existingEvent) {
@@ -2173,7 +2194,22 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
             const timeline = this.getTimelineForEvent(newEventId);
             if (timeline) {
                 // we've already received the event via the event stream.
-                // nothing more to do here.
+                // nothing more to do here, assuming the transaction ID was correctly matched.
+                // Let's check that.
+                const remoteEvent = this.findEventById(newEventId);
+                const remoteTxnId = remoteEvent.getUnsigned().transaction_id;
+                if (!remoteTxnId) {
+                    // This code path is mostly relevant for the Sliding Sync proxy.
+                    // The remote event did not contain a transaction ID, so we did not handle
+                    // the remote echo yet. Handle it now.
+                    const unsigned = remoteEvent.getUnsigned();
+                    unsigned.transaction_id = event.getTxnId();
+                    remoteEvent.setUnsigned(unsigned);
+                    // the remote event is _already_ in the timeline, so we need to remove it so
+                    // we can convert the local event into the final event.
+                    this.removeEvent(remoteEvent.getId());
+                    this.handleRemoteEcho(remoteEvent, event);
+                }
                 return;
             }
         }
@@ -2387,9 +2423,9 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
      */
     public addEphemeralEvents(events: MatrixEvent[]): void {
         for (const event of events) {
-            if (event.getType() === 'm.typing') {
+            if (event.getType() === EventType.Typing) {
                 this.currentState.setTypingEvent(event);
-            } else if (event.getType() === 'm.receipt') {
+            } else if (event.getType() === EventType.Receipt) {
                 this.addReceipt(event);
             } // else ignore - life is too short for us to care about these events
         }
@@ -2478,7 +2514,7 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
      */
     public getUsersReadUpTo(event: MatrixEvent): string[] {
         return this.getReceiptsForEvent(event).filter(function(receipt) {
-            return [ReceiptType.Read, ReceiptType.ReadPrivate].includes(receipt.type);
+            return utils.isSupportedReceiptType(receipt.type);
         }).map(function(receipt) {
             return receipt.userId;
         });
@@ -2512,25 +2548,64 @@ export class Room extends TypedEventEmitter<EmittedEvents, RoomEventHandlerMap>
      * @return {String} ID of the latest event that the given user has read, or null.
      */
     public getEventReadUpTo(userId: string, ignoreSynthesized = false): string | null {
-        const timelineSet = this.getUnfilteredTimelineSet();
-        const publicReadReceipt = this.getReadReceiptForUserId(userId, ignoreSynthesized, ReceiptType.Read);
-        const privateReadReceipt = this.getReadReceiptForUserId(userId, ignoreSynthesized, ReceiptType.ReadPrivate);
+        // XXX: This is very very ugly and I hope I won't have to ever add a new
+        // receipt type here again. IMHO this should be done by the server in
+        // some more intelligent manner or the client should just use timestamps
 
-        // If we have both, compare them
-        let comparison: number | undefined;
-        if (publicReadReceipt?.eventId && privateReadReceipt?.eventId) {
-            comparison = timelineSet.compareEventOrdering(publicReadReceipt?.eventId, privateReadReceipt?.eventId);
+        const timelineSet = this.getUnfilteredTimelineSet();
+        const publicReadReceipt = this.getReadReceiptForUserId(
+            userId,
+            ignoreSynthesized,
+            ReceiptType.Read,
+        );
+        const privateReadReceipt = this.getReadReceiptForUserId(
+            userId,
+            ignoreSynthesized,
+            ReceiptType.ReadPrivate,
+        );
+        const unstablePrivateReadReceipt = this.getReadReceiptForUserId(
+            userId,
+            ignoreSynthesized,
+            ReceiptType.UnstableReadPrivate,
+        );
+
+        // If we have all, compare them
+        if (publicReadReceipt?.eventId && privateReadReceipt?.eventId && unstablePrivateReadReceipt?.eventId) {
+            const comparison1 = timelineSet.compareEventOrdering(
+                publicReadReceipt.eventId,
+                privateReadReceipt.eventId,
+            );
+            const comparison2 = timelineSet.compareEventOrdering(
+                publicReadReceipt.eventId,
+                unstablePrivateReadReceipt.eventId,
+            );
+            const comparison3 = timelineSet.compareEventOrdering(
+                privateReadReceipt.eventId,
+                unstablePrivateReadReceipt.eventId,
+            );
+            if (comparison1 && comparison2 && comparison3) {
+                return (comparison1 > 0)
+                    ? ((comparison2 > 0) ? publicReadReceipt.eventId : unstablePrivateReadReceipt.eventId)
+                    : ((comparison3 > 0) ? privateReadReceipt.eventId : unstablePrivateReadReceipt.eventId);
+            }
         }
 
-        // If we didn't get a comparison try to compare the ts of the receipts
-        if (!comparison) comparison = publicReadReceipt?.data?.ts - privateReadReceipt?.data?.ts;
+        let latest = privateReadReceipt;
+        [unstablePrivateReadReceipt, publicReadReceipt].forEach((receipt) => {
+            if (receipt?.data?.ts > latest?.data?.ts) {
+                latest = receipt;
+            }
+        });
+        if (latest?.eventId) return latest?.eventId;
 
-        // The public receipt is more likely to drift out of date so the private
-        // one has precedence
-        if (!comparison) return privateReadReceipt?.eventId ?? publicReadReceipt?.eventId ?? null;
-
-        // If public read receipt is older, return the private one
-        return (comparison < 0) ? privateReadReceipt?.eventId : publicReadReceipt?.eventId;
+        // The more less likely it is for a read receipt to drift out of date
+        // the bigger is its precedence
+        return (
+            privateReadReceipt?.eventId ??
+            unstablePrivateReadReceipt?.eventId ??
+            publicReadReceipt?.eventId ??
+            null
+        );
     }
 
     /**

src/models/thread.ts

@@ -279,7 +279,10 @@ export class Thread extends TypedEventEmitter<EmittedEvents, EventHandlerMap> {
             this.replyCount = bundledRelationship.count;
             this._currentUserParticipated = bundledRelationship.current_user_participated;
 
-            const event = new MatrixEvent(bundledRelationship.latest_event);
+            const event = new MatrixEvent({
+                room_id: this.rootEvent.getRoomId(),
+                ...bundledRelationship.latest_event,
+            });
             this.setEventMetadata(event);
             event.setThread(this);
             this.lastEvent = event;

src/realtime-callbacks.ts

@@ -48,20 +48,6 @@ const callbackList: {
 // var debuglog = logger.log.bind(logger);
 const debuglog = function(...params: any[]) {};
 
-/**
- * Replace the function used by this module to get the current time.
- *
- * Intended for use by the unit tests.
- *
- * @param {function} [f] function which should return a millisecond counter
- *
- * @internal
- */
-export function setNow(f: () => number): void {
-    now = f || Date.now;
-}
-let now = Date.now;
-
 /**
  * reimplementation of window.setTimeout, which will call the callback if
  * the wallclock time goes past the deadline.
@@ -78,7 +64,7 @@ export function setTimeout(func: (...params: any[]) => void, delayMs: number, ..
         delayMs = 0;
     }
 
-    const runAt = now() + delayMs;
+    const runAt = Date.now() + delayMs;
     const key = count++;
     debuglog("setTimeout: scheduling cb", key, "at", runAt,
         "(delay", delayMs, ")");
@@ -141,7 +127,7 @@ function scheduleRealCallback(): void {
         return;
     }
 
-    const timestamp = now();
+    const timestamp = Date.now();
     const delayMs = Math.min(first.runAt - timestamp, TIMER_CHECK_PERIOD_MS);
 
     debuglog("scheduleRealCallback: now:", timestamp, "delay:", delayMs);
@@ -150,7 +136,7 @@ function scheduleRealCallback(): void {
 
 function runCallbacks(): void {
     let cb;
-    const timestamp = now();
+    const timestamp = Date.now();
     debuglog("runCallbacks: now:", timestamp);
 
     // get the list of things to call

src/scheduler.ts

@@ -57,7 +57,7 @@ export class MatrixScheduler<T = ISendEventResponse> {
      * failure was due to a rate limited request, the time specified in the error is
      * waited before being retried.
      * @param {MatrixEvent} event
-     * @param {Number} attempts
+     * @param {Number} attempts Number of attempts that have been made, including the one that just failed (ie. starting at 1)
      * @param {MatrixError} err
      * @return {Number}
      * @see module:scheduler~retryAlgorithm

src/sliding-sync-sdk.ts

@@ -0,0 +1,835 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { NotificationCountType, Room, RoomEvent } from "./models/room";
+import { logger } from './logger';
+import * as utils from "./utils";
+import { EventTimeline } from "./models/event-timeline";
+import { ClientEvent, IStoredClientOpts, MatrixClient, PendingEventOrdering } from "./client";
+import { ISyncStateData, SyncState } from "./sync";
+import { MatrixEvent } from "./models/event";
+import { Crypto } from "./crypto";
+import { IMinimalEvent, IRoomEvent, IStateEvent, IStrippedState } from "./sync-accumulator";
+import { MatrixError } from "./http-api";
+import { RoomStateEvent } from "./models/room-state";
+import { RoomMemberEvent } from "./models/room-member";
+import {
+    Extension,
+    ExtensionState,
+    MSC3575RoomData,
+    MSC3575SlidingSyncResponse,
+    SlidingSync,
+    SlidingSyncEvent,
+    SlidingSyncState,
+} from "./sliding-sync";
+import { EventType, IPushRules } from "./matrix";
+import { PushProcessor } from "./pushprocessor";
+
+// Number of consecutive failed syncs that will lead to a syncState of ERROR as opposed
+// to RECONNECTING. This is needed to inform the client of server issues when the
+// keepAlive is successful but the server /sync fails.
+const FAILED_SYNC_ERROR_THRESHOLD = 3;
+
+class ExtensionE2EE implements Extension {
+    constructor(private readonly crypto: Crypto) {}
+
+    public name(): string {
+        return "e2ee";
+    }
+
+    public when(): ExtensionState {
+        return ExtensionState.PreProcess;
+    }
+
+    public onRequest(isInitial: boolean): object {
+        if (!isInitial) {
+            return undefined;
+        }
+        return {
+            enabled: true, // this is sticky so only send it on the initial request
+        };
+    }
+
+    public async onResponse(data: object): Promise<void> {
+        // Handle device list updates
+        if (data["device_lists"]) {
+            await this.crypto.handleDeviceListChanges({
+                oldSyncToken: "yep", // XXX need to do this so the device list changes get processed :(
+            }, data["device_lists"]);
+        }
+
+        // Handle one_time_keys_count
+        if (data["device_one_time_keys_count"]) {
+            const currentCount = data["device_one_time_keys_count"].signed_curve25519 || 0;
+            this.crypto.updateOneTimeKeyCount(currentCount);
+        }
+        if (data["device_unused_fallback_key_types"] ||
+                data["org.matrix.msc2732.device_unused_fallback_key_types"]) {
+            // The presence of device_unused_fallback_key_types indicates that the
+            // server supports fallback keys. If there's no unused
+            // signed_curve25519 fallback key we need a new one.
+            const unusedFallbackKeys = data["device_unused_fallback_key_types"] ||
+                data["org.matrix.msc2732.device_unused_fallback_key_types"];
+            this.crypto.setNeedsNewFallback(
+                unusedFallbackKeys instanceof Array &&
+                !unusedFallbackKeys.includes("signed_curve25519"),
+            );
+        }
+    }
+}
+
+class ExtensionToDevice implements Extension {
+    private nextBatch?: string = null;
+
+    constructor(private readonly client: MatrixClient) {}
+
+    public name(): string {
+        return "to_device";
+    }
+
+    public when(): ExtensionState {
+        return ExtensionState.PreProcess;
+    }
+
+    public onRequest(isInitial: boolean): object {
+        const extReq = {
+            since: this.nextBatch !== null ? this.nextBatch : undefined,
+        };
+        if (isInitial) {
+            extReq["limit"] = 100;
+            extReq["enabled"] = true;
+        }
+        return extReq;
+    }
+
+    public async onResponse(data: object): Promise<void> {
+        const cancelledKeyVerificationTxns = [];
+        data["events"] = data["events"] || [];
+        data["events"]
+            .map(this.client.getEventMapper())
+            .map((toDeviceEvent) => { // map is a cheap inline forEach
+                // We want to flag m.key.verification.start events as cancelled
+                // if there's an accompanying m.key.verification.cancel event, so
+                // we pull out the transaction IDs from the cancellation events
+                // so we can flag the verification events as cancelled in the loop
+                // below.
+                if (toDeviceEvent.getType() === "m.key.verification.cancel") {
+                    const txnId = toDeviceEvent.getContent()['transaction_id'];
+                    if (txnId) {
+                        cancelledKeyVerificationTxns.push(txnId);
+                    }
+                }
+
+                // as mentioned above, .map is a cheap inline forEach, so return
+                // the unmodified event.
+                return toDeviceEvent;
+            })
+            .forEach(
+                (toDeviceEvent) => {
+                    const content = toDeviceEvent.getContent();
+                    if (
+                        toDeviceEvent.getType() == "m.room.message" &&
+                        content.msgtype == "m.bad.encrypted"
+                    ) {
+                        // the mapper already logged a warning.
+                        logger.log(
+                            'Ignoring undecryptable to-device event from ' +
+                            toDeviceEvent.getSender(),
+                        );
+                        return;
+                    }
+
+                    if (toDeviceEvent.getType() === "m.key.verification.start"
+                        || toDeviceEvent.getType() === "m.key.verification.request") {
+                        const txnId = content['transaction_id'];
+                        if (cancelledKeyVerificationTxns.includes(txnId)) {
+                            toDeviceEvent.flagCancelled();
+                        }
+                    }
+
+                    this.client.emit(ClientEvent.ToDeviceEvent, toDeviceEvent);
+                },
+            );
+
+        this.nextBatch = data["next_batch"];
+    }
+}
+
+class ExtensionAccountData implements Extension {
+    constructor(private readonly client: MatrixClient) {}
+
+    public name(): string {
+        return "account_data";
+    }
+
+    public when(): ExtensionState {
+        return ExtensionState.PostProcess;
+    }
+
+    public onRequest(isInitial: boolean): object {
+        if (!isInitial) {
+            return undefined;
+        }
+        return {
+            enabled: true,
+        };
+    }
+
+    public onResponse(data: {global: object[], rooms: Record<string, object[]>}): void {
+        if (data.global && data.global.length > 0) {
+            this.processGlobalAccountData(data.global);
+        }
+
+        for (const roomId in data.rooms) {
+            const accountDataEvents = mapEvents(this.client, roomId, data.rooms[roomId]);
+            const room = this.client.getRoom(roomId);
+            if (!room) {
+                logger.warn("got account data for room but room doesn't exist on client:", roomId);
+                continue;
+            }
+            room.addAccountData(accountDataEvents);
+            accountDataEvents.forEach((e) => {
+                this.client.emit(ClientEvent.Event, e);
+            });
+        }
+    }
+
+    private processGlobalAccountData(globalAccountData: object[]): void {
+        const events = mapEvents(this.client, undefined, globalAccountData);
+        const prevEventsMap = events.reduce((m, c) => {
+            m[c.getId()] = this.client.store.getAccountData(c.getType());
+            return m;
+        }, {});
+        this.client.store.storeAccountDataEvents(events);
+        events.forEach(
+            (accountDataEvent) => {
+                // Honour push rules that come down the sync stream but also
+                // honour push rules that were previously cached. Base rules
+                // will be updated when we receive push rules via getPushRules
+                // (see sync) before syncing over the network.
+                if (accountDataEvent.getType() === EventType.PushRules) {
+                    const rules = accountDataEvent.getContent<IPushRules>();
+                    this.client.pushRules = PushProcessor.rewriteDefaultRules(rules);
+                }
+                const prevEvent = prevEventsMap[accountDataEvent.getId()];
+                this.client.emit(ClientEvent.AccountData, accountDataEvent, prevEvent);
+                return accountDataEvent;
+            },
+        );
+    }
+}
+
+/**
+ * A copy of SyncApi such that it can be used as a drop-in replacement for sync v2. For the actual
+ * sliding sync API, see sliding-sync.ts or the class SlidingSync.
+ */
+export class SlidingSyncSdk {
+    private syncState: SyncState = null;
+    private syncStateData: ISyncStateData;
+    private lastPos: string = null;
+    private failCount = 0;
+    private notifEvents: MatrixEvent[] = []; // accumulator of sync events in the current sync response
+
+    constructor(
+        private readonly slidingSync: SlidingSync,
+        private readonly client: MatrixClient,
+        private readonly opts: Partial<IStoredClientOpts> = {},
+    ) {
+        this.opts.initialSyncLimit = this.opts.initialSyncLimit ?? 8;
+        this.opts.resolveInvitesToProfiles = this.opts.resolveInvitesToProfiles || false;
+        this.opts.pollTimeout = this.opts.pollTimeout || (30 * 1000);
+        this.opts.pendingEventOrdering = this.opts.pendingEventOrdering || PendingEventOrdering.Chronological;
+        this.opts.experimentalThreadSupport = this.opts.experimentalThreadSupport === true;
+
+        if (!opts.canResetEntireTimeline) {
+            opts.canResetEntireTimeline = (_roomId: string) => {
+                return false;
+            };
+        }
+
+        if (client.getNotifTimelineSet()) {
+            client.reEmitter.reEmit(client.getNotifTimelineSet(), [
+                RoomEvent.Timeline,
+                RoomEvent.TimelineReset,
+            ]);
+        }
+
+        this.slidingSync.on(SlidingSyncEvent.Lifecycle, this.onLifecycle.bind(this));
+        this.slidingSync.on(SlidingSyncEvent.RoomData, this.onRoomData.bind(this));
+        const extensions: Extension[] = [
+            new ExtensionToDevice(this.client),
+            new ExtensionAccountData(this.client),
+        ];
+        if (this.opts.crypto) {
+            extensions.push(
+                new ExtensionE2EE(this.opts.crypto),
+            );
+        }
+        extensions.forEach((ext) => {
+            this.slidingSync.registerExtension(ext);
+        });
+    }
+
+    private onRoomData(roomId: string, roomData: MSC3575RoomData): void {
+        let room = this.client.store.getRoom(roomId);
+        if (!room) {
+            if (!roomData.initial) {
+                logger.debug("initial flag not set but no stored room exists for room ", roomId, roomData);
+                return;
+            }
+            room = createRoom(this.client, roomId, this.opts);
+        }
+        this.processRoomData(this.client, room, roomData);
+    }
+
+    private onLifecycle(state: SlidingSyncState, resp: MSC3575SlidingSyncResponse, err?: Error): void {
+        if (err) {
+            logger.debug("onLifecycle", state, err);
+        }
+        switch (state) {
+            case SlidingSyncState.Complete:
+                this.purgeNotifications();
+                // Element won't stop showing the initial loading spinner unless we fire SyncState.Prepared
+                if (!this.lastPos) {
+                    this.updateSyncState(SyncState.Prepared, {
+                        oldSyncToken: this.lastPos,
+                        nextSyncToken: resp.pos,
+                        catchingUp: false,
+                        fromCache: false,
+                    });
+                }
+                // Conversely, Element won't show the room list unless there is at least 1x SyncState.Syncing
+                // so hence for the very first sync we will fire prepared then immediately syncing.
+                this.updateSyncState(SyncState.Syncing, {
+                    oldSyncToken: this.lastPos,
+                    nextSyncToken: resp.pos,
+                    catchingUp: false,
+                    fromCache: false,
+                });
+                this.lastPos = resp.pos;
+                break;
+            case SlidingSyncState.RequestFinished:
+                if (err) {
+                    this.failCount += 1;
+                    this.updateSyncState(
+                        this.failCount > FAILED_SYNC_ERROR_THRESHOLD ? SyncState.Error : SyncState.Reconnecting,
+                        {
+                            error: new MatrixError(err),
+                        },
+                    );
+                    if (this.shouldAbortSync(new MatrixError(err))) {
+                        return; // shouldAbortSync actually stops syncing too so we don't need to do anything.
+                    }
+                } else {
+                    this.failCount = 0;
+                }
+                break;
+        }
+    }
+
+    /**
+     * Sync rooms the user has left.
+     * @return {Promise} Resolved when they've been added to the store.
+     */
+    public async syncLeftRooms() {
+        return []; // TODO
+    }
+
+    /**
+     * Peek into a room. This will result in the room in question being synced so it
+     * is accessible via getRooms(). Live updates for the room will be provided.
+     * @param {string} roomId The room ID to peek into.
+     * @return {Promise} A promise which resolves once the room has been added to the
+     * store.
+     */
+    public async peek(_roomId: string): Promise<Room> {
+        return null; // TODO
+    }
+
+    /**
+     * Stop polling for updates in the peeked room. NOPs if there is no room being
+     * peeked.
+     */
+    public stopPeeking(): void {
+        // TODO
+    }
+
+    /**
+     * Returns the current state of this sync object
+     * @see module:client~MatrixClient#event:"sync"
+     * @return {?String}
+     */
+    public getSyncState(): SyncState {
+        return this.syncState;
+    }
+
+    /**
+     * Returns the additional data object associated with
+     * the current sync state, or null if there is no
+     * such data.
+     * Sync errors, if available, are put in the 'error' key of
+     * this object.
+     * @return {?Object}
+     */
+    public getSyncStateData(): ISyncStateData {
+        return this.syncStateData;
+    }
+
+    private shouldAbortSync(error: MatrixError): boolean {
+        if (error.errcode === "M_UNKNOWN_TOKEN") {
+            // The logout already happened, we just need to stop.
+            logger.warn("Token no longer valid - assuming logout");
+            this.stop();
+            this.updateSyncState(SyncState.Error, { error });
+            return true;
+        }
+        return false;
+    }
+
+    private async processRoomData(client: MatrixClient, room: Room, roomData: MSC3575RoomData) {
+        roomData = ensureNameEvent(client, room.roomId, roomData);
+        const stateEvents = mapEvents(this.client, room.roomId, roomData.required_state);
+        // Prevent events from being decrypted ahead of time
+        // this helps large account to speed up faster
+        // room::decryptCriticalEvent is in charge of decrypting all the events
+        // required for a client to function properly
+        const timelineEvents = mapEvents(this.client, room.roomId, roomData.timeline, false);
+        const ephemeralEvents = []; // TODO this.mapSyncEventsFormat(joinObj.ephemeral);
+
+        const encrypted = this.client.isRoomEncrypted(room.roomId);
+        // we do this first so it's correct when any of the events fire
+        if (roomData.notification_count != null) {
+            room.setUnreadNotificationCount(
+                NotificationCountType.Total,
+                roomData.notification_count,
+            );
+        }
+
+        if (roomData.highlight_count != null) {
+            // We track unread notifications ourselves in encrypted rooms, so don't
+            // bother setting it here. We trust our calculations better than the
+            // server's for this case, and therefore will assume that our non-zero
+            // count is accurate.
+            if (!encrypted
+                || (encrypted && room.getUnreadNotificationCount(NotificationCountType.Highlight) <= 0)) {
+                room.setUnreadNotificationCount(
+                    NotificationCountType.Highlight,
+                    roomData.highlight_count,
+                );
+            }
+        }
+
+        if (roomData.invite_state) {
+            const inviteStateEvents = mapEvents(this.client, room.roomId, roomData.invite_state);
+            this.processRoomEvents(room, inviteStateEvents);
+            if (roomData.initial) {
+                room.recalculate();
+                this.client.store.storeRoom(room);
+                this.client.emit(ClientEvent.Room, room);
+            }
+            inviteStateEvents.forEach((e) => {
+                this.client.emit(ClientEvent.Event, e);
+            });
+            room.updateMyMembership("invite");
+            return;
+        }
+
+        if (roomData.initial) {
+            // set the back-pagination token. Do this *before* adding any
+            // events so that clients can start back-paginating.
+            room.getLiveTimeline().setPaginationToken(
+                roomData.prev_batch, EventTimeline.BACKWARDS);
+        }
+
+        /* TODO
+        else if (roomData.limited) {
+
+            let limited = true;
+
+            // we've got a limited sync, so we *probably* have a gap in the
+            // timeline, so should reset. But we might have been peeking or
+            // paginating and already have some of the events, in which
+            // case we just want to append any subsequent events to the end
+            // of the existing timeline.
+            //
+            // This is particularly important in the case that we already have
+            // *all* of the events in the timeline - in that case, if we reset
+            // the timeline, we'll end up with an entirely empty timeline,
+            // which we'll try to paginate but not get any new events (which
+            // will stop us linking the empty timeline into the chain).
+            //
+            for (let i = timelineEvents.length - 1; i >= 0; i--) {
+                const eventId = timelineEvents[i].getId();
+                if (room.getTimelineForEvent(eventId)) {
+                    logger.debug("Already have event " + eventId + " in limited " +
+                        "sync - not resetting");
+                    limited = false;
+
+                    // we might still be missing some of the events before i;
+                    // we don't want to be adding them to the end of the
+                    // timeline because that would put them out of order.
+                    timelineEvents.splice(0, i);
+
+                    // XXX: there's a problem here if the skipped part of the
+                    // timeline modifies the state set in stateEvents, because
+                    // we'll end up using the state from stateEvents rather
+                    // than the later state from timelineEvents. We probably
+                    // need to wind stateEvents forward over the events we're
+                    // skipping.
+                    break;
+                }
+            }
+
+            if (limited) {
+                deregisterStateListeners(room);
+                room.resetLiveTimeline(
+                    roomData.prev_batch,
+                    null, // TODO this.opts.canResetEntireTimeline(room.roomId) ? null : syncEventData.oldSyncToken,
+                );
+
+                // We have to assume any gap in any timeline is
+                // reason to stop incrementally tracking notifications and
+                // reset the timeline.
+                this.client.resetNotifTimelineSet();
+                registerStateListeners(this.client, room);
+            }
+        } */
+
+        this.processRoomEvents(room, stateEvents, timelineEvents, false);
+
+        // we deliberately don't add ephemeral events to the timeline
+        room.addEphemeralEvents(ephemeralEvents);
+
+        room.recalculate();
+        if (roomData.initial) {
+            client.store.storeRoom(room);
+            client.emit(ClientEvent.Room, room);
+        }
+
+        // check if any timeline events should bing and add them to the notifEvents array:
+        // we'll purge this once we've fully processed the sync response
+        this.addNotifications(timelineEvents);
+
+        const processRoomEvent = async (e: MatrixEvent) => {
+            client.emit(ClientEvent.Event, e);
+            if (e.isState() && e.getType() == EventType.RoomEncryption && this.opts.crypto) {
+                await this.opts.crypto.onCryptoEvent(e);
+            }
+        };
+
+        await utils.promiseMapSeries(stateEvents, processRoomEvent);
+        await utils.promiseMapSeries(timelineEvents, processRoomEvent);
+        ephemeralEvents.forEach(function(e) {
+            client.emit(ClientEvent.Event, e);
+        });
+
+        room.updateMyMembership("join");
+
+        // Decrypt only the last message in all rooms to make sure we can generate a preview
+        // And decrypt all events after the recorded read receipt to ensure an accurate
+        // notification count
+        room.decryptCriticalEvents();
+    }
+
+    /**
+     * @param {Room} room
+     * @param {MatrixEvent[]} stateEventList A list of state events. This is the state
+     * at the *START* of the timeline list if it is supplied.
+     * @param {MatrixEvent[]} [timelineEventList] A list of timeline events. Lower index
+     * @param {boolean} fromCache whether the sync response came from cache
+     * is earlier in time. Higher index is later.
+     */
+    private processRoomEvents(
+        room: Room,
+        stateEventList: MatrixEvent[],
+        timelineEventList?: MatrixEvent[],
+        fromCache = false,
+    ): void {
+        timelineEventList = timelineEventList || [];
+        stateEventList = stateEventList || [];
+
+        // If there are no events in the timeline yet, initialise it with
+        // the given state events
+        const liveTimeline = room.getLiveTimeline();
+        const timelineWasEmpty = liveTimeline.getEvents().length == 0;
+        if (timelineWasEmpty) {
+            // Passing these events into initialiseState will freeze them, so we need
+            // to compute and cache the push actions for them now, otherwise sync dies
+            // with an attempt to assign to read only property.
+            // XXX: This is pretty horrible and is assuming all sorts of behaviour from
+            // these functions that it shouldn't be. We should probably either store the
+            // push actions cache elsewhere so we can freeze MatrixEvents, or otherwise
+            // find some solution where MatrixEvents are immutable but allow for a cache
+            // field.
+            for (const ev of stateEventList) {
+                this.client.getPushActionsForEvent(ev);
+            }
+            liveTimeline.initialiseState(stateEventList);
+        }
+
+        // If the timeline wasn't empty, we process the state events here: they're
+        // defined as updates to the state before the start of the timeline, so this
+        // starts to roll the state forward.
+        // XXX: That's what we *should* do, but this can happen if we were previously
+        // peeking in a room, in which case we obviously do *not* want to add the
+        // state events here onto the end of the timeline. Historically, the js-sdk
+        // has just set these new state events on the old and new state. This seems
+        // very wrong because there could be events in the timeline that diverge the
+        // state, in which case this is going to leave things out of sync. However,
+        // for now I think it;s best to behave the same as the code has done previously.
+        if (!timelineWasEmpty) {
+            // XXX: As above, don't do this...
+            //room.addLiveEvents(stateEventList || []);
+            // Do this instead...
+            room.oldState.setStateEvents(stateEventList);
+            room.currentState.setStateEvents(stateEventList);
+        }
+
+        // execute the timeline events. This will continue to diverge the current state
+        // if the timeline has any state events in it.
+        // This also needs to be done before running push rules on the events as they need
+        // to be decorated with sender etc.
+        room.addLiveEvents(timelineEventList, {
+            fromCache: fromCache,
+        });
+
+        room.recalculate();
+
+        // resolve invites now we have set the latest state
+        this.resolveInvites(room);
+    }
+
+    private resolveInvites(room: Room): void {
+        if (!room || !this.opts.resolveInvitesToProfiles) {
+            return;
+        }
+        const client = this.client;
+        // For each invited room member we want to give them a displayname/avatar url
+        // if they have one (the m.room.member invites don't contain this).
+        room.getMembersWithMembership("invite").forEach(function(member) {
+            if (member._requestedProfileInfo) return;
+            member._requestedProfileInfo = true;
+            // try to get a cached copy first.
+            const user = client.getUser(member.userId);
+            let promise;
+            if (user) {
+                promise = Promise.resolve({
+                    avatar_url: user.avatarUrl,
+                    displayname: user.displayName,
+                });
+            } else {
+                promise = client.getProfileInfo(member.userId);
+            }
+            promise.then(function(info) {
+                // slightly naughty by doctoring the invite event but this means all
+                // the code paths remain the same between invite/join display name stuff
+                // which is a worthy trade-off for some minor pollution.
+                const inviteEvent = member.events.member;
+                if (inviteEvent.getContent().membership !== "invite") {
+                    // between resolving and now they have since joined, so don't clobber
+                    return;
+                }
+                inviteEvent.getContent().avatar_url = info.avatar_url;
+                inviteEvent.getContent().displayname = info.displayname;
+                // fire listeners
+                member.setMembershipEvent(inviteEvent, room.currentState);
+            }, function(_err) {
+                // OH WELL.
+            });
+        });
+    }
+
+    public retryImmediately(): boolean {
+        return true;
+    }
+
+    /**
+     * Main entry point. Blocks until stop() is called.
+     */
+    public async sync() {
+        logger.debug("Sliding sync init loop");
+
+        //   1) We need to get push rules so we can check if events should bing as we get
+        //      them from /sync.
+        while (!this.client.isGuest()) {
+            try {
+                logger.debug("Getting push rules...");
+                const result = await this.client.getPushRules();
+                logger.debug("Got push rules");
+                this.client.pushRules = result;
+                break;
+            } catch (err) {
+                logger.error("Getting push rules failed", err);
+                if (this.shouldAbortSync(err)) {
+                    return;
+                }
+            }
+        }
+
+        // start syncing
+        await this.slidingSync.start();
+    }
+
+    /**
+     * Stops the sync object from syncing.
+     */
+    public stop(): void {
+        logger.debug("SyncApi.stop");
+        this.slidingSync.stop();
+    }
+
+    /**
+     * Sets the sync state and emits an event to say so
+     * @param {String} newState The new state string
+     * @param {Object} data Object of additional data to emit in the event
+     */
+    private updateSyncState(newState: SyncState, data?: ISyncStateData): void {
+        const old = this.syncState;
+        this.syncState = newState;
+        this.syncStateData = data;
+        this.client.emit(ClientEvent.Sync, this.syncState, old, data);
+    }
+
+    /**
+     * Takes a list of timelineEvents and adds and adds to notifEvents
+     * as appropriate.
+     * This must be called after the room the events belong to has been stored.
+     *
+     * @param {MatrixEvent[]} [timelineEventList] A list of timeline events. Lower index
+     * is earlier in time. Higher index is later.
+     */
+    private addNotifications(timelineEventList: MatrixEvent[]): void {
+        // gather our notifications into this.notifEvents
+        if (!this.client.getNotifTimelineSet()) {
+            return;
+        }
+        for (const timelineEvent of timelineEventList) {
+            const pushActions = this.client.getPushActionsForEvent(timelineEvent);
+            if (pushActions && pushActions.notify &&
+                pushActions.tweaks && pushActions.tweaks.highlight) {
+                this.notifEvents.push(timelineEvent);
+            }
+        }
+    }
+
+    /**
+     * Purge any events in the notifEvents array. Used after a /sync has been complete.
+     * This should not be called at a per-room scope (e.g in onRoomData) because otherwise the ordering
+     * will be messed up e.g room A gets a bing, room B gets a newer bing, but both in the same /sync
+     * response. If we purge at a per-room scope then we could process room B before room A leading to
+     * room B appearing earlier in the notifications timeline, even though it has the higher origin_server_ts.
+     */
+    private purgeNotifications(): void {
+        this.notifEvents.sort(function(a, b) {
+            return a.getTs() - b.getTs();
+        });
+        this.notifEvents.forEach((event) => {
+            this.client.getNotifTimelineSet().addLiveEvent(event);
+        });
+        this.notifEvents = [];
+    }
+}
+
+function ensureNameEvent(client: MatrixClient, roomId: string, roomData: MSC3575RoomData): MSC3575RoomData {
+    // make sure m.room.name is in required_state if there is a name, replacing anything previously
+    // there if need be. This ensures clients transparently 'calculate' the right room name. Native
+    // sliding sync clients should just read the "name" field.
+    if (!roomData.name) {
+        return roomData;
+    }
+    for (const stateEvent of roomData.required_state) {
+        if (stateEvent.type === EventType.RoomName && stateEvent.state_key === "") {
+            stateEvent.content = {
+                name: roomData.name,
+            };
+            return roomData;
+        }
+    }
+    roomData.required_state.push({
+        event_id: "$fake-sliding-sync-name-event-" + roomId,
+        state_key: "",
+        type: EventType.RoomName,
+        content: {
+            name: roomData.name,
+        },
+        sender: client.getUserId(),
+        origin_server_ts: new Date().getTime(),
+    });
+    return roomData;
+}
+
+// Helper functions which set up JS SDK structs are below and are identical to the sync v2 counterparts,
+// just outside the class.
+
+function createRoom(client: MatrixClient, roomId: string, opts: Partial<IStoredClientOpts>): Room { // XXX cargoculted from sync.ts
+    const { timelineSupport } = client;
+    const room = new Room(roomId, client, client.getUserId(), {
+        lazyLoadMembers: opts.lazyLoadMembers,
+        pendingEventOrdering: opts.pendingEventOrdering,
+        timelineSupport,
+    });
+    client.reEmitter.reEmit(room, [
+        RoomEvent.Name,
+        RoomEvent.Redaction,
+        RoomEvent.RedactionCancelled,
+        RoomEvent.Receipt,
+        RoomEvent.Tags,
+        RoomEvent.LocalEchoUpdated,
+        RoomEvent.AccountData,
+        RoomEvent.MyMembership,
+        RoomEvent.Timeline,
+        RoomEvent.TimelineReset,
+    ]);
+    registerStateListeners(client, room);
+    return room;
+}
+
+function registerStateListeners(client: MatrixClient, room: Room): void { // XXX cargoculted from sync.ts
+    // we need to also re-emit room state and room member events, so hook it up
+    // to the client now. We need to add a listener for RoomState.members in
+    // order to hook them correctly.
+    client.reEmitter.reEmit(room.currentState, [
+        RoomStateEvent.Events,
+        RoomStateEvent.Members,
+        RoomStateEvent.NewMember,
+        RoomStateEvent.Update,
+    ]);
+    room.currentState.on(RoomStateEvent.NewMember, function(event, state, member) {
+        member.user = client.getUser(member.userId);
+        client.reEmitter.reEmit(member, [
+            RoomMemberEvent.Name,
+            RoomMemberEvent.Typing,
+            RoomMemberEvent.PowerLevel,
+            RoomMemberEvent.Membership,
+        ]);
+    });
+}
+
+/*
+function deregisterStateListeners(room: Room): void { // XXX cargoculted from sync.ts
+    // could do with a better way of achieving this.
+    room.currentState.removeAllListeners(RoomStateEvent.Events);
+    room.currentState.removeAllListeners(RoomStateEvent.Members);
+    room.currentState.removeAllListeners(RoomStateEvent.NewMember);
+} */
+
+function mapEvents(client: MatrixClient, roomId: string, events: object[], decrypt = true): MatrixEvent[] {
+    const mapper = client.getEventMapper({ decrypt });
+    return (events as Array<IStrippedState | IRoomEvent | IStateEvent | IMinimalEvent>).map(function(e) {
+        e["room_id"] = roomId;
+        return mapper(e);
+    });
+}

src/sliding-sync.ts

@@ -0,0 +1,760 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { logger } from './logger';
+import { IAbortablePromise } from "./@types/partials";
+import { MatrixClient } from "./client";
+import { IRoomEvent, IStateEvent } from "./sync-accumulator";
+import { TypedEventEmitter } from "./models//typed-event-emitter";
+import { sleep } from "./utils";
+
+// /sync requests allow you to set a timeout= but the request may continue
+// beyond that and wedge forever, so we need to track how long we are willing
+// to keep open the connection. This constant is *ADDED* to the timeout= value
+// to determine the max time we're willing to wait.
+const BUFFER_PERIOD_MS = 10 * 1000;
+
+/**
+ * Represents a subscription to a room or set of rooms. Controls which events are returned.
+ */
+export interface MSC3575RoomSubscription {
+    required_state?: string[][];
+    timeline_limit?: number;
+}
+
+/**
+ * Controls which rooms are returned in a given list.
+ */
+export interface MSC3575Filter {
+    is_dm?: boolean;
+    is_encrypted?: boolean;
+    is_invite?: boolean;
+    is_tombstoned?: boolean;
+    room_name_like?: string;
+    room_types?: string[];
+    not_room_types?: string[];
+    spaces?: string[];
+}
+
+/**
+ * Represents a list subscription.
+ */
+export interface MSC3575List extends MSC3575RoomSubscription {
+    ranges: number[][];
+    sort?: string[];
+    filters?: MSC3575Filter;
+    slow_get_all_rooms?: boolean;
+}
+
+/**
+ * A complete Sliding Sync request.
+ */
+export interface MSC3575SlidingSyncRequest {
+    // json body params
+    lists?: MSC3575List[];
+    unsubscribe_rooms?: string[];
+    room_subscriptions?: Record<string, MSC3575RoomSubscription>;
+    extensions?: object;
+
+    // query params
+    pos?: string;
+    timeout?: number;
+    clientTimeout?: number;
+}
+
+export interface MSC3575RoomData {
+    name: string;
+    required_state: IStateEvent[];
+    timeline: (IRoomEvent | IStateEvent)[];
+    notification_count?: number;
+    highlight_count?: number;
+    invite_state?: IStateEvent[];
+    initial?: boolean;
+    limited?: boolean;
+    is_dm?: boolean;
+    prev_batch?: string;
+}
+
+interface ListResponse {
+    count: number;
+    ops: Operation[];
+}
+
+interface BaseOperation {
+    op: string;
+}
+
+interface DeleteOperation extends BaseOperation {
+    op: "DELETE";
+    index: number;
+}
+
+interface InsertOperation extends BaseOperation {
+    op: "INSERT";
+    index: number;
+    room_id: string;
+}
+
+interface InvalidateOperation extends BaseOperation {
+    op: "INVALIDATE";
+    range: [number, number];
+}
+
+interface SyncOperation extends BaseOperation {
+    op: "SYNC";
+    range: [number, number];
+    room_ids: string[];
+}
+
+type Operation = DeleteOperation | InsertOperation | InvalidateOperation | SyncOperation;
+
+/**
+ * A complete Sliding Sync response
+ */
+export interface MSC3575SlidingSyncResponse {
+    pos: string;
+    lists: ListResponse[];
+    rooms: Record<string, MSC3575RoomData>;
+    extensions: object;
+}
+
+export enum SlidingSyncState {
+    /**
+     * Fired by SlidingSyncEvent.Lifecycle event immediately before processing the response.
+     */
+    RequestFinished = "FINISHED",
+    /**
+     * Fired by SlidingSyncEvent.Lifecycle event immediately after all room data listeners have been
+     * invoked, but before list listeners.
+     */
+    Complete = "COMPLETE",
+}
+
+/**
+ * Internal Class. SlidingList represents a single list in sliding sync. The list can have filters,
+ * multiple sliding windows, and maintains the index->room_id mapping.
+ */
+class SlidingList {
+    private list: MSC3575List;
+    private isModified: boolean;
+
+    // returned data
+    public roomIndexToRoomId: Record<number, string>;
+    public joinedCount: number;
+
+    /**
+     * Construct a new sliding list.
+     * @param {MSC3575List} list The range, sort and filter values to use for this list.
+     */
+    constructor(list: MSC3575List) {
+        this.replaceList(list);
+    }
+
+    /**
+     * Mark this list as modified or not. Modified lists will return sticky params with calls to getList.
+     * This is useful for the first time the list is sent, or if the list has changed in some way.
+     * @param modified True to mark this list as modified so all sticky parameters will be re-sent.
+     */
+    public setModified(modified: boolean): void {
+        this.isModified = modified;
+    }
+
+    /**
+     * Update the list range for this list. Does not affect modified status as list ranges are non-sticky.
+     * @param newRanges The new ranges for the list
+     */
+    public updateListRange(newRanges: number[][]): void {
+        this.list.ranges = JSON.parse(JSON.stringify(newRanges));
+    }
+
+    /**
+     * Replace list parameters. All fields will be replaced with the new list parameters.
+     * @param list The new list parameters
+     */
+    public replaceList(list: MSC3575List): void {
+        list.filters = list.filters || {};
+        list.ranges = list.ranges || [];
+        this.list = JSON.parse(JSON.stringify(list));
+        this.isModified = true;
+
+        // reset values as the join count may be very different (if filters changed) including the rooms
+        // (e.g. sort orders or sliding window ranges changed)
+
+        // the constantly changing sliding window ranges. Not an array for performance reasons
+        // E.g. tracking ranges 0-99, 500-599, we don't want to have a 600 element array
+        this.roomIndexToRoomId = {};
+        // the total number of joined rooms according to the server, always >= len(roomIndexToRoomId)
+        this.joinedCount = 0;
+    }
+
+    /**
+     * Return a copy of the list suitable for a request body.
+     * @param {boolean} forceIncludeAllParams True to forcibly include all params even if the list
+     * hasn't been modified. Callers may want to do this if they are modifying the list prior to calling
+     * updateList.
+     */
+    public getList(forceIncludeAllParams: boolean): MSC3575List {
+        let list = {
+            ranges: JSON.parse(JSON.stringify(this.list.ranges)),
+        };
+        if (this.isModified || forceIncludeAllParams) {
+            list = JSON.parse(JSON.stringify(this.list));
+        }
+        return list;
+    }
+
+    /**
+     * Check if a given index is within the list range. This is required even though the /sync API
+     * provides explicit updates with index positions because of the following situation:
+     *   0 1 2 3 4 5 6 7 8   indexes
+     *   a b c       d e f   COMMANDS: SYNC 0 2 a b c; SYNC 6 8 d e f;
+     *   a b c       d _ f   COMMAND: DELETE 7;
+     *   e a b c       d f   COMMAND: INSERT 0 e;
+     *   c=3 is wrong as we are not tracking it, ergo we need to see if `i` is in range else drop it
+     * @param i The index to check
+     * @returns True if the index is within a sliding window
+     */
+    public isIndexInRange(i: number): boolean {
+        for (const r of this.list.ranges) {
+            if (r[0] <= i && i <= r[1]) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
+
+/**
+ * When onResponse extensions should be invoked: before or after processing the main response.
+ */
+export enum ExtensionState {
+    // Call onResponse before processing the response body. This is useful when your extension is
+    // preparing the ground for the response body e.g. processing to-device messages before the
+    // encrypted event arrives.
+    PreProcess = "ExtState.PreProcess",
+    // Call onResponse after processing the response body. This is useful when your extension is
+    // decorating data from the client, and you rely on MatrixClient.getRoom returning the Room object
+    // e.g. room account data.
+    PostProcess = "ExtState.PostProcess",
+}
+
+/**
+ * An interface that must be satisfied to register extensions
+ */
+export interface Extension {
+    /**
+     * The extension name to go under 'extensions' in the request body.
+     * @returns The JSON key.
+     */
+    name(): string;
+    /**
+     * A function which is called when the request JSON is being formed.
+     * Returns the data to insert under this key.
+     * @param isInitial True when this is part of the initial request (send sticky params)
+     * @returns The request JSON to send.
+     */
+    onRequest(isInitial: boolean): object;
+    /**
+     * A function which is called when there is response JSON under this extension.
+     * @param data The response JSON under the extension name.
+     */
+    onResponse(data: object);
+    /**
+     * Controls when onResponse should be called.
+     * @returns The state when it should be called.
+     */
+    when(): ExtensionState;
+}
+
+/**
+ * Events which can be fired by the SlidingSync class. These are designed to provide different levels
+ * of information when processing sync responses.
+ *  - RoomData: concerns rooms, useful for SlidingSyncSdk to update its knowledge of rooms.
+ *  - Lifecycle: concerns callbacks at various well-defined points in the sync process.
+ *  - List: concerns lists, useful for UI layers to re-render room lists.
+ * Specifically, the order of event invocation is:
+ *  - Lifecycle (state=RequestFinished)
+ *  - RoomData (N times)
+ *  - Lifecycle (state=Complete)
+ *  - List (at most once per list)
+ */
+export enum SlidingSyncEvent {
+    /**
+     * This event fires when there are updates for a room. Fired as and when rooms are encountered
+     * in the response.
+     */
+    RoomData = "SlidingSync.RoomData",
+    /**
+     * This event fires at various points in the /sync loop lifecycle.
+     *  - SlidingSyncState.RequestFinished: Fires after we receive a valid response but before the
+     * response has been processed. Perform any pre-process steps here. If there was a problem syncing,
+     * `err` will be set (e.g network errors).
+     *  - SlidingSyncState.Complete: Fires after all SlidingSyncEvent.RoomData have been fired but before
+     * SlidingSyncEvent.List.
+     */
+    Lifecycle = "SlidingSync.Lifecycle",
+    /**
+     * This event fires whenever there has been a change to this list index. It fires exactly once
+     * per list, even if there were multiple operations for the list.
+     * It fires AFTER Lifecycle and RoomData events.
+     */
+    List = "SlidingSync.List",
+}
+
+export type SlidingSyncEventHandlerMap = {
+    [SlidingSyncEvent.RoomData]: (roomId: string, roomData: MSC3575RoomData) => void;
+    [SlidingSyncEvent.Lifecycle]: (state: SlidingSyncState, resp: MSC3575SlidingSyncResponse, err: Error) => void;
+    [SlidingSyncEvent.List]: (
+        listIndex: number, joinedCount: number, roomIndexToRoomId: Record<number, string>,
+    ) => void;
+};
+
+/**
+ * SlidingSync is a high-level data structure which controls the majority of sliding sync.
+ * It has no hooks into JS SDK except for needing a MatrixClient to perform the HTTP request.
+ * This means this class (and everything it uses) can be used in isolation from JS SDK if needed.
+ * To hook this up with the JS SDK, you need to use SlidingSyncSdk.
+ */
+export class SlidingSync extends TypedEventEmitter<SlidingSyncEvent, SlidingSyncEventHandlerMap> {
+    private lists: SlidingList[];
+    private listModifiedCount = 0;
+    private terminated = false;
+    // flag set when resend() is called because we cannot rely on detecting AbortError in JS SDK :(
+    private needsResend = false;
+    // map of extension name to req/resp handler
+    private extensions: Record<string, Extension> = {};
+
+    private desiredRoomSubscriptions = new Set<string>(); // the *desired* room subscriptions
+    private confirmedRoomSubscriptions = new Set<string>();
+
+    private pendingReq?: IAbortablePromise<MSC3575SlidingSyncResponse>;
+
+    /**
+     * Create a new sliding sync instance
+     * @param {string} proxyBaseUrl The base URL of the sliding sync proxy
+     * @param {MSC3575List[]} lists The lists to use for sliding sync.
+     * @param {MSC3575RoomSubscription} roomSubscriptionInfo The params to use for room subscriptions.
+     * @param {MatrixClient} client The client to use for /sync calls.
+     * @param {number} timeoutMS The number of milliseconds to wait for a response.
+     */
+    constructor(
+        private readonly proxyBaseUrl: string,
+        lists: MSC3575List[],
+        private roomSubscriptionInfo: MSC3575RoomSubscription,
+        private readonly client: MatrixClient,
+        private readonly timeoutMS: number,
+    ) {
+        super();
+        this.lists = lists.map((l) => new SlidingList(l));
+    }
+
+    /**
+     * Get the length of the sliding lists.
+     * @returns The number of lists in the sync request
+     */
+    public listLength(): number {
+        return this.lists.length;
+    }
+
+    /**
+     * Get the room data for a list.
+     * @param index The list index
+     * @returns The list data which contains the rooms in this list
+     */
+    public getListData(index: number): {joinedCount: number, roomIndexToRoomId: Record<number, string>} {
+        if (!this.lists[index]) {
+            return null;
+        }
+        return {
+            joinedCount: this.lists[index].joinedCount,
+            roomIndexToRoomId: Object.assign({}, this.lists[index].roomIndexToRoomId),
+        };
+    }
+
+    /**
+     * Get the full list parameters for a list index. This function is provided for callers to use
+     * in conjunction with setList to update fields on an existing list.
+     * @param index The list index to get the list for.
+     * @returns A copy of the list or undefined.
+     */
+    public getList(index: number): MSC3575List {
+        if (!this.lists[index]) {
+            return null;
+        }
+        return this.lists[index].getList(true);
+    }
+
+    /**
+     * Set new ranges for an existing list. Calling this function when _only_ the ranges have changed
+     * is more efficient than calling setList(index,list) as this function won't resend sticky params,
+     * whereas setList always will.
+     * @param index The list index to modify
+     * @param ranges The new ranges to apply.
+     */
+    public setListRanges(index: number, ranges: number[][]): void {
+        this.lists[index].updateListRange(ranges);
+        this.resend();
+    }
+
+    /**
+     * Add or replace a list. Calling this function will interrupt the /sync request to resend new
+     * lists.
+     * @param index The index to modify
+     * @param list The new list parameters.
+     */
+    public setList(index: number, list: MSC3575List): void {
+        if (this.lists[index]) {
+            this.lists[index].replaceList(list);
+        } else {
+            this.lists[index] = new SlidingList(list);
+        }
+        this.listModifiedCount += 1;
+        this.resend();
+    }
+
+    /**
+     * Get the room subscriptions for the sync API.
+     * @returns A copy of the desired room subscriptions.
+     */
+    public getRoomSubscriptions(): Set<string> {
+        return new Set(Array.from(this.desiredRoomSubscriptions));
+    }
+
+    /**
+     * Modify the room subscriptions for the sync API. Calling this function will interrupt the
+     * /sync request to resend new subscriptions. If the /sync stream has not started, this will
+     * prepare the room subscriptions for when start() is called.
+     * @param s The new desired room subscriptions.
+     */
+    public modifyRoomSubscriptions(s: Set<string>) {
+        this.desiredRoomSubscriptions = s;
+        this.resend();
+    }
+
+    /**
+     * Modify which events to retrieve for room subscriptions. Invalidates all room subscriptions
+     * such that they will be sent up afresh.
+     * @param rs The new room subscription fields to fetch.
+     */
+    public modifyRoomSubscriptionInfo(rs: MSC3575RoomSubscription): void {
+        this.roomSubscriptionInfo = rs;
+        this.confirmedRoomSubscriptions = new Set<string>();
+        this.resend();
+    }
+
+    /**
+     * Register an extension to send with the /sync request.
+     * @param ext The extension to register.
+     */
+    public registerExtension(ext: Extension): void {
+        if (this.extensions[ext.name()]) {
+            throw new Error(`registerExtension: ${ext.name()} already exists as an extension`);
+        }
+        this.extensions[ext.name()] = ext;
+    }
+
+    private getExtensionRequest(isInitial: boolean): object {
+        const ext = {};
+        Object.keys(this.extensions).forEach((extName) => {
+            ext[extName] = this.extensions[extName].onRequest(isInitial);
+        });
+        return ext;
+    }
+
+    private onPreExtensionsResponse(ext: object): void {
+        Object.keys(ext).forEach((extName) => {
+            if (this.extensions[extName].when() == ExtensionState.PreProcess) {
+                this.extensions[extName].onResponse(ext[extName]);
+            }
+        });
+    }
+
+    private onPostExtensionsResponse(ext: object): void {
+        Object.keys(ext).forEach((extName) => {
+            if (this.extensions[extName].when() == ExtensionState.PostProcess) {
+                this.extensions[extName].onResponse(ext[extName]);
+            }
+        });
+    }
+
+    /**
+     * Invoke all attached room data listeners.
+     * @param {string} roomId The room which received some data.
+     * @param {object} roomData The raw sliding sync response JSON.
+     */
+    private invokeRoomDataListeners(roomId: string, roomData: MSC3575RoomData): void {
+        if (!roomData.required_state) { roomData.required_state = []; }
+        if (!roomData.timeline) { roomData.timeline = []; }
+        this.emit(SlidingSyncEvent.RoomData, roomId, roomData);
+    }
+
+    /**
+     * Invoke all attached lifecycle listeners.
+     * @param {SlidingSyncState} state The Lifecycle state
+     * @param {object} resp The raw sync response JSON
+     * @param {Error?} err Any error that occurred when making the request e.g. network errors.
+     */
+    private invokeLifecycleListeners(state: SlidingSyncState, resp: MSC3575SlidingSyncResponse, err?: Error): void {
+        this.emit(SlidingSyncEvent.Lifecycle, state, resp, err);
+    }
+
+    private processListOps(list: ListResponse, listIndex: number): void {
+        let gapIndex = -1;
+        list.ops.forEach((op: Operation) => {
+            switch (op.op) {
+                case "DELETE": {
+                    logger.debug("DELETE", listIndex, op.index, ";");
+                    delete this.lists[listIndex].roomIndexToRoomId[op.index];
+                    gapIndex = op.index;
+                    break;
+                }
+                case "INSERT": {
+                    logger.debug(
+                        "INSERT",
+                        listIndex,
+                        op.index,
+                        op.room_id,
+                        ";",
+                    );
+                    if (this.lists[listIndex].roomIndexToRoomId[op.index]) {
+                        // something is in this space, shift items out of the way
+                        if (gapIndex < 0) {
+                            logger.debug(
+                                "cannot work out where gap is, INSERT without previous DELETE! List: ",
+                                listIndex,
+                            );
+                            return;
+                        }
+                        //  0,1,2,3  index
+                        // [A,B,C,D]
+                        //   DEL 3
+                        // [A,B,C,_]
+                        //   INSERT E 0
+                        // [E,A,B,C]
+                        // gapIndex=3, op.index=0
+                        if (gapIndex > op.index) {
+                            // the gap is further down the list, shift every element to the right
+                            // starting at the gap so we can just shift each element in turn:
+                            // [A,B,C,_] gapIndex=3, op.index=0
+                            // [A,B,C,C] i=3
+                            // [A,B,B,C] i=2
+                            // [A,A,B,C] i=1
+                            // Terminate. We'll assign into op.index next.
+                            for (let i = gapIndex; i > op.index; i--) {
+                                if (this.lists[listIndex].isIndexInRange(i)) {
+                                    this.lists[listIndex].roomIndexToRoomId[i] =
+                                        this.lists[listIndex].roomIndexToRoomId[
+                                            i - 1
+                                        ];
+                                }
+                            }
+                        } else if (gapIndex < op.index) {
+                            // the gap is further up the list, shift every element to the left
+                            // starting at the gap so we can just shift each element in turn
+                            for (let i = gapIndex; i < op.index; i++) {
+                                if (this.lists[listIndex].isIndexInRange(i)) {
+                                    this.lists[listIndex].roomIndexToRoomId[i] =
+                                        this.lists[listIndex].roomIndexToRoomId[
+                                            i + 1
+                                        ];
+                                }
+                            }
+                        }
+                    }
+                    this.lists[listIndex].roomIndexToRoomId[op.index] = op.room_id;
+                    break;
+                }
+                case "INVALIDATE": {
+                    const startIndex = op.range[0];
+                    for (let i = startIndex; i <= op.range[1]; i++) {
+                        delete this.lists[listIndex].roomIndexToRoomId[i];
+                    }
+                    logger.debug(
+                        "INVALIDATE",
+                        listIndex,
+                        op.range[0],
+                        op.range[1],
+                        ";",
+                    );
+                    break;
+                }
+                case "SYNC": {
+                    const startIndex = op.range[0];
+                    for (let i = startIndex; i <= op.range[1]; i++) {
+                        const roomId = op.room_ids[i - startIndex];
+                        if (!roomId) {
+                            break; // we are at the end of list
+                        }
+                        this.lists[listIndex].roomIndexToRoomId[i] = roomId;
+                    }
+                    logger.debug(
+                        "SYNC",
+                        listIndex,
+                        op.range[0],
+                        op.range[1],
+                        (op.room_ids || []).join(" "),
+                        ";",
+                    );
+                    break;
+                }
+            }
+        });
+    }
+
+    /**
+     * Resend a Sliding Sync request. Used when something has changed in the request.
+     */
+    public resend(): void {
+        this.needsResend = true;
+        this.pendingReq?.abort();
+    }
+
+    /**
+     * Stop syncing with the server.
+     */
+    public stop(): void {
+        this.terminated = true;
+        this.pendingReq?.abort();
+        // remove all listeners so things can be GC'd
+        this.removeAllListeners(SlidingSyncEvent.Lifecycle);
+        this.removeAllListeners(SlidingSyncEvent.List);
+        this.removeAllListeners(SlidingSyncEvent.RoomData);
+    }
+
+    /**
+     * Start syncing with the server. Blocks until stopped.
+     */
+    public async start() {
+        let currentPos: string;
+        while (!this.terminated) {
+            this.needsResend = false;
+            let doNotUpdateList = false;
+            let resp: MSC3575SlidingSyncResponse;
+            try {
+                const listModifiedCount = this.listModifiedCount;
+                const reqBody: MSC3575SlidingSyncRequest = {
+                    lists: this.lists.map((l) => {
+                        return l.getList(false);
+                    }),
+                    pos: currentPos,
+                    timeout: this.timeoutMS,
+                    clientTimeout: this.timeoutMS + BUFFER_PERIOD_MS,
+                    extensions: this.getExtensionRequest(currentPos === undefined),
+                };
+                // check if we are (un)subscribing to a room and modify request this one time for it
+                const newSubscriptions = difference(this.desiredRoomSubscriptions, this.confirmedRoomSubscriptions);
+                const unsubscriptions = difference(this.confirmedRoomSubscriptions, this.desiredRoomSubscriptions);
+                if (unsubscriptions.size > 0) {
+                    reqBody.unsubscribe_rooms = Array.from(unsubscriptions);
+                }
+                if (newSubscriptions.size > 0) {
+                    reqBody.room_subscriptions = {};
+                    for (const roomId of newSubscriptions) {
+                        reqBody.room_subscriptions[roomId] = this.roomSubscriptionInfo;
+                    }
+                }
+                this.pendingReq = this.client.slidingSync(reqBody, this.proxyBaseUrl);
+                resp = await this.pendingReq;
+                logger.debug(resp);
+                currentPos = resp.pos;
+                // update what we think we're subscribed to.
+                for (const roomId of newSubscriptions) {
+                    this.confirmedRoomSubscriptions.add(roomId);
+                }
+                for (const roomId of unsubscriptions) {
+                    this.confirmedRoomSubscriptions.delete(roomId);
+                }
+                if (listModifiedCount !== this.listModifiedCount) {
+                    // the lists have been modified whilst we were waiting for 'await' to return, but the abort()
+                    // call did nothing. It is NOT SAFE to modify the list array now. We'll process the response but
+                    // not update list pointers.
+                    logger.debug("list modified during await call, not updating list");
+                    doNotUpdateList = true;
+                }
+                // mark all these lists as having been sent as sticky so we don't keep sending sticky params
+                this.lists.forEach((l) => {
+                    l.setModified(false);
+                });
+                // set default empty values so we don't need to null check
+                resp.lists = resp.lists || [];
+                resp.rooms = resp.rooms || {};
+                resp.extensions = resp.extensions || {};
+                resp.lists.forEach((val, i) => {
+                    this.lists[i].joinedCount = val.count;
+                });
+                this.invokeLifecycleListeners(
+                    SlidingSyncState.RequestFinished,
+                    resp,
+                );
+            } catch (err) {
+                if (err.httpStatus) {
+                    this.invokeLifecycleListeners(
+                        SlidingSyncState.RequestFinished,
+                        null,
+                        err,
+                    );
+                    await sleep(3000);
+                } else if (this.needsResend || err === "aborted") {
+                    // don't sleep as we caused this error by abort()ing the request.
+                    // we check for 'aborted' because that's the error Jest returns and without it
+                    // we get warnings about not exiting fast enough.
+                    continue;
+                } else {
+                    logger.error(err);
+                    await sleep(3000);
+                }
+            }
+            if (!resp) {
+                continue;
+            }
+            this.onPreExtensionsResponse(resp.extensions);
+
+            Object.keys(resp.rooms).forEach((roomId) => {
+                this.invokeRoomDataListeners(
+                    roomId,
+                    resp.rooms[roomId],
+                );
+            });
+
+            const listIndexesWithUpdates: Set<number> = new Set();
+            if (!doNotUpdateList) {
+                resp.lists.forEach((list, listIndex) => {
+                    list.ops = list.ops || [];
+                    if (list.ops.length > 0) {
+                        listIndexesWithUpdates.add(listIndex);
+                    }
+                    this.processListOps(list, listIndex);
+                });
+            }
+            this.invokeLifecycleListeners(SlidingSyncState.Complete, resp);
+            this.onPostExtensionsResponse(resp.extensions);
+            listIndexesWithUpdates.forEach((i) => {
+                this.emit(
+                    SlidingSyncEvent.List,
+                    i, this.lists[i].joinedCount, Object.assign({}, this.lists[i].roomIndexToRoomId),
+                );
+            });
+        }
+    }
+}
+
+const difference = (setA: Set<string>, setB: Set<string>): Set<string> => {
+    const diff = new Set(setA);
+    for (const elem of setB) {
+        diff.delete(elem);
+    }
+    return diff;
+};

src/store/index.ts

@@ -23,6 +23,7 @@ import { RoomSummary } from "../models/room-summary";
 import { IMinimalEvent, IRooms, ISyncResponse } from "../sync-accumulator";
 import { IStartClientOpts } from "../client";
 import { IStateEventWithRoomId } from "../@types/search";
+import { IndexedToDeviceBatch, ToDeviceBatchWithTxnId } from "../models/ToDeviceMessage";
 
 export interface ISavedSync {
     nextBatch: string;
@@ -31,8 +32,7 @@ export interface ISavedSync {
 }
 
 /**
- * Construct a stub store. This does no-ops on most store methods.
- * @constructor
+ * A store for most of the data js-sdk needs to store, apart from crypto data
  */
 export interface IStore {
     readonly accountData: Record<string, MatrixEvent>; // type : content
@@ -57,21 +57,21 @@ export interface IStore {
     setSyncToken(token: string): void;
 
     /**
-     * No-op.
-     * @param {Room} room
+     * Store the given room.
+     * @param {Room} room The room to be stored. All properties must be stored.
      */
     storeRoom(room: Room): void;
 
     /**
-     * No-op.
-     * @param {string} roomId
-     * @return {null}
+     * Retrieve a room by its' room ID.
+     * @param {string} roomId The room ID.
+     * @return {Room} The room or null.
      */
     getRoom(roomId: string): Room | null;
 
     /**
-     * No-op.
-     * @return {Array} An empty array.
+     * Retrieve all known rooms.
+     * @return {Room[]} A list of rooms, which may be empty.
      */
     getRooms(): Room[];
 
@@ -82,35 +82,36 @@ export interface IStore {
     removeRoom(roomId: string): void;
 
     /**
-     * No-op.
-     * @return {Array} An empty array.
+     * Retrieve a summary of all the rooms.
+     * @return {RoomSummary[]} A summary of each room.
      */
     getRoomSummaries(): RoomSummary[];
 
     /**
-     * No-op.
-     * @param {User} user
+     * Store a User.
+     * @param {User} user The user to store.
      */
     storeUser(user: User): void;
 
     /**
-     * No-op.
-     * @param {string} userId
-     * @return {null}
+     * Retrieve a User by its' user ID.
+     * @param {string} userId The user ID.
+     * @return {User} The user or null.
      */
     getUser(userId: string): User | null;
 
     /**
-     * No-op.
-     * @return {User[]}
+     * Retrieve all known users.
+     * @return {User[]} A list of users, which may be empty.
      */
     getUsers(): User[];
 
     /**
-     * No-op.
-     * @param {Room} room
-     * @param {number} limit
-     * @return {Array}
+     * Retrieve scrollback for this room.
+     * @param {Room} room The matrix room
+     * @param {number} limit The max number of old events to retrieve.
+     * @return {Array<Object>} An array of objects which will be at most 'limit'
+     * length and at least 0. The objects are the raw event JSON.
      */
     scrollback(room: Room, limit: number): MatrixEvent[];
 
@@ -209,8 +210,23 @@ export interface IStore {
      */
     deleteAllData(): Promise<void>;
 
+    /**
+     * Returns the out-of-band membership events for this room that
+     * were previously loaded.
+     * @param {string} roomId
+     * @returns {event[]} the events, potentially an empty array if OOB loading didn't yield any new members
+     * @returns {null} in case the members for this room haven't been stored yet
+     */
     getOutOfBandMembers(roomId: string): Promise<IStateEventWithRoomId[] | null>;
 
+    /**
+     * Stores the out-of-band membership events for this room. Note that
+     * it still makes sense to store an empty array as the OOB status for the room is
+     * marked as fetched, and getOutOfBandMembers will return an empty array instead of null
+     * @param {string} roomId
+     * @param {event[]} membershipEvents the membership events to store
+     * @returns {Promise} when all members have been stored
+     */
     setOutOfBandMembers(roomId: string, membershipEvents: IStateEventWithRoomId[]): Promise<void>;
 
     clearOutOfBandMembers(roomId: string): Promise<void>;
@@ -222,4 +238,19 @@ export interface IStore {
     getPendingEvents(roomId: string): Promise<Partial<IEvent>[]>;
 
     setPendingEvents(roomId: string, events: Partial<IEvent>[]): Promise<void>;
+
+    /**
+     * Stores batches of outgoing to-device messages
+     */
+     saveToDeviceBatches(batch: ToDeviceBatchWithTxnId[]): Promise<void>;
+
+     /**
+      * Fetches the oldest batch of to-device messages in the queue
+      */
+     getOldestToDeviceBatch(): Promise<IndexedToDeviceBatch>;
+
+     /**
+      * Removes a specific batch of to-device messages from the queue
+      */
+     removeToDeviceBatch(id: number): Promise<void>;
 }

src/store/indexeddb-backend.ts

@@ -16,6 +16,7 @@ limitations under the License.
 
 import { ISavedSync } from "./index";
 import { IEvent, IStartClientOpts, IStateEventWithRoomId, ISyncResponse } from "..";
+import { IndexedToDeviceBatch, ToDeviceBatchWithTxnId } from "../models/ToDeviceMessage";
 
 export interface IIndexedDBBackend {
     connect(): Promise<void>;
@@ -31,6 +32,9 @@ export interface IIndexedDBBackend {
     getUserPresenceEvents(): Promise<UserTuple[]>;
     getClientOptions(): Promise<IStartClientOpts>;
     storeClientOptions(options: IStartClientOpts): Promise<void>;
+    saveToDeviceBatches(batches: ToDeviceBatchWithTxnId[]): Promise<void>;
+    getOldestToDeviceBatch(): Promise<IndexedToDeviceBatch>;
+    removeToDeviceBatch(id: number): Promise<void>;
 }
 
 export type UserTuple = [userId: string, presenceEvent: Partial<IEvent>];

src/store/indexeddb-local-backend.ts

@@ -21,8 +21,9 @@ import { logger } from '../logger';
 import { IStartClientOpts, IStateEventWithRoomId } from "..";
 import { ISavedSync } from "./index";
 import { IIndexedDBBackend, UserTuple } from "./indexeddb-backend";
+import { IndexedToDeviceBatch, ToDeviceBatchWithTxnId } from "../models/ToDeviceMessage";
 
-const VERSION = 3;
+const VERSION = 4;
 
 function createDatabase(db: IDBDatabase): void {
     // Make user store, clobber based on user ID. (userId property of User objects)
@@ -49,6 +50,10 @@ function upgradeSchemaV3(db: IDBDatabase): void {
         { keyPath: ["clobber"] });
 }
 
+function upgradeSchemaV4(db: IDBDatabase): void {
+    db.createObjectStore("to_device_queue", { autoIncrement: true });
+}
+
 /**
  * Helper method to collect results from a Cursor and promiseify it.
  * @param {ObjectStore|Index} store The store to perform openCursor on.
@@ -112,7 +117,7 @@ function reqAsPromise(req: IDBRequest): Promise<IDBRequest> {
     });
 }
 
-function reqAsCursorPromise(req: IDBRequest<IDBCursor | null>): Promise<IDBCursor> {
+function reqAsCursorPromise<T>(req: IDBRequest<T>): Promise<T> {
     return reqAsEventPromise(req).then((event) => req.result);
 }
 
@@ -177,6 +182,9 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend {
             if (oldVersion < 3) {
                 upgradeSchemaV3(db);
             }
+            if (oldVersion < 4) {
+                upgradeSchemaV4(db);
+            }
             // Expand as needed.
         };
 
@@ -401,8 +409,6 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend {
     }
 
     public async syncToDatabase(userTuples: UserTuple[]): Promise<void> {
-        const syncData = this.syncAccumulator.getJSON(true);
-
         if (this.isPersisting) {
             logger.warn("Skipping syncToDatabase() as persist already in flight");
             this.pendingUserPresenceData.push(...userTuples);
@@ -413,6 +419,8 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend {
         }
 
         try {
+            const syncData = this.syncAccumulator.getJSON(true);
+
             await Promise.all([
                 this.persistUserPresenceEvents(userTuples),
                 this.persistAccountData(syncData.accountData),
@@ -561,4 +569,36 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend {
         }); // put == UPSERT
         await txnAsPromise(txn);
     }
+
+    public async saveToDeviceBatches(batches: ToDeviceBatchWithTxnId[]): Promise<void> {
+        const txn = this.db.transaction(["to_device_queue"], "readwrite");
+        const store = txn.objectStore("to_device_queue");
+        for (const batch of batches) {
+            store.add(batch);
+        }
+        await txnAsPromise(txn);
+    }
+
+    public async getOldestToDeviceBatch(): Promise<IndexedToDeviceBatch | null> {
+        const txn = this.db.transaction(["to_device_queue"], "readonly");
+        const store = txn.objectStore("to_device_queue");
+        const cursor = await reqAsCursorPromise(store.openCursor());
+        if (!cursor) return null;
+
+        const resultBatch = cursor.value as ToDeviceBatchWithTxnId;
+
+        return {
+            id: cursor.key as number,
+            txnId: resultBatch.txnId,
+            eventType: resultBatch.eventType,
+            batch: resultBatch.batch,
+        };
+    }
+
+    public async removeToDeviceBatch(id: number): Promise<void> {
+        const txn = this.db.transaction(["to_device_queue"], "readwrite");
+        const store = txn.objectStore("to_device_queue");
+        store.delete(id);
+        await txnAsPromise(txn);
+    }
 }

src/store/indexeddb-remote-backend.ts

@@ -20,6 +20,7 @@ import { ISavedSync } from "./index";
 import { IStartClientOpts } from "../client";
 import { IStateEventWithRoomId, ISyncResponse } from "..";
 import { IIndexedDBBackend, UserTuple } from "./indexeddb-backend";
+import { IndexedToDeviceBatch, ToDeviceBatchWithTxnId } from "../models/ToDeviceMessage";
 
 export class RemoteIndexedDBStoreBackend implements IIndexedDBBackend {
     private worker: Worker;
@@ -133,6 +134,18 @@ export class RemoteIndexedDBStoreBackend implements IIndexedDBBackend {
         return this.doCmd('getUserPresenceEvents');
     }
 
+    public async saveToDeviceBatches(batches: ToDeviceBatchWithTxnId[]): Promise<void> {
+        return this.doCmd('saveToDeviceBatches', [batches]);
+    }
+
+    public async getOldestToDeviceBatch(): Promise<IndexedToDeviceBatch> {
+        return this.doCmd('getOldestToDeviceBatch');
+    }
+
+    public async removeToDeviceBatch(id: number): Promise<void> {
+        return this.doCmd('removeToDeviceBatch', [id]);
+    }
+
     private ensureStarted(): Promise<void> {
         if (this.startPromise === null) {
             this.worker = this.workerFactory();

src/store/indexeddb-store-worker.ts

@@ -103,6 +103,15 @@ export class IndexedDBStoreWorker {
             case 'storeClientOptions':
                 prom = this.backend.storeClientOptions(msg.args[0]);
                 break;
+            case 'saveToDeviceBatches':
+                prom = this.backend.saveToDeviceBatches(msg.args[0]);
+                break;
+            case 'getOldestToDeviceBatch':
+                prom = this.backend.getOldestToDeviceBatch();
+                break;
+            case 'removeToDeviceBatch':
+                prom = this.backend.removeToDeviceBatch(msg.args[0]);
+                break;
         }
 
         if (prom === undefined) {

src/store/indexeddb.ts

@@ -27,6 +27,7 @@ import { IIndexedDBBackend } from "./indexeddb-backend";
 import { ISyncResponse } from "../sync-accumulator";
 import { TypedEventEmitter } from "../models/typed-event-emitter";
 import { IStateEventWithRoomId } from "../@types/search";
+import { IndexedToDeviceBatch, ToDeviceBatchWithTxnId } from "../models/ToDeviceMessage";
 
 /**
  * This is an internal module. See {@link IndexedDBStore} for the public class.
@@ -351,6 +352,18 @@ export class IndexedDBStore extends MemoryStore {
             this.localStorage.removeItem(pendingEventsKey(roomId));
         }
     }
+
+    public saveToDeviceBatches(batches: ToDeviceBatchWithTxnId[]): Promise<void> {
+        return this.backend.saveToDeviceBatches(batches);
+    }
+
+    public getOldestToDeviceBatch(): Promise<IndexedToDeviceBatch> {
+        return this.backend.getOldestToDeviceBatch();
+    }
+
+    public removeToDeviceBatch(id: number): Promise<void> {
+        return this.backend.removeToDeviceBatch(id);
+    }
 }
 
 /**

src/store/memory.ts

@@ -30,6 +30,7 @@ import { ISavedSync, IStore } from "./index";
 import { RoomSummary } from "../models/room-summary";
 import { ISyncResponse } from "../sync-accumulator";
 import { IStateEventWithRoomId } from "../@types/search";
+import { IndexedToDeviceBatch, ToDeviceBatchWithTxnId } from "../models/ToDeviceMessage";
 
 function isValidFilterId(filterId: string): boolean {
     const isValidStr = typeof filterId === "string" &&
@@ -64,6 +65,8 @@ export class MemoryStore implements IStore {
     private oobMembers: Record<string, IStateEventWithRoomId[]> = {}; // roomId: [member events]
     private pendingEvents: { [roomId: string]: Partial<IEvent>[] } = {};
     private clientOptions = {};
+    private pendingToDeviceBatches: IndexedToDeviceBatch[] = [];
+    private nextToDeviceBatchId = 0;
 
     constructor(opts: IOpts = {}) {
         this.localStorage = opts.localStorage;
@@ -429,4 +432,26 @@ export class MemoryStore implements IStore {
     public async setPendingEvents(roomId: string, events: Partial<IEvent>[]): Promise<void> {
         this.pendingEvents[roomId] = events;
     }
+
+    public saveToDeviceBatches(batches: ToDeviceBatchWithTxnId[]): Promise<void> {
+        for (const batch of batches) {
+            this.pendingToDeviceBatches.push({
+                id: this.nextToDeviceBatchId++,
+                eventType: batch.eventType,
+                txnId: batch.txnId,
+                batch: batch.batch,
+            });
+        }
+        return Promise.resolve();
+    }
+
+    public async getOldestToDeviceBatch(): Promise<IndexedToDeviceBatch | null> {
+        if (this.pendingToDeviceBatches.length === 0) return null;
+        return this.pendingToDeviceBatches[0];
+    }
+
+    public removeToDeviceBatch(id: number): Promise<void> {
+        this.pendingToDeviceBatches = this.pendingToDeviceBatches.filter(batch => batch.id !== id);
+        return Promise.resolve();
+    }
 }

src/store/stub.ts

@@ -28,6 +28,7 @@ import { ISavedSync, IStore } from "./index";
 import { RoomSummary } from "../models/room-summary";
 import { ISyncResponse } from "../sync-accumulator";
 import { IStateEventWithRoomId } from "../@types/search";
+import { IndexedToDeviceBatch, ToDeviceBatch } from "../models/ToDeviceMessage";
 
 /**
  * Construct a stub store. This does no-ops on most store methods.
@@ -270,4 +271,16 @@ export class StubStore implements IStore {
     public setPendingEvents(roomId: string, events: Partial<IEvent>[]): Promise<void> {
         return Promise.resolve();
     }
+
+    public async saveToDeviceBatches(batch: ToDeviceBatch[]): Promise<void> {
+        return Promise.resolve();
+    }
+
+    public getOldestToDeviceBatch(): Promise<IndexedToDeviceBatch | null> {
+        return Promise.resolve(null);
+    }
+
+    public async removeToDeviceBatch(id: number): Promise<void> {
+        return Promise.resolve();
+    }
 }

src/sync-accumulator.ts

@@ -20,7 +20,7 @@ limitations under the License.
  */
 
 import { logger } from './logger';
-import { deepCopy } from "./utils";
+import { deepCopy, isSupportedReceiptType } from "./utils";
 import { IContent, IUnsigned } from "./models/event";
 import { IRoomSummary } from "./models/room-summary";
 import { EventType } from "./@types/event";
@@ -417,31 +417,18 @@ export class SyncAccumulator {
                 // of a hassle to work with. We'll inflate this back out when
                 // getJSON() is called.
                 Object.keys(e.content).forEach((eventId) => {
-                    if (!e.content[eventId][ReceiptType.Read] && !e.content[eventId][ReceiptType.ReadPrivate]) {
-                        return;
-                    }
-                    const read = e.content[eventId][ReceiptType.Read];
-                    if (read) {
-                        Object.keys(read).forEach((userId) => {
+                    Object.entries(e.content[eventId]).forEach(([key, value]) => {
+                        if (!isSupportedReceiptType(key)) return;
+
+                        Object.keys(value).forEach((userId) => {
                             // clobber on user ID
                             currentData._readReceipts[userId] = {
-                                data: e.content[eventId][ReceiptType.Read][userId],
-                                type: ReceiptType.Read,
+                                data: e.content[eventId][key][userId],
+                                type: key as ReceiptType,
                                 eventId: eventId,
                             };
                         });
-                    }
-                    const readPrivate = e.content[eventId][ReceiptType.ReadPrivate];
-                    if (readPrivate) {
-                        Object.keys(readPrivate).forEach((userId) => {
-                            // clobber on user ID
-                            currentData._readReceipts[userId] = {
-                                data: e.content[eventId][ReceiptType.ReadPrivate][userId],
-                                type: ReceiptType.ReadPrivate,
-                                eventId: eventId,
-                            };
-                        });
-                    }
+                    });
                 });
             });
         }

src/utils.ts

@@ -24,8 +24,9 @@ import unhomoglyph from "unhomoglyph";
 import promiseRetry from "p-retry";
 
 import type * as NodeCrypto from "crypto";
-import { MatrixEvent } from ".";
+import { MatrixClient, MatrixEvent } from ".";
 import { M_TIMESTAMP } from "./@types/location";
+import { ReceiptType } from "./@types/read_receipts";
 
 /**
  * Encode a dictionary of query parameters.
@@ -144,23 +145,6 @@ export function checkObjectHasKeys(obj: object, keys: string[]) {
     }
 }
 
-/**
- * Checks that the given object has no extra keys other than the specified ones.
- * @param {Object} obj The object to check.
- * @param {string[]} allowedKeys The list of allowed key names.
- * @throws If there are extra keys.
- */
-export function checkObjectHasNoAdditionalKeys(obj: object, allowedKeys: string[]): void {
-    for (const key in obj) {
-        if (!obj.hasOwnProperty(key)) {
-            continue;
-        }
-        if (allowedKeys.indexOf(key) === -1) {
-            throw new Error("Unknown key: " + key);
-        }
-    }
-}
-
 /**
  * Deep copy the given object. The object MUST NOT have circular references and
  * MUST NOT have functions.
@@ -283,69 +267,6 @@ export function deepSortedObjectEntries(obj: any): [string, any][] {
     return pairs;
 }
 
-/**
- * Inherit the prototype methods from one constructor into another. This is a
- * port of the Node.js implementation with an Object.create polyfill.
- *
- * @param {function} ctor Constructor function which needs to inherit the
- *     prototype.
- * @param {function} superCtor Constructor function to inherit prototype from.
- */
-export function inherits(ctor: Function, superCtor: Function) {
-    // Add util.inherits from Node.js
-    // Source:
-    // https://github.com/joyent/node/blob/master/lib/util.js
-    // Copyright Joyent, Inc. and other Node contributors.
-    //
-    // Permission is hereby granted, free of charge, to any person obtaining a
-    // copy of this software and associated documentation files (the
-    // "Software"), to deal in the Software without restriction, including
-    // without limitation the rights to use, copy, modify, merge, publish,
-    // distribute, sublicense, and/or sell copies of the Software, and to permit
-    // persons to whom the Software is furnished to do so, subject to the
-    // following conditions:
-    //
-    // The above copyright notice and this permission notice shall be included
-    // in all copies or substantial portions of the Software.
-    //
-    // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-    // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-    // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
-    // NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-    // DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
-    // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
-    // USE OR OTHER DEALINGS IN THE SOFTWARE.
-    (ctor as any).super_ = superCtor;
-    ctor.prototype = Object.create(superCtor.prototype, {
-        constructor: {
-            value: ctor,
-            enumerable: false,
-            writable: true,
-            configurable: true,
-        },
-    });
-}
-
-/**
- * Polyfills inheritance for prototypes by allowing different kinds of
- * super types. Typically prototypes would use `SuperType.call(this, params)`
- * though this doesn't always work in some environments - this function
- * falls back to using `Object.assign()` to clone a constructed copy
- * of the super type onto `thisArg`.
- * @param {any} thisArg The child instance. Modified in place.
- * @param {any} SuperType The type to act as a super instance
- * @param {any} params Arguments to supply to the super type's constructor
- */
-export function polyfillSuper(thisArg: any, SuperType: any, ...params: any[]) {
-    try {
-        SuperType.call(thisArg, ...params);
-    } catch (e) {
-        // fall back to Object.assign to just clone the thing
-        const fakeSuper = new SuperType(...params);
-        Object.assign(thisArg, fakeSuper);
-    }
-}
-
 /**
  * Returns whether the given value is a finite number without type-coercion
  *
@@ -728,3 +649,18 @@ function getContentTimestampWithFallback(event: MatrixEvent): number {
 export function sortEventsByLatestContentTimestamp(left: MatrixEvent, right: MatrixEvent): number {
     return getContentTimestampWithFallback(right) - getContentTimestampWithFallback(left);
 }
+
+export async function getPrivateReadReceiptField(client: MatrixClient): Promise<ReceiptType | null> {
+    if (await client.doesServerSupportUnstableFeature("org.matrix.msc2285.stable")) return ReceiptType.ReadPrivate;
+    if (await client.doesServerSupportUnstableFeature("org.matrix.msc2285")) return ReceiptType.UnstableReadPrivate;
+    return null;
+}
+
+export function isSupportedReceiptType(receiptType: string): boolean {
+    return [
+        ReceiptType.Read,
+        ReceiptType.ReadPrivate,
+        ReceiptType.UnstableReadPrivate,
+    ].includes(receiptType as ReceiptType);
+}
+

src/webrtc/call.ts

@@ -523,24 +523,22 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
             return;
         }
 
-        // Try to find a feed with the same purpose as the new stream,
-        // if we find it replace the old stream with the new one
-        const existingFeed = this.getRemoteFeeds().find((feed) => feed.purpose === purpose);
-        if (existingFeed) {
-            existingFeed.setNewStream(stream);
-        } else {
-            this.feeds.push(new CallFeed({
-                client: this.client,
-                roomId: this.roomId,
-                userId,
-                stream,
-                purpose,
-                audioMuted,
-                videoMuted,
-            }));
-            this.emit(CallEvent.FeedsChanged, this.feeds);
+        if (this.getFeedByStreamId(stream.id)) {
+            logger.warn(`Ignoring stream with id ${stream.id} because we already have a feed for it`);
+            return;
         }
 
+        this.feeds.push(new CallFeed({
+            client: this.client,
+            roomId: this.roomId,
+            userId,
+            stream,
+            purpose,
+            audioMuted,
+            videoMuted,
+        }));
+        this.emit(CallEvent.FeedsChanged, this.feeds);
+
         logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}", purpose=${purpose})`);
     }
 
@@ -562,24 +560,22 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
             return;
         }
 
-        // Try to find a feed with the same stream id as the new stream,
-        // if we find it replace the old stream with the new one
-        const feed = this.getFeedByStreamId(stream.id);
-        if (feed) {
-            feed.setNewStream(stream);
-        } else {
-            this.feeds.push(new CallFeed({
-                client: this.client,
-                roomId: this.roomId,
-                audioMuted: false,
-                videoMuted: false,
-                userId,
-                stream,
-                purpose,
-            }));
-            this.emit(CallEvent.FeedsChanged, this.feeds);
+        if (this.getFeedByStreamId(stream.id)) {
+            logger.warn(`Ignoring stream with id ${stream.id} because we already have a feed for it`);
+            return;
         }
 
+        this.feeds.push(new CallFeed({
+            client: this.client,
+            roomId: this.roomId,
+            audioMuted: false,
+            videoMuted: false,
+            userId,
+            stream,
+            purpose,
+        }));
+        this.emit(CallEvent.FeedsChanged, this.feeds);
+
         logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`);
     }
 
@@ -592,25 +588,23 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
         setTracksEnabled(stream.getAudioTracks(), true);
         setTracksEnabled(stream.getVideoTracks(), true);
 
-        // We try to replace an existing feed if there already is one with the same purpose
-        const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose);
-        if (existingFeed) {
-            existingFeed.setNewStream(stream);
-        } else {
-            this.pushLocalFeed(
-                new CallFeed({
-                    client: this.client,
-                    roomId: this.roomId,
-                    audioMuted: false,
-                    videoMuted: false,
-                    userId,
-                    stream,
-                    purpose,
-                }),
-                addToPeerConnection,
-            );
-            this.emit(CallEvent.FeedsChanged, this.feeds);
+        if (this.getFeedByStreamId(stream.id)) {
+            logger.warn(`Ignoring stream with id ${stream.id} because we already have a feed for it`);
+            return;
         }
+
+        this.pushLocalFeed(
+            new CallFeed({
+                client: this.client,
+                roomId: this.roomId,
+                audioMuted: false,
+                videoMuted: false,
+                userId,
+                stream,
+                purpose,
+            }),
+            addToPeerConnection,
+        );
     }
 
     /**
@@ -709,7 +703,7 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
         const statsReport = await this.peerConn.getStats();
         const stats = [];
         statsReport.forEach(item => {
-            stats.push(item[1]);
+            stats.push(item);
         });
 
         return stats;
@@ -1142,7 +1136,7 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
             await this.upgradeCall(false, true);
             return this.isLocalVideoMuted();
         }
-        this.localUsermediaFeed?.setVideoMuted(muted);
+        this.localUsermediaFeed?.setAudioVideoMuted(null, muted);
         this.updateMuteStatus();
         return this.isLocalVideoMuted();
     }
@@ -1174,7 +1168,7 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
             await this.upgradeCall(true, false);
             return this.isMicrophoneMuted();
         }
-        this.localUsermediaFeed?.setAudioMuted(muted);
+        this.localUsermediaFeed?.setAudioVideoMuted(muted, null);
         this.updateMuteStatus();
         return this.isMicrophoneMuted();
     }
@@ -1518,7 +1512,7 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
         if (selectedPartyId !== this.ourPartyId) {
             logger.info(`Got select_answer for party ID ${selectedPartyId}: we are party ID ${this.ourPartyId}.`);
             // The other party has picked somebody else's answer
-            this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true);
+            await this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true);
         }
     }
 
@@ -1585,8 +1579,9 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
         this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true);
         for (const feed of this.getRemoteFeeds()) {
             const streamId = feed.stream.id;
-            feed.setAudioMuted(this.remoteSDPStreamMetadata[streamId]?.audio_muted);
-            feed.setVideoMuted(this.remoteSDPStreamMetadata[streamId]?.video_muted);
+            const metadata = this.remoteSDPStreamMetadata[streamId];
+
+            feed.setAudioVideoMuted(metadata?.audio_muted, metadata?.video_muted);
             feed.purpose = this.remoteSDPStreamMetadata[streamId]?.purpose;
         }
     }

src/webrtc/callFeed.ts

@@ -175,30 +175,18 @@ export class CallFeed extends TypedEventEmitter<CallFeedEvent, EventHandlerMap>
     }
 
     /**
-     * Replaces the current MediaStream with a new one.
-     * This method should be only used by MatrixCall.
-     * @param newStream new stream with which to replace the current one
-     */
-    public setNewStream(newStream: MediaStream): void {
-        this.updateStream(this.stream, newStream);
-    }
-
-    /**
-     * Set feed's internal audio mute state
-     * @param muted is the feed's audio muted?
-     */
-    public setAudioMuted(muted: boolean): void {
-        this.audioMuted = muted;
-        this.speakingVolumeSamples.fill(-Infinity);
-        this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
-    }
-
-    /**
-     * Set feed's internal video mute state
+     * Set one or both of feed's internal audio and video video mute state
+     * Either value may be null to leave it as-is
      * @param muted is the feed's video muted?
      */
-    public setVideoMuted(muted: boolean): void {
-        this.videoMuted = muted;
+    public setAudioVideoMuted(audioMuted: boolean, videoMuted: boolean): void {
+        if (audioMuted !== null) {
+            if (this.audioMuted !== audioMuted) {
+                this.speakingVolumeSamples.fill(-Infinity);
+            }
+            this.audioMuted = audioMuted;
+        }
+        if (videoMuted !== null) this.videoMuted = videoMuted;
         this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
     }