v6.0.0

Add progress callback for key backups

CHANGELOG.md

@@ -1,3 +1,98 @@
+Changes in [6.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0) (2020-05-05)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.0.0-rc.2...v6.0.0)
+
+ * Add progress callback for key backups
+   [\#1368](https://github.com/matrix-org/matrix-js-sdk/pull/1368)
+
+Changes in [6.0.0-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0-rc.2) (2020-05-01)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.0.0-rc.1...v6.0.0-rc.2)
+
+ * Emit event when a trusted self-key is stored
+   [\#1365](https://github.com/matrix-org/matrix-js-sdk/pull/1365)
+
+Changes in [6.0.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0-rc.1) (2020-04-30)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.4...v6.0.0-rc.1)
+
+BREAKING CHANGES
+---
+
+ * client.getStoredDevicesForUser and client.getStoredDevices are no longer async
+
+All Changes
+---
+
+ * Add initialFetch param to willUpdateDevices / devicesUpdated
+   [\#1362](https://github.com/matrix-org/matrix-js-sdk/pull/1362)
+ * Fix race between sending .request and receiving .ready over to_device
+   [\#1361](https://github.com/matrix-org/matrix-js-sdk/pull/1361)
+ * Handle race between sending and await next event from other party
+   [\#1358](https://github.com/matrix-org/matrix-js-sdk/pull/1358)
+ * Add crypto.willUpdateDevices event and make
+   getStoredDevices/getStoredDevicesForUser synchronous
+   [\#1356](https://github.com/matrix-org/matrix-js-sdk/pull/1356)
+ * Remove redundant key backup setup path
+   [\#1355](https://github.com/matrix-org/matrix-js-sdk/pull/1355)
+
+Changes in [5.3.1-rc.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.4) (2020-04-23)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.3...v5.3.1-rc.4)
+
+ * Retry account data upload requests
+   [\#1347](https://github.com/matrix-org/matrix-js-sdk/pull/1347)
+ * Fix: handle filter not found
+   [\#1341](https://github.com/matrix-org/matrix-js-sdk/pull/1341)
+ * Make getAccountDataFromServer return null if not found
+   [\#1339](https://github.com/matrix-org/matrix-js-sdk/pull/1339)
+ * Fix setDefaultKeyId to fail if the request fails
+   [\#1337](https://github.com/matrix-org/matrix-js-sdk/pull/1337)
+ * Fix: don't do extra /filter request when enabling lazy loading of members
+   [\#1333](https://github.com/matrix-org/matrix-js-sdk/pull/1333)
+ * Reject attemptAuth promise if no auth flow found
+   [\#1331](https://github.com/matrix-org/matrix-js-sdk/pull/1331)
+ * Serialise Olm prekey decryptions
+   [\#1330](https://github.com/matrix-org/matrix-js-sdk/pull/1330)
+
+Changes in [5.3.1-rc.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.3) (2020-04-17)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.2...v5.3.1-rc.3)
+
+ * Fix cross-signing/SSSS reset
+   [\#1323](https://github.com/matrix-org/matrix-js-sdk/pull/1323)
+ * Fix: crash when backup key needs fixing from corruption issue
+   [\#1325](https://github.com/matrix-org/matrix-js-sdk/pull/1325)
+
+Changes in [5.3.1-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.2) (2020-04-16)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.1...v5.3.1-rc.2)
+
+ * Implement QR code reciprocate for self-verification with untrusted MSK
+   [\#1321](https://github.com/matrix-org/matrix-js-sdk/pull/1321)
+
+Changes in [5.3.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.1) (2020-04-15)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.0-rc.1...v5.3.1-rc.1)
+
+ * Adapt release script for riot-desktop
+   [\#1319](https://github.com/matrix-org/matrix-js-sdk/pull/1319)
+ * Fix: prevent spurious notifications from indexer
+   [\#1318](https://github.com/matrix-org/matrix-js-sdk/pull/1318)
+ * Always create our own user object
+   [\#1317](https://github.com/matrix-org/matrix-js-sdk/pull/1317)
+ * Fix incorrect backup key format in SSSS
+   [\#1311](https://github.com/matrix-org/matrix-js-sdk/pull/1311)
+ * Fix e2ee crash after refreshing after having received a cross-singing key
+   reset
+   [\#1315](https://github.com/matrix-org/matrix-js-sdk/pull/1315)
+ * Fix: catch send errors in SAS verifier
+   [\#1314](https://github.com/matrix-org/matrix-js-sdk/pull/1314)
+ * Clear cross-signing keys when detecting the keys have changed
+   [\#1312](https://github.com/matrix-org/matrix-js-sdk/pull/1312)
+ * Upgrade deps
+   [\#1310](https://github.com/matrix-org/matrix-js-sdk/pull/1310)
+
 Changes in [5.3.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.0-rc.1) (2020-04-08)
 ==========================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.2.0...v5.3.0-rc.1)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-js-sdk",
-  "version": "5.3.0-rc.1",
+  "version": "6.0.0",
   "description": "Matrix Client-Server SDK for Javascript",
   "scripts": {
     "prepare": "yarn build",

release.sh

@@ -38,6 +38,7 @@ $USAGE
     -c changelog_file:  specify name of file containing changelog
     -x:                 skip updating the changelog
     -z:                 skip generating the jsdoc
+    -n:                 skip publish to NPM
 EOF
 }
 
@@ -60,9 +61,10 @@ fi
 
 skip_changelog=
 skip_jsdoc=
+skip_npm=
 changelog_file="CHANGELOG.md"
 expected_npm_user="matrixdotorg"
-while getopts hc:u:xz f; do
+while getopts hc:u:xzn f; do
     case $f in
         h)
             help
@@ -77,6 +79,9 @@ while getopts hc:u:xz f; do
         z)
             skip_jsdoc=1
             ;;
+        n)
+            skip_npm=1
+            ;;
         u)
             expected_npm_user="$OPTARG"
             ;;
@@ -96,10 +101,12 @@ fi
 
 # Login and publish continues to use `npm`, as it seems to have more clearly
 # defined options and semantics than `yarn` for writing to the registry.
-actual_npm_user=`npm whoami`;
-if [ $expected_npm_user != $actual_npm_user ]; then
-    echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
-    exit 1
+if [ -z "$skip_npm" ]; then
+    actual_npm_user=`npm whoami`;
+    if [ $expected_npm_user != $actual_npm_user ]; then
+        echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
+        exit 1
+    fi
 fi
 
 # ignore leading v on release
@@ -298,11 +305,13 @@ rm "${latest_changes}"
 # defined options and semantics than `yarn` for writing to the registry.
 # Tag both releases and prereleases as `next` so the last stable release remains
 # the default.
-npm publish --tag next
-if [ $prerelease -eq 0 ]; then
-    # For a release, also add the default `latest` tag.
-    package=$(cat package.json | jq -er .name)
-    npm dist-tag add "$package@$release" latest
+if [ -z "$skip_npm" ]; then
+    npm publish --tag next
+    if [ $prerelease -eq 0 ]; then
+        # For a release, also add the default `latest` tag.
+        package=$(cat package.json | jq -er .name)
+        npm dist-tag add "$package@$release" latest
+    fi
 fi
 
 if [ -z "$skip_jsdoc" ]; then
@@ -338,8 +347,10 @@ if [ -z "$skip_jsdoc" ]; then
     git push origin gh-pages
 fi
 
-# finally, merge master back onto develop
-git checkout develop
-git pull
-git merge master
-git push origin develop
+# finally, merge master back onto develop (if it exists)
+if [ $(git branch -lr | grep origin/develop -c) -ge 1 ]; then
+    git checkout develop
+    git pull
+    git merge master
+    git push origin develop
+fi

spec/integ/matrix-client-opts.spec.js

@@ -3,6 +3,7 @@ import HttpBackend from "matrix-mock-request";
 import {MatrixClient} from "../../src/matrix";
 import {MatrixScheduler} from "../../src/scheduler";
 import {MemoryStore} from "../../src/store/memory";
+import {MatrixError} from "../../src/http-api";
 
 describe("MatrixClient opts", function() {
     const baseUrl = "http://localhost.or.something";
@@ -132,10 +133,10 @@ describe("MatrixClient opts", function() {
         });
 
         it("shouldn't retry sending events", function(done) {
-            httpBackend.when("PUT", "/txn1").fail(500, {
+            httpBackend.when("PUT", "/txn1").fail(500, new MatrixError({
                 errcode: "M_SOMETHING",
                 error: "Ruh roh",
-            });
+            }));
             client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                 expect(false).toBe(true, "sendTextMessage resolved but shouldn't");
             }, function(err) {

spec/unit/crypto/secrets.spec.js

@@ -20,6 +20,7 @@ import {SECRET_STORAGE_ALGORITHM_V1_AES} from "../../../src/crypto/SecretStorage
 import {MatrixEvent} from "../../../src/models/event";
 import {TestClient} from '../../TestClient';
 import {makeTestClients} from './verification/util';
+import {encryptAES} from "../../../src/crypto/aes";
 
 import * as utils from "../../../src/utils";
 
@@ -49,6 +50,13 @@ async function makeTestClient(userInfo, options) {
     return client;
 }
 
+// Wrapper around pkSign to return a signed object. pkSign returns the
+// signature, rather than the signed object.
+function sign(obj, key, userId) {
+    olmlib.pkSign(obj, key, userId);
+    return obj;
+}
+
 describe("Secrets", function() {
     if (!global.Olm) {
         console.warn('Not running megolm backup unit tests: libolm not present');
@@ -266,104 +274,708 @@ describe("Secrets", function() {
         expect(secret).toBe("bar");
     });
 
-    it("bootstraps when no storage or cross-signing keys locally", async function() {
-        const key = new Uint8Array(16);
-        for (let i = 0; i < 16; i++) key[i] = i;
-        const getKey = jest.fn(e => {
-            return [Object.keys(e.keys)[0], key];
-        });
-
-        const bob = await makeTestClient(
-            {
-                userId: "@bob:example.com",
-                deviceId: "bob1",
-            },
-            {
-                cryptoCallbacks: {
-                    getSecretStorageKey: getKey,
-                },
-            },
+    describe("bootstrap", function() {
+        // keys used in some of the tests
+        const XSK = new Uint8Array(
+            olmlib.decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="),
         );
-        bob.uploadDeviceSigningKeys = async () => {};
-        bob.uploadKeySignatures = async () => {};
-        bob.setAccountData = async function(eventType, contents, callback) {
-            const event = new MatrixEvent({
-                type: eventType,
-                content: contents,
+        const XSPubKey = "DRb8pFVJyEJ9OWvXeUoM0jq/C2Wt+NxzBZVuk2nRb+0";
+        const USK = new Uint8Array(
+            olmlib.decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="),
+        );
+        const USPubKey = "CUpoiTtHiyXpUmd+3ohb7JVxAlUaOG1NYs9Jlx8soQU";
+        const SSK = new Uint8Array(
+            olmlib.decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="),
+        );
+        const SSPubKey = "0DfNsRDzEvkCLA0gD3m7VAGJ5VClhjEsewI35xq873Q";
+        const backupKey = new Uint8Array(
+            olmlib.decodeBase64(
+                "OTmOKrPsWP1qxmt0s2qat4vYxULG+U9cmFAWFco3A5U=",
+            ),
+        );
+        const SSSSKey = new Uint8Array(
+            olmlib.decodeBase64(
+                "XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0=",
+            ),
+        );
+        const SSSSPubKey = "v3A8HTypbccUm6jaRCRw5l+7lSdzQUACeY9xpQ5BVmE";
+
+        it("bootstraps when no storage or cross-signing keys locally", async function() {
+            const key = new Uint8Array(16);
+            for (let i = 0; i < 16; i++) key[i] = i;
+            const getKey = jest.fn(e => {
+                return [Object.keys(e.keys)[0], key];
             });
-            this.store.storeAccountDataEvents([
-                event,
-            ]);
-            this.emit("accountData", event);
-        };
 
-        await bob.bootstrapSecretStorage();
-
-        const crossSigning = bob._crypto._crossSigningInfo;
-        const secretStorage = bob._crypto._secretStorage;
-
-        expect(crossSigning.getId()).toBeTruthy();
-        expect(await crossSigning.isStoredInSecretStorage(secretStorage)).toBeTruthy();
-        expect(await secretStorage.hasKey()).toBeTruthy();
-    });
-
-    it("bootstraps when cross-signing keys in secret storage", async function() {
-        const decryption = new global.Olm.PkDecryption();
-        const storagePublicKey = decryption.generate_key();
-        const storagePrivateKey = decryption.get_private_key();
-
-        const bob = await makeTestClient(
-            {
-                userId: "@bob:example.com",
-                deviceId: "bob1",
-            },
-            {
-                cryptoCallbacks: {
-                    getSecretStorageKey: async request => {
-                        const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
-                        expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
-                        return [defaultKeyId, storagePrivateKey];
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: getKey,
                     },
                 },
-            },
-        );
+            );
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
 
-        bob.uploadDeviceSigningKeys = async () => {};
-        bob.uploadKeySignatures = async () => {};
-        bob.setAccountData = async function(eventType, contents, callback) {
-            const event = new MatrixEvent({
-                type: eventType,
-                content: contents,
-            });
-            this.store.storeAccountDataEvents([
-                event,
-            ]);
-            this.emit("accountData", event);
-        };
-        bob._crypto.checkKeyBackup = async () => {};
+            await bob.bootstrapSecretStorage();
 
-        const crossSigning = bob._crypto._crossSigningInfo;
-        const secretStorage = bob._crypto._secretStorage;
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;
 
-        // Set up cross-signing keys from scratch with specific storage key
-        await bob.bootstrapSecretStorage({
-            createSecretStorageKey: async () => ({
-                // `pubkey` not used anymore with symmetric 4S
-                keyInfo: { pubkey: storagePublicKey },
-                privateKey: storagePrivateKey,
-            }),
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
         });
 
-        // Clear local cross-signing keys and read from secret storage
-        bob._crypto._deviceList.storeCrossSigningForUser(
-            "@bob:example.com",
-            crossSigning.toStorage(),
-        );
-        crossSigning.keys = {};
-        await bob.bootstrapSecretStorage();
+        it("bootstraps when cross-signing keys in secret storage", async function() {
+            const decryption = new global.Olm.PkDecryption();
+            const storagePublicKey = decryption.generate_key();
+            const storagePrivateKey = decryption.get_private_key();
 
-        expect(crossSigning.getId()).toBeTruthy();
-        expect(await crossSigning.isStoredInSecretStorage(secretStorage)).toBeTruthy();
-        expect(await secretStorage.hasKey()).toBeTruthy();
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: async request => {
+                            const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
+                            expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
+                            return [defaultKeyId, storagePrivateKey];
+                        },
+                    },
+                },
+            );
+
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
+            bob._crypto.checkKeyBackup = async () => {};
+
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;
+
+            // Set up cross-signing keys from scratch with specific storage key
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey: async () => ({
+                    // `pubkey` not used anymore with symmetric 4S
+                    keyInfo: { pubkey: storagePublicKey },
+                    privateKey: storagePrivateKey,
+                }),
+            });
+
+            // Clear local cross-signing keys and read from secret storage
+            bob._crypto._deviceList.storeCrossSigningForUser(
+                "@bob:example.com",
+                crossSigning.toStorage(),
+            );
+            crossSigning.keys = {};
+            await bob.bootstrapSecretStorage();
+
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
+        });
+
+        it("converts asymmetric SSSS to symmetric SSSS", async function() {
+            let crossSigningKeys = {};
+            const secretStorageKeys = {
+                "old_key_id": SSSSKey,
+                "new_key_id": SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            const encryption = new global.Olm.PkEncryption();
+            encryption.set_recipient_key(SSSSPubKey);
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "old_key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.old_key_id",
+                    content: sign({
+                        algorithm: "m.secret_storage.v1.curve25519-aes-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                        pubkey: SSSSPubKey,
+                    }, XSK, "@alice:example.com"),
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(olmlib.encodeBase64(XSK)),
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(olmlib.encodeBase64(SSK)),
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(olmlib.encodeBase64(USK)),
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.megolm_backup.v1",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(
+                                olmlib.encodeBase64(backupKey),
+                            ),
+                        },
+                    },
+                }),
+            ]);
+            encryption.free();
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            const origAddSecretStorageKey = alice._crypto.addSecretStorageKey;
+            alice._crypto.addSecretStorageKey = async function(algorithm, opts, keyId) {
+                return await origAddSecretStorageKey.call(
+                    alice._crypto, algorithm, opts, keyId || "new_key_id",
+                );
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            // it should create a new key with the same parameters
+            expect(alice.getAccountData("m.secret_storage.default_key").getContent())
+                .toEqual({key: "new_key_id"});
+            const newKeyInfo = alice.getAccountData("m.secret_storage.key.new_key_id")
+                .getContent();
+            expect(newKeyInfo.algorithm)
+                .toEqual("m.secret_storage.v1.aes-hmac-sha2");
+            expect(newKeyInfo.passphrase).toEqual({
+                algorithm: "m.pbkdf2",
+                iterations: 500000,
+                salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+            });
+            expect(newKeyInfo).toHaveProperty("iv");
+            expect(newKeyInfo).toHaveProperty("mac");
+            expect(alice.checkSecretStorageKey(secretStorageKeys.new_key_id, newKeyInfo))
+                .toBeTruthy();
+
+            const crossSigningMaster = alice.getAccountData("m.cross_signing.master")
+                .getContent();
+            expect(crossSigningMaster.encrypted).toHaveProperty("new_key_id");
+            expect(crossSigningMaster.encrypted).not.toHaveProperty("old_key_id");
+            expect(await alice.getSecret("m.cross_signing.master"))
+                .toEqual(olmlib.encodeBase64(XSK));
+
+            const USKInfo = alice.getAccountData("m.cross_signing.self_signing")
+                .getContent();
+            expect(USKInfo.encrypted).toHaveProperty("new_key_id");
+            expect(USKInfo.encrypted).not.toHaveProperty("old_key_id");
+            expect(await alice.getSecret("m.cross_signing.user_signing"))
+                .toEqual(olmlib.encodeBase64(USK));
+
+            const SSKInfo = alice.getAccountData("m.cross_signing.self_signing")
+                .getContent();
+            expect(SSKInfo.encrypted).toHaveProperty("new_key_id");
+            expect(SSKInfo.encrypted).not.toHaveProperty("old_key_id");
+            expect(await alice.getSecret("m.cross_signing.self_signing"))
+                .toEqual(olmlib.encodeBase64(SSK));
+
+            const backupKeyInfo = alice.getAccountData("m.megolm_backup.v1")
+                .getContent();
+            expect(backupKeyInfo.encrypted).toHaveProperty("new_key_id");
+            expect(backupKeyInfo.encrypted).not.toHaveProperty("old_key_id");
+            expect(await alice.getSecret("m.megolm_backup.v1"))
+                .toEqual(olmlib.encodeBase64(backupKey));
+        });
+        it("converts asymmetric SSSS with passthrough", async function() {
+            let crossSigningKeys = {};
+            const secretStorageKeys = {
+                "old_key_id": SSSSKey,
+                "new_key_id": SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            const encryption = new global.Olm.PkEncryption();
+            encryption.set_recipient_key(SSSSPubKey);
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "old_key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.old_key_id",
+                    content: sign({
+                        algorithm: "m.secret_storage.v1.curve25519-aes-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                        pubkey: "v3A8HTypbccUm6jaRCRw5l+7lSdzQUACeY9xpQ5BVmE",
+                    }, XSK, "@alice:example.com"),
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(olmlib.encodeBase64(XSK)),
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(olmlib.encodeBase64(SSK)),
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(olmlib.encodeBase64(USK)),
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.megolm_backup.v1",
+                    content: {
+                        encrypted: {
+                            old_key_id: {
+                                passthrough: true,
+                            },
+                        },
+                    },
+                }),
+            ]);
+            encryption.free();
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "v3A8HTypbccUm6jaRCRw5l+7lSdzQUACeY9xpQ5BVmE",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            const origAddSecretStorageKey = alice._crypto.addSecretStorageKey;
+            alice._crypto.addSecretStorageKey = async function(algorithm, opts, keyId) {
+                return await origAddSecretStorageKey.call(
+                    alice._crypto, algorithm, opts, keyId || "new_key_id",
+                );
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            // the new backup key should be the encoded SSSS key
+            const backupKeyInfo = alice.getAccountData("m.megolm_backup.v1")
+                .getContent();
+            expect(backupKeyInfo.encrypted).toHaveProperty("new_key_id");
+            expect(backupKeyInfo.encrypted).not.toHaveProperty("old_key_id");
+            expect(await alice.getSecret("m.megolm_backup.v1"))
+                .toEqual(olmlib.encodeBase64(SSSSKey));
+        });
+        it("adds passphrase checking if it's lacking", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                // we never use these values, other than checking that they
+                // exist, so just use dummy values
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            expect(alice.getAccountData("m.secret_storage.default_key").getContent())
+                .toEqual({key: "key_id"});
+            const keyInfo = alice.getAccountData("m.secret_storage.key.key_id")
+                .getContent();
+            expect(keyInfo.algorithm)
+                .toEqual("m.secret_storage.v1.aes-hmac-sha2");
+            expect(keyInfo.passphrase).toEqual({
+                algorithm: "m.pbkdf2",
+                iterations: 500000,
+                salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+            });
+            expect(keyInfo).toHaveProperty("iv");
+            expect(keyInfo).toHaveProperty("mac");
+            expect(alice.checkSecretStorageKey(secretStorageKeys.key_id, keyInfo))
+                .toBeTruthy();
+        });
+        it("fixes backup keys in the wrong format", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.megolm_backup.v1",
+                    content: {
+                        encrypted: {
+                            key_id: await encryptAES(
+                                "123,45,6,7,89,1,234,56,78,90,12,34,5,67,8,90",
+                                secretStorageKeys.key_id, "m.megolm_backup.v1",
+                            ),
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            const backupKey = alice.getAccountData("m.megolm_backup.v1")
+                .getContent();
+            expect(backupKey.encrypted).toHaveProperty("key_id");
+            expect(await alice.getSecret("m.megolm_backup.v1"))
+                .toEqual("ey0GB1kB6jhOWgwiBUMIWg==");
+        });
     });
 });

spec/unit/crypto/verification/sas.spec.js

@@ -44,7 +44,16 @@ describe("SAS verification", function() {
     });
 
     it("should error on an unexpected event", async function() {
-        const sas = new SAS({}, "@alice:example.com", "ABCDEFG");
+        //channel, baseApis, userId, deviceId, startEvent, request
+        const request = {
+            onVerifierCancelled: function() {},
+        };
+        const channel = {
+            send: function() {
+                return Promise.resolve();
+            },
+        };
+        const sas = new SAS(channel, {}, "@alice:example.com", "ABCDEFG", null, request);
         sas.handleEvent(new MatrixEvent({
             sender: "@alice:example.com",
             type: "es.inquisition",

spec/unit/interactive-auth.spec.js

@@ -143,4 +143,33 @@ describe("InteractiveAuth", function() {
             expect(stateUpdated).toBeCalledTimes(1);
         });
     });
+
+    it("should start an auth stage and reject if no auth flow", function() {
+        const doRequest = jest.fn();
+        const stateUpdated = jest.fn();
+
+        const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
+            doRequest: doRequest,
+            stateUpdated: stateUpdated,
+        });
+
+        doRequest.mockImplementation(function(authData) {
+            logger.log("request1", authData);
+            expect(authData).toEqual({});
+            const err = new MatrixError({
+                session: "sessionId",
+                flows: [],
+                params: {
+                    "logintype": { param: "aa" },
+                },
+            });
+            err.httpStatus = 401;
+            throw err;
+        });
+
+        return ia.attemptAuth().catch(function(error) {
+            expect(error.message).toBe('No appropriate authentication flow found');
+        });
+    });
 });

src/client.js

@@ -35,19 +35,25 @@ import {StubStore} from "./store/stub";
 import {createNewMatrixCall} from "./webrtc/call";
 import * as utils from './utils';
 import {sleep} from './utils';
-import {MatrixError, PREFIX_MEDIA_R0, PREFIX_UNSTABLE} from "./http-api";
+import {
+    MatrixError,
+    PREFIX_MEDIA_R0,
+    PREFIX_UNSTABLE,
+    retryNetworkOperation,
+} from "./http-api";
 import {getHttpUriForMxc} from "./content-repo";
 import * as ContentHelpers from "./content-helpers";
 import * as olmlib from "./crypto/olmlib";
 import {ReEmitter} from './ReEmitter';
 import {RoomList} from './crypto/RoomList';
 import {logger} from './logger';
-import {Crypto, isCryptoAvailable} from './crypto';
+import {Crypto, isCryptoAvailable, fixBackupKey} from './crypto';
 import {decodeRecoveryKey} from './crypto/recoverykey';
 import {keyFromAuthData} from './crypto/key_passphrase';
 import {randomString} from './randomstring';
 import {PushProcessor} from "./pushprocessor";
 import {encodeBase64, decodeBase64} from "./crypto/olmlib";
+import { User } from "./models/user";
 
 const SCROLLBACK_DELAY_MS = 3000;
 export const CRYPTO_ENABLED = isCryptoAvailable();
@@ -731,6 +737,7 @@ MatrixClient.prototype.initCrypto = async function() {
         "crypto.roomKeyRequestCancellation",
         "crypto.warning",
         "crypto.devicesUpdated",
+        "crypto.willUpdateDevices",
         "deviceVerificationChanged",
         "userTrustStatusChanged",
         "crossSigning.keysChanged",
@@ -819,9 +826,9 @@ MatrixClient.prototype.downloadKeys = function(userIds, forceDownload) {
  *
  * @param {string} userId the user to list keys for.
  *
- * @return {Promise<module:crypto/deviceinfo[]>} list of devices
+ * @return {module:crypto/deviceinfo[]} list of devices
  */
-MatrixClient.prototype.getStoredDevicesForUser = async function(userId) {
+MatrixClient.prototype.getStoredDevicesForUser = function(userId) {
     if (this._crypto === null) {
         throw new Error("End-to-end encryption disabled");
     }
@@ -834,9 +841,9 @@ MatrixClient.prototype.getStoredDevicesForUser = async function(userId) {
  * @param {string} userId the user to list keys for.
  * @param {string} deviceId unique identifier for the device
  *
- * @return {Promise<?module:crypto/deviceinfo>} device or null
+ * @return {module:crypto/deviceinfo} device or null
  */
-MatrixClient.prototype.getStoredDevice = async function(userId, deviceId) {
+MatrixClient.prototype.getStoredDevice = function(userId, deviceId) {
     if (this._crypto === null) {
         throw new Error("End-to-end encryption disabled");
     }
@@ -1429,15 +1436,17 @@ MatrixClient.prototype.exportRoomKeys = function() {
  * Import a list of room keys previously exported by exportRoomKeys
  *
  * @param {Object[]} keys a list of session export objects
+ * @param {Object} opts
+ * @param {Function} opts.progressCallback called with an object that has a "stage" param
  *
  * @return {Promise} a promise which resolves when the keys
  *    have been imported
  */
-MatrixClient.prototype.importRoomKeys = function(keys) {
+MatrixClient.prototype.importRoomKeys = function(keys, opts) {
     if (!this._crypto) {
         throw new Error("End-to-end encryption disabled");
     }
-    return this._crypto.importRoomKeys(keys);
+    return this._crypto.importRoomKeys(keys, opts);
 };
 
 /**
@@ -1815,7 +1824,17 @@ MatrixClient.prototype.restoreKeyBackupWithPassword = async function(
 MatrixClient.prototype.restoreKeyBackupWithSecretStorage = async function(
     backupInfo, targetRoomId, targetSessionId, opts,
 ) {
-    const privKey = decodeBase64(await this.getSecret("m.megolm_backup.v1"));
+    const storedKey = await this.getSecret("m.megolm_backup.v1");
+
+    // ensure that the key is in the right format.  If not, fix the key and
+    // store the fixed version
+    const fixedKey = fixBackupKey(storedKey);
+    if (fixedKey) {
+        const [keyId] = await this._crypto.getSecretStorageKey();
+        await this.storeSecret("m.megolm_backup.v1", fixedKey, [keyId]);
+    }
+
+    const privKey = decodeBase64(fixedKey || storedKey);
     return this._restoreKeyBackup(
         privKey, targetRoomId, targetSessionId, backupInfo, opts,
     );
@@ -1870,7 +1889,10 @@ MatrixClient.prototype.restoreKeyBackupWithCache = async function(
 
 MatrixClient.prototype._restoreKeyBackup = function(
     privKey, targetRoomId, targetSessionId, backupInfo,
-    { cacheCompleteCallback }={}, // For sequencing during tests
+    {
+        cacheCompleteCallback, // For sequencing during tests
+        progressCallback,
+    }={},
 ) {
     if (this._crypto === null) {
         throw new Error("End-to-end encryption disabled");
@@ -1905,6 +1927,12 @@ MatrixClient.prototype._restoreKeyBackup = function(
         console.warn("Error caching session backup key:", e);
     }).then(cacheCompleteCallback);
 
+    if (progressCallback) {
+        progressCallback({
+            stage: "fetch",
+        });
+    }
+
     return this._http.authedRequest(
         undefined, "GET", path.path, path.queryData, undefined,
         {prefix: PREFIX_UNSTABLE},
@@ -1939,7 +1967,7 @@ MatrixClient.prototype._restoreKeyBackup = function(
             }
         }
 
-        return this.importRoomKeys(keys);
+        return this.importRoomKeys(keys, { progressCallback });
     }).then(() => {
         return this._crypto.setTrustedBackupPubKey(backupPubKey);
     }).then(() => {
@@ -2075,6 +2103,7 @@ MatrixClient.prototype.getUsers = function() {
 
 /**
  * Set account data event for the current user.
+ * It will retry the request up to 5 times.
  * @param {string} eventType The event type
  * @param {Object} contents the contents object for the event
  * @param {module:client.callback} callback Optional.
@@ -2086,9 +2115,13 @@ MatrixClient.prototype.setAccountData = function(eventType, contents, callback)
         $userId: this.credentials.userId,
         $type: eventType,
     });
-    return this._http.authedRequest(
-        callback, "PUT", path, undefined, contents,
-    );
+    const promise = retryNetworkOperation(5, () => {
+        return this._http.authedRequest(undefined, "PUT", path, undefined, contents);
+    });
+    if (callback) {
+        promise.then(result => callback(null, result), callback);
+    }
+    return promise;
 };
 
 /**
@@ -2123,9 +2156,17 @@ MatrixClient.prototype.getAccountDataFromServer = async function(eventType) {
         $userId: this.credentials.userId,
         $type: eventType,
     });
-    return this._http.authedRequest(
-        undefined, "GET", path, undefined,
-    );
+    try {
+        const result = await this._http.authedRequest(
+            undefined, "GET", path, undefined,
+        );
+        return result;
+    } catch (e) {
+        if (e.data && e.data.errcode === 'M_NOT_FOUND') {
+            return null;
+        }
+        throw e;
+    }
 };
 
 /**
@@ -4499,64 +4540,54 @@ MatrixClient.prototype.getFilter = function(userId, filterId, allowCached) {
  * @param {Filter} filter
  * @return {Promise<String>} Filter ID
  */
-MatrixClient.prototype.getOrCreateFilter = function(filterName, filter) {
+MatrixClient.prototype.getOrCreateFilter = async function(filterName, filter) {
     const filterId = this.store.getFilterIdByName(filterName);
-    let promise = Promise.resolve();
-    const self = this;
+    let existingId = undefined;
 
     if (filterId) {
         // check that the existing filter matches our expectations
-        promise = self.getFilter(self.credentials.userId,
-                         filterId, true,
-        ).then(function(existingFilter) {
-            const oldDef = existingFilter.getDefinition();
-            const newDef = filter.getDefinition();
+        try {
+            const existingFilter =
+                await this.getFilter(this.credentials.userId, filterId, true);
+            if (existingFilter) {
+                const oldDef = existingFilter.getDefinition();
+                const newDef = filter.getDefinition();
 
-            if (utils.deepCompare(oldDef, newDef)) {
-                // super, just use that.
-                // debuglog("Using existing filter ID %s: %s", filterId,
-                //          JSON.stringify(oldDef));
-                return Promise.resolve(filterId);
+                if (utils.deepCompare(oldDef, newDef)) {
+                    // super, just use that.
+                    // debuglog("Using existing filter ID %s: %s", filterId,
+                    //          JSON.stringify(oldDef));
+                    existingId = filterId;
+                }
             }
-            // debuglog("Existing filter ID %s: %s; new filter: %s",
-            //          filterId, JSON.stringify(oldDef), JSON.stringify(newDef));
-            self.store.setFilterIdByName(filterName, undefined);
-            return undefined;
-        }, function(error) {
+        } catch (error) {
             // Synapse currently returns the following when the filter cannot be found:
             // {
             //     errcode: "M_UNKNOWN",
             //     name: "M_UNKNOWN",
             //     message: "No row found",
-            //     data: Object, httpStatus: 404
             // }
-            if (error.httpStatus === 404 &&
-                (error.errcode === "M_UNKNOWN" || error.errcode === "M_NOT_FOUND")) {
-                // Clear existing filterId from localStorage
-                // if it no longer exists on the server
-                self.store.setFilterIdByName(filterName, undefined);
-                // Return a undefined value for existingId further down the promise chain
-                return undefined;
-            } else {
+            if (error.errcode !== "M_UNKNOWN" && error.errcode !== "M_NOT_FOUND") {
                 throw error;
             }
-        });
+        }
+        // if the filter doesn't exist anymore on the server, remove from store
+        if (!existingId) {
+            this.store.setFilterIdByName(filterName, undefined);
+        }
     }
 
-    return promise.then(function(existingId) {
-        if (existingId) {
-            return existingId;
-        }
+    if (existingId) {
+        return existingId;
+    }
 
-        // create a new filter
-        return self.createFilter(filter.getDefinition(),
-        ).then(function(createdFilter) {
-            // debuglog("Created new filter ID %s: %s", createdFilter.filterId,
-            //          JSON.stringify(createdFilter.getDefinition()));
-            self.store.setFilterIdByName(filterName, createdFilter.filterId);
-            return createdFilter.filterId;
-        });
-    });
+    // create a new filter
+    const createdFilter = await this.createFilter(filter.getDefinition());
+
+    // debuglog("Created new filter ID %s: %s", createdFilter.filterId,
+    //          JSON.stringify(createdFilter.getDefinition()));
+    this.store.setFilterIdByName(filterName, createdFilter.filterId);
+    return createdFilter.filterId;
 };
 
 
@@ -4734,6 +4765,13 @@ MatrixClient.prototype.startClient = async function(opts) {
         };
     }
 
+    // Create our own user object artificially (instead of waiting for sync)
+    // so it's always available, even if the user is not in any rooms etc.
+    const userId = this.getUserId();
+    if (userId) {
+        this.store.storeUser(new User(userId));
+    }
+
     if (this._crypto) {
         this._crypto.uploadDeviceKeys();
         this._crypto.start();
@@ -5247,17 +5285,20 @@ function _resolve(callback, resolve, res) {
     resolve(res);
 }
 
-function _PojoToMatrixEventMapper(client) {
+function _PojoToMatrixEventMapper(client, options) {
+    const preventReEmit = Boolean(options && options.preventReEmit);
     function mapper(plainOldJsObject) {
         const event = new MatrixEvent(plainOldJsObject);
         if (event.isEncrypted()) {
-            client.reEmitter.reEmit(event, [
-                "Event.decrypted",
-            ]);
+            if (!preventReEmit) {
+                client.reEmitter.reEmit(event, [
+                    "Event.decrypted",
+                ]);
+            }
             event.attemptDecryption(client._crypto);
         }
         const room = client.getRoom(event.getRoomId());
-        if (room) {
+        if (room && !preventReEmit) {
             room.reEmitter.reEmit(event, ["Event.replaced"]);
         }
         return event;
@@ -5266,10 +5307,12 @@ function _PojoToMatrixEventMapper(client) {
 }
 
 /**
+ * @param {object} [options]
+ * @param {bool} options.preventReEmit don't reemit events emitted on an event mapped by this mapper on the client
  * @return {Function}
  */
-MatrixClient.prototype.getEventMapper = function() {
-    return _PojoToMatrixEventMapper(this);
+MatrixClient.prototype.getEventMapper = function(options = undefined) {
+    return _PojoToMatrixEventMapper(this, options);
 };
 
 /**
@@ -5555,6 +5598,22 @@ MatrixClient.prototype.generateClientSecret = function() {
  * });
  */
 
+/**
+ * Fires whenever the stored devices for a user have changed
+ * @event module:client~MatrixClient#"crypto.devicesUpdated"
+ * @param {String[]} users A list of user IDs that were updated
+ * @param {bool} initialFetch If true, the store was empty (apart
+ *     from our own device) and has been seeded.
+ */
+
+/**
+ * Fires whenever the stored devices for a user will be updated
+ * @event module:client~MatrixClient#"crypto.willUpdateDevices"
+ * @param {String[]} users A list of user IDs that will be updated
+ * @param {bool} initialFetch If true, the store is empty (apart
+ *     from our own device) and is being seeded.
+ */
+
 /**
  * Fires whenever the status of e2e key backup changes, as returned by getKeyBackupEnabled()
  * @event module:client~MatrixClient#"crypto.keyBackupStatus"

src/crypto/CrossSigning.js

@@ -310,6 +310,13 @@ export class CrossSigningInfo extends EventEmitter {
         }
     }
 
+    /**
+     * unsets the keys, used when another session has reset the keys, to disable cross-signing
+     */
+    clearKeys() {
+        this.keys = {};
+    }
+
     setKeys(keys) {
         const signingKeys = {};
         if (keys.master) {

src/crypto/DeviceList.js

@@ -109,6 +109,9 @@ export class DeviceList extends EventEmitter {
         this._savePromiseTime = null;
         // The timer used to delay the save
         this._saveTimer = null;
+        // True if we have fetched data from the server or loaded a non-empty
+        // set of device data from the store
+        this._hasFetched = null;
     }
 
     /**
@@ -118,6 +121,7 @@ export class DeviceList extends EventEmitter {
         await this._cryptoStore.doTxn(
             'readonly', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
                 this._cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
+                    this._hasFetched = Boolean(deviceData && deviceData.devices);
                     this._devices = deviceData ? deviceData.devices : {},
                     this._crossSigningInfo = deviceData ?
                         deviceData.crossSigningInfo || {} : {};
@@ -652,6 +656,7 @@ export class DeviceList extends EventEmitter {
         });
 
         const finished = (success) => {
+            this.emit("crypto.willUpdateDevices", users, !this._hasFetched);
             users.forEach((u) => {
                 this._dirty = true;
 
@@ -677,7 +682,8 @@ export class DeviceList extends EventEmitter {
                 }
             });
             this.saveIfDirty();
-            this.emit("crypto.devicesUpdated", users);
+            this.emit("crypto.devicesUpdated", users, !this._hasFetched);
+            this._hasFetched = true;
         };
 
         return prom;

src/crypto/OlmDevice.js

@@ -105,6 +105,9 @@ export function OlmDevice(cryptoStore) {
     // Keep track of sessions that we're starting, so that we don't start
     // multiple sessions for the same device at the same time.
     this._sessionsInProgress = {};
+
+    // Used by olm to serialise prekey message decryptions
+    this._olmPrekeyPromise = Promise.resolve();
 }
 
 /**

src/crypto/SecretStorage.js

@@ -20,6 +20,7 @@ import * as olmlib from './olmlib';
 import {pkVerify} from './olmlib';
 import {randomString} from '../randomstring';
 import {encryptAES, decryptAES} from './aes';
+import {encodeBase64} from "./olmlib";
 
 export const SECRET_STORAGE_ALGORITHM_V1_AES
     = "m.secret_storage.v1.aes-hmac-sha2";
@@ -52,7 +53,7 @@ export class SecretStorage extends EventEmitter {
     }
 
     setDefaultKeyId(keyId) {
-        return new Promise((resolve) => {
+        return new Promise(async (resolve, reject) => {
             const listener = (ev) => {
                 if (
                     ev.getType() === 'm.secret_storage.default_key' &&
@@ -64,10 +65,15 @@ export class SecretStorage extends EventEmitter {
             };
             this._baseApis.on('accountData', listener);
 
-            this._baseApis.setAccountData(
-                'm.secret_storage.default_key',
-                { key: keyId },
-            );
+            try {
+                await this._baseApis.setAccountData(
+                    'm.secret_storage.default_key',
+                    { key: keyId },
+                );
+            } catch (e) {
+                this._baseApis.removeListener('accountData', listener);
+                reject(e);
+            }
         });
     }
 
@@ -98,7 +104,7 @@ export class SecretStorage extends EventEmitter {
                 keyData.passphrase = opts.passphrase;
             }
             if (opts.key) {
-                const {iv, mac} = await encryptAES(ZERO_STR, opts.key, "");
+                const {iv, mac} = await SecretStorage._calculateKeyCheck(opts.key);
                 keyData.iv = iv;
                 keyData.mac = mac;
             }
@@ -209,7 +215,7 @@ export class SecretStorage extends EventEmitter {
         case SECRET_STORAGE_ALGORITHM_V1_AES:
         {
             if (info.mac) {
-                const {mac} = await encryptAES(ZERO_STR, key, "", info.iv);
+                const {mac} = await SecretStorage._calculateKeyCheck(key, info.iv);
                 return info.mac === mac;
             } else {
                 // if we have no information, we have to assume the key is right
@@ -235,6 +241,10 @@ export class SecretStorage extends EventEmitter {
         }
     }
 
+    static async _calculateKeyCheck(key, iv) {
+        return await encryptAES(ZERO_STR, key, "", iv);
+    }
+
     /**
      * Store an encrypted secret on the server
      *
@@ -372,8 +382,9 @@ export class SecretStorage extends EventEmitter {
             const encInfo = secretInfo.encrypted[keyId];
 
             // We don't actually need the decryption object if it's a passthrough
-            // since we just want to return the key itself.
-            if (encInfo.passthrough) return decryption.get_private_key();
+            // since we just want to return the key itself. It must be base64
+            // encoded, since this is how a key would normally be stored.
+            if (encInfo.passthrough) return encodeBase64(decryption.get_private_key());
 
             return await decryption.decrypt(encInfo);
         } finally {
@@ -586,7 +597,7 @@ export class SecretStorage extends EventEmitter {
                     this._baseApis,
                     {
                         [sender]: [
-                            await this._baseApis.getStoredDevice(sender, deviceId),
+                            this._baseApis.getStoredDevice(sender, deviceId),
                         ],
                     },
                 );
@@ -596,7 +607,7 @@ export class SecretStorage extends EventEmitter {
                     this._baseApis.deviceId,
                     this._baseApis._crypto._olmDevice,
                     sender,
-                    this._baseApis._crypto.getStoredDevice(sender, deviceId),
+                    this._baseApis.getStoredDevice(sender, deviceId),
                     payload,
                 );
                 const contentMap = {

src/crypto/algorithms/olm.js

@@ -264,6 +264,25 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
  */
 OlmDecryption.prototype._decryptMessage = async function(
     theirDeviceIdentityKey, message,
+) {
+    // This is a wrapper that serialises decryptions of prekey messages, because
+    // otherwise we race between deciding we have no active sessions for the message
+    // and creating a new one, which we can only do once because it removes the OTK.
+    if (message.type !== 0) {
+        // not a prekey message: we can safely just try & decrypt it
+        return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+    } else {
+        const myPromise = this._olmDevice._olmPrekeyPromise.then(() => {
+            return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+        });
+        // we want the error, but don't propagate it to the next decryption
+        this._olmDevice._olmPrekeyPromise = myPromise.catch(() => {});
+        return await myPromise;
+    }
+};
+
+OlmDecryption.prototype._reallyDecryptMessage = async function(
+    theirDeviceIdentityKey, message,
 ) {
     const sessionIds = await this._olmDevice.getSessionIdsForDevice(
         theirDeviceIdentityKey,

src/crypto/index.js

@@ -169,7 +169,9 @@ export function Crypto(baseApis, sessionStore, userId, deviceId,
     this._deviceList.on(
         'userCrossSigningUpdated', this._onDeviceListUserCrossSigningUpdated,
     );
-    this._reEmitter.reEmit(this._deviceList, ["crypto.devicesUpdated"]);
+    this._reEmitter.reEmit(this._deviceList, [
+        "crypto.devicesUpdated", "crypto.willUpdateDevices",
+    ]);
 
     // the last time we did a check for the number of one-time-keys on the
     // server.
@@ -306,7 +308,8 @@ Crypto.prototype.init = async function(opts) {
         'readonly', [IndexedDBCryptoStore.STORE_ACCOUNT],
         (txn) => {
             this._cryptoStore.getCrossSigningKeys(txn, (keys) => {
-                if (keys) {
+                // can be an empty object after resetting cross-signing keys, see _storeTrustedSelfKeys
+                if (keys && Object.keys(keys).length !== 0) {
                     logger.log("Loaded cross-signing public keys from crypto store");
                     this._crossSigningInfo.setKeys(keys);
                 }
@@ -432,6 +435,14 @@ Crypto.prototype.isCrossSigningReady = async function() {
  * up, then no changes are made, so this is safe to run to ensure secret storage
  * is ready for use.
  *
+ * This function
+ * - creates a new Secure Secret Storage key if no default key exists
+ *   - if a key backup exists, it is migrated to store the key in the Secret
+ *     Storage
+ * - creates a backup if none exists, and one is requested
+ * - migrates Secure Secret Storage to use the latest algorithm, if an outdated
+ *   algorithm is found
+ *
  * @param {function} [opts.authUploadDeviceSigningKeys] Optional. Function
  * called to await an interactive auth flow when uploading device signing keys.
  * Args:
@@ -501,172 +512,220 @@ Crypto.prototype.bootstrapSecretStorage = async function({
             return key;
         };
 
+    // create a new SSSS key and set it as default
+    const createSSSS = async (opts, privateKey) => {
+        opts = opts || {};
+        if (privateKey) {
+            opts.key = privateKey;
+        }
+
+        const keyId = await this.addSecretStorageKey(
+            SECRET_STORAGE_ALGORITHM_V1_AES, opts,
+        );
+        await this.setDefaultSecretStorageKeyId(keyId);
+
+        if (privateKey) {
+            // cache the private key so that we can access it again
+            ssssKeys[keyId] = privateKey;
+        }
+        return keyId;
+    };
+
+    // reset the cross-signing keys
+    const resetCrossSigning = async () => {
+        this._baseApis._cryptoCallbacks.saveCrossSigningKeys =
+            keys => Object.assign(crossSigningPrivateKeys, keys);
+        this._baseApis._cryptoCallbacks.getCrossSigningKey =
+            name => crossSigningPrivateKeys[name];
+        await this.resetCrossSigningKeys(
+            CrossSigningLevel.MASTER,
+            { authUploadDeviceSigningKeys },
+        );
+    };
+
+    const ensureCanCheckPassphrase = async (keyId, keyInfo) => {
+        if (!keyInfo.mac) {
+            const key = await this._baseApis._cryptoCallbacks.getSecretStorageKey(
+                {keys: {[keyId]: keyInfo}}, "",
+            );
+            if (key) {
+                const keyData = key[1];
+                ssssKeys[keyId] = keyData;
+                const {iv, mac} = await SecretStorage._calculateKeyCheck(keyData);
+                keyInfo.iv = iv;
+                keyInfo.mac = mac;
+
+                await this._crossSigningInfo.signObject(keyInfo, 'master');
+
+                await this._baseApis.setAccountData(
+                    `m.secret_storage.key.${keyId}`, keyInfo,
+                );
+            }
+        }
+    };
+
     try {
+        const oldSSSSKey = await this.getSecretStorageKey();
+        const [oldKeyId, oldKeyInfo] = oldSSSSKey || [null, null];
         const decryptionKeys =
               await this._crossSigningInfo.isStoredInSecretStorage(this._secretStorage);
         const inStorage = !setupNewSecretStorage && decryptionKeys;
-        if (decryptionKeys && !(Object.values(decryptionKeys).some(
+
+        if (!inStorage && !keyBackupInfo) {
+            // either we don't have anything, or we've been asked to restart
+            // from scratch
+            logger.log(
+                "Cross-signing private keys not found in secret storage, " +
+                    "creating new keys",
+            );
+
+            await resetCrossSigning();
+
+            if (
+                setupNewSecretStorage ||
+                !oldKeyInfo ||
+                oldKeyInfo.algorithm !== SECRET_STORAGE_ALGORITHM_V1_AES
+            ) {
+                // if we already have a usable default SSSS key and aren't resetting SSSS just use it.
+                // otherwise, create a new one
+                // Note: we leave the old SSSS key in place: there could be other secrets using it, in theory.
+                // We could move them to the new key but a) that would mean we'd need to prompt for the old
+                // passphrase, and b) it's not clear that would be the right thing to do anyway.
+                const { keyInfo, privateKey } = await createSecretStorageKey();
+                newKeyId = await createSSSS(keyInfo, privateKey);
+            }
+
+            if (oldKeyInfo && oldKeyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                await ensureCanCheckPassphrase(oldKeyId, oldKeyInfo);
+            }
+        } else if (!inStorage && keyBackupInfo) {
+            // we have an existing backup, but no SSSS
+
+            logger.log("Secret storage default key not found, using key backup key");
+
+            // if we have the backup key already cached, use it; otherwise use the
+            // callback to prompt for the key
+            const backupKey = await this.getSessionBackupPrivateKey() ||
+                              await getKeyBackupPassphrase();
+
+            // create new cross-signing keys
+            await resetCrossSigning();
+
+            // create a new SSSS key and use the backup key as the new SSSS key
+            const opts = {};
+
+            if (
+                keyBackupInfo.auth_data.private_key_salt &&
+                keyBackupInfo.auth_data.private_key_iterations
+            ) {
+                opts.passphrase = {
+                    algorithm: "m.pbkdf2",
+                    iterations: keyBackupInfo.auth_data.private_key_iterations,
+                    salt: keyBackupInfo.auth_data.private_key_salt,
+                    bits: 256,
+                };
+            }
+
+            newKeyId = await createSSSS(opts, backupKey);
+
+            // store the backup key in secret storage
+            await this.storeSecret(
+                "m.megolm_backup.v1", olmlib.encodeBase64(backupKey), [newKeyId],
+            );
+
+            // The backup is trusted because the user provided the private key.
+            // Sign the backup with the cross signing key so the key backup can
+            // be trusted via cross-signing.
+            logger.log("Adding cross signing signature to key backup");
+            await this._crossSigningInfo.signObject(
+                keyBackupInfo.auth_data, "master",
+            );
+            await this._baseApis._http.authedRequest(
+                undefined, "PUT", "/room_keys/version/" + keyBackupInfo.version,
+                undefined, keyBackupInfo,
+                {prefix: httpApi.PREFIX_UNSTABLE},
+            );
+        } else if (!(Object.values(decryptionKeys).some(
             info => info.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES,
         ))) {
-            // we already have cross-signing keys, but they're encrypted using
-            // the old algorithm
-            logger.log("Switching to symmetric");
+            // we have an asymmetric SSSS
+            logger.log("Asymmetric SSSS found. Switching SSSS to symmetric");
             const keys = {};
             // fetch the cross-signing private keys (needed to sign the new
-            // SSSS key).  We store the cross-signing keys, and temporarily set
-            // a callback so that when the private key is needed while setting
-            // things up, we can provide it.
+            // SSSS key, and so that we can re-encrypt them with the new key).
+            // We store the cross-signing keys, and temporarily set a callback
+            // so that when the private keys are needed while setting things
+            // up, we can provide it.
             this._baseApis._cryptoCallbacks.getCrossSigningKey =
                 name => crossSigningPrivateKeys[name];
             for (const type of ["master", "self_signing", "user_signing"]) {
                 const secretName = `m.cross_signing.${type}`;
                 const secret = await this.getSecret(secretName);
                 keys[type] = secret;
-                crossSigningPrivateKeys[type] = olmlib.decodeBase64(secret);
+                crossSigningPrivateKeys[type]
+                    = new Uint8Array(olmlib.decodeBase64(secret));
             }
+
             await this.checkOwnCrossSigningTrust();
+
             const opts = {};
-            let oldKeyId = null;
+
+            let oldKey = null;
             for (const [keyId, keyInfo] of Object.entries(decryptionKeys)) {
                 // See if the old key was generated from a passphrase.  If
                 // yes, use the same settings.
                 if (keyId in ssssKeys) {
-                    oldKeyId = keyId;
+                    oldKey = ssssKeys[keyId];
                     if (keyInfo.passphrase) {
                         opts.passphrase = keyInfo.passphrase;
                     }
                     break;
                 }
             }
-            if (oldKeyId) {
-                opts.key = ssssKeys[oldKeyId];
-            }
-            // create new symmetric SSSS key and set it as default
-            newKeyId = await this.addSecretStorageKey(
-                SECRET_STORAGE_ALGORITHM_V1_AES, opts,
-            );
-            if (oldKeyId) {
-                ssssKeys[newKeyId] = ssssKeys[oldKeyId];
-            }
-            await this.setDefaultSecretStorageKeyId(newKeyId);
-            // re-encrypt all the keys with the new key
+
+            // create new symmetric SSSS key
+            newKeyId = await createSSSS(opts, oldKey);
+
+            logger.log("re-encrypting cross-signing keys");
+            // re-encrypt all the cross-signing keys with the new key
             for (const type of ["master", "self_signing", "user_signing"]) {
                 const secretName = `m.cross_signing.${type}`;
                 await this.storeSecret(secretName, keys[type], [newKeyId]);
             }
-        } else if (!this._crossSigningInfo.getId() || !inStorage) {
-            // create new cross-signing keys if necessary.
-            logger.log(
-                "Cross-signing public and/or private keys not found, " +
-                "checking secret storage for private keys",
-            );
-            if (inStorage) {
-                logger.log("Cross-signing private keys found in secret storage");
-                await this.checkOwnCrossSigningTrust();
-            } else {
-                logger.log(
-                    "Cross-signing private keys not found in secret storage, " +
-                    "creating new keys",
-                );
-                this._baseApis._cryptoCallbacks.saveCrossSigningKeys =
-                    keys => Object.assign(crossSigningPrivateKeys, keys);
-                this._baseApis._cryptoCallbacks.getCrossSigningKey =
-                    name => crossSigningPrivateKeys[name];
-                await this.resetCrossSigningKeys(
-                    CrossSigningLevel.MASTER,
-                    { authUploadDeviceSigningKeys },
+
+            if (await this.isSecretStored("m.megolm_backup.v1", false)) {
+                // re-encrypt the backup key if we had one too
+                logger.log("re-encrypting backup key");
+                const backupKey = await this.getSecret("m.megolm_backup.v1");
+
+                await this.storeSecret(
+                    "m.megolm_backup.v1", fixBackupKey(backupKey) || backupKey,
+                    [newKeyId],
                 );
             }
+        } else if (!this._crossSigningInfo.getId()) {
+            // we have SSSS, but we don't know if the server's cross-signing
+            // keys should be trusted
+            logger.log("Cross-signing private keys found in secret storage");
+
+            // fetch the private keys and set up our local copy of the keys for
+            // use
+            await this.checkOwnCrossSigningTrust();
+
+            if (oldKeyInfo && oldKeyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                // make sure that the default key has the information needed to
+                // check the passphrase
+                await ensureCanCheckPassphrase(oldKeyId, oldKeyInfo);
+            }
         } else {
+            // we have SSSS and we cross-signing is already set up
             logger.log("Cross signing keys are present in secret storage");
-        }
 
-        // Check if we need to create a new secret storage key
-        // - we're resetting secret storage
-        // - we don't have a default secret storage key yet
-        // - our default secret storage key is using an older algorithm
-        // We will also run this part if we created a new secret storage key
-        // above, so that we can (re-)encrypt the backup with it.
-        const defaultSSSSKey = await this.getSecretStorageKey();
-        if (setupNewSecretStorage || newKeyId || !defaultSSSSKey
-            || defaultSSSSKey[1].algorithm !== SECRET_STORAGE_ALGORITHM_V1_AES) {
-            if (keyBackupInfo) {
-                // if we already have a backup key, use the same key as the
-                // secret storage key
-                logger.log("Secret storage default key not found, using key backup key");
-
-                const backupKey = await getKeyBackupPassphrase();
-
-                if (!newKeyId) {
-                    const opts = {};
-
-                    if (
-                        keyBackupInfo.auth_data.private_key_salt &&
-                        keyBackupInfo.auth_data.private_key_iterations
-                    ) {
-                        opts.passphrase = {
-                            algorithm: "m.pbkdf2",
-                            iterations: keyBackupInfo.auth_data.private_key_iterations,
-                            salt: keyBackupInfo.auth_data.private_key_salt,
-                            bits: 256,
-                        };
-                    }
-
-                    // use the backup key as the new ssss key
-                    ssssKeys[newKeyId] = backupKey;
-                    opts.key = backupKey;
-
-                    newKeyId = await this.addSecretStorageKey(
-                        SECRET_STORAGE_ALGORITHM_V1_AES, opts,
-                    );
-                    await this.setDefaultSecretStorageKeyId(newKeyId);
-                }
-
-                // if this key backup is trusted, sign it with the cross signing key
-                // so the key backup can be trusted via cross-signing.
-                const backupSigStatus = await this.checkKeyBackup(keyBackupInfo);
-                if (backupSigStatus.trustInfo.usable) {
-                    logger.log("Adding cross signing signature to key backup");
-                    await this._crossSigningInfo.signObject(
-                        keyBackupInfo.auth_data, "master",
-                    );
-                    await this._baseApis._http.authedRequest(
-                        undefined, "PUT", "/room_keys/version/" + keyBackupInfo.version,
-                        undefined, keyBackupInfo,
-                        {prefix: httpApi.PREFIX_UNSTABLE},
-                    );
-                    await this.storeSecret(
-                        "m.megolm_backup.v1", olmlib.encodeBase64(backupKey), [newKeyId],
-                    );
-                } else {
-                    logger.log(
-                        "Key backup is NOT TRUSTED: NOT adding cross signing signature",
-                    );
-                }
-            } else {
-                if (!newKeyId) {
-                    logger.log("Secret storage default key not found, creating new key");
-                    const { keyInfo, privateKey } = await createSecretStorageKey();
-                    if (keyInfo && privateKey) {
-                        keyInfo.key = privateKey;
-                    }
-                    newKeyId = await this.addSecretStorageKey(
-                        SECRET_STORAGE_ALGORITHM_V1_AES,
-                        keyInfo,
-                    );
-                    await this.setDefaultSecretStorageKeyId(newKeyId);
-                    ssssKeys[newKeyId] = privateKey;
-                }
-                if (await this.isSecretStored("m.megolm_backup.v1")) {
-                    // we created a new SSSS, and we previously encrypted the
-                    // backup key with the old SSSS key, so re-encrypt with the
-                    // new key
-                    const backupKey = await this.getSecret("m.megolm_backup.v1");
-                    await this.storeSecret("m.megolm_backup.v1", backupKey, [newKeyId]);
-                }
+            if (oldKeyInfo && oldKeyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                // make sure that the default key has the information needed to
+                // check the passphrase
+                await ensureCanCheckPassphrase(oldKeyId, oldKeyInfo);
             }
-        } else {
-            logger.log("Have secret storage key");
         }
 
         // If cross-signing keys were reset, store them in Secure Secret Storage.
@@ -711,8 +770,18 @@ Crypto.prototype.bootstrapSecretStorage = async function({
         const sessionBackupKey = await this.getSecret('m.megolm_backup.v1');
         if (sessionBackupKey) {
             logger.info("Got session backup key from secret storage: caching");
-            const decoded = olmlib.decodeBase64(sessionBackupKey);
-            await this.storeSessionBackupPrivateKey(decoded);
+            // fix up the backup key if it's in the wrong format, and replace
+            // in secret storage
+            const fixedBackupKey = fixBackupKey(sessionBackupKey);
+            if (fixedBackupKey) {
+                await this.storeSecret(
+                    "m.megolm_backup.v1", fixedBackupKey, [newKeyId || oldKeyId],
+                );
+            }
+            const decodedBackupKey = new Uint8Array(olmlib.decodeBase64(
+                fixedBackupKey || sessionBackupKey,
+            ));
+            await this.storeSessionBackupPrivateKey(decodedBackupKey);
         }
     } finally {
         // Restore the original callbacks. NB. we must do this by manipulating
@@ -729,6 +798,26 @@ Crypto.prototype.bootstrapSecretStorage = async function({
     logger.log("Secure Secret Storage ready");
 };
 
+/**
+ * Fix up the backup key, that may be in the wrong format due to a bug in a
+ * migration step.  Some backup keys were stored as a comma-separated list of
+ * integers, rather than a base64-encoded byte array.  If this function is
+ * passed a string that looks like a list of integers rather than a base64
+ * string, it will attempt to convert it to the right format.
+ *
+ * @param {string} key the key to check
+ * @returns {null | string} If the key is in the wrong format, then the fixed
+ * key will be returned. Otherwise null will be returned.
+ *
+ */
+export function fixBackupKey(key) {
+    if (key.indexOf(",") < 0) {
+        return null;
+    }
+    const fixedKey = Uint8Array.from(key.split(","), x => parseInt(x));
+    return olmlib.encodeBase64(fixedKey);
+}
+
 Crypto.prototype.addSecretStorageKey = function(algorithm, opts, keyID) {
     return this._secretStorage.addKey(algorithm, opts, keyID);
 };
@@ -802,7 +891,7 @@ Crypto.prototype.checkSecretStoragePrivateKey = function(privateKey, expectedPub
  * @returns {Promise} the key, if any, or null
  */
 Crypto.prototype.getSessionBackupPrivateKey = async function() {
-    return new Promise((resolve) => {
+    let key = await new Promise((resolve) => {
         this._cryptoStore.doTxn(
             'readonly',
             [IndexedDBCryptoStore.STORE_ACCOUNT],
@@ -815,6 +904,13 @@ Crypto.prototype.getSessionBackupPrivateKey = async function() {
             },
         );
     });
+
+    // make sure we have a Uint8Array, rather than a string
+    if (key && typeof(key === "string")) {
+        key = new Uint8Array(olmlib.decodeBase64(fixBackupKey(key) || key));
+        await this.storeSessionBackupPrivateKey(key);
+    }
+    return key;
 };
 
 /**
@@ -1145,13 +1241,18 @@ Crypto.prototype._onDeviceListUserCrossSigningUpdated = async function(userId) {
             // If it's not changed, just make sure everything is up to date
             await this.checkOwnCrossSigningTrust();
         } else {
-            this.emit("crossSigning.keysChanged", {});
             // We'll now be in a state where cross-signing on the account is not trusted
             // because our locally stored cross-signing keys will not match the ones
-            // on the server for our account. The app must call checkOwnCrossSigningTrust()
-            // to fix this.
-            // XXX: Do we need to do something to emit events saying every device has become
-            // untrusted?
+            // on the server for our account. So we clear our own stored cross-signing keys,
+            // effectively disabling cross-signing until the user gets verified by the device
+            // that reset the keys
+            this._storeTrustedSelfKeys(null);
+            // emit cross-signing has been disabled
+            this.emit("crossSigning.keysChanged", {});
+            // as the trust for our own user has changed,
+            // also emit an event for this
+            this.emit("userTrustStatusChanged",
+                this._userId, this.checkUserTrust(userId));
         }
     } else {
         await this._checkDeviceVerifications(userId);
@@ -1307,7 +1408,11 @@ Crypto.prototype.checkOwnCrossSigningTrust = async function() {
  * @param {object} keys The new trusted set of keys
  */
 Crypto.prototype._storeTrustedSelfKeys = async function(keys) {
-    this._crossSigningInfo.setKeys(keys);
+    if (keys) {
+        this._crossSigningInfo.setKeys(keys);
+    } else {
+        this._crossSigningInfo.clearKeys();
+    }
     await this._cryptoStore.doTxn(
         'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
         (txn) => {
@@ -1916,6 +2021,10 @@ Crypto.prototype.setDeviceVerification = async function(
 
         if (!this._crossSigningInfo.getId() && userId === this._crossSigningInfo.userId) {
             this._storeTrustedSelfKeys(xsk.keys);
+            // This will cause our own user trust to change, so emit the event
+            this.emit(
+                "userTrustStatusChanged", this._userId, this.checkUserTrust(userId),
+            );
         }
 
         // Now sign the master key with our user signing key (unless it's ourself)
@@ -2058,7 +2167,8 @@ Crypto.prototype.requestVerification = function(userId, devices) {
     if (existingRequest) {
         return Promise.resolve(existingRequest);
     }
-    const channel = new ToDeviceChannel(this._baseApis, userId, devices);
+    const channel = new ToDeviceChannel(this._baseApis, userId, devices,
+        ToDeviceChannel.makeTransactionId());
     return this._requestVerificationWithChannel(
         userId,
         channel,
@@ -2071,6 +2181,10 @@ Crypto.prototype._requestVerificationWithChannel = async function(
 ) {
     let request = new VerificationRequest(
         channel, this._verificationMethods, this._baseApis);
+    // if transaction id is already known, add request
+    if (channel.transactionId) {
+        requestsMap.setRequestByChannel(channel, request);
+    }
     await request.sendRequest();
     // don't replace the request created by a racing remote echo
     const racingRequest = requestsMap.getRequestByChannel(channel);
@@ -2438,17 +2552,37 @@ Crypto.prototype.exportRoomKeys = async function() {
  * Import a list of room keys previously exported by exportRoomKeys
  *
  * @param {Object[]} keys a list of session export objects
+ * @param {Object} opts
+ * @param {Function} opts.progressCallback called with an object which has a stage param
  * @return {Promise} a promise which resolves once the keys have been imported
  */
-Crypto.prototype.importRoomKeys = function(keys) {
+Crypto.prototype.importRoomKeys = function(keys, opts = {}) {
+    let successes = 0;
+    let failures = 0;
+    const total = keys.length;
+
+    function updateProgress() {
+        opts.progressCallback({
+            stage: "load_keys",
+            successes,
+            failures,
+            total,
+        });
+    }
+
     return Promise.all(keys.map((key) => {
         if (!key.room_id || !key.algorithm) {
             logger.warn("ignoring room key entry with missing fields", key);
+            failures++;
+            if (opts.progressCallback) { updateProgress(); }
             return null;
         }
 
         const alg = this._getRoomDecryptor(key.room_id, key.algorithm);
-        return alg.importRoomKey(key);
+        return alg.importRoomKey(key).finally((r) => {
+            successes++;
+            if (opts.progressCallback) { updateProgress(); }
+        });
     }));
 };
 

src/crypto/verification/Base.js

@@ -122,6 +122,11 @@ export class VerificationBase extends EventEmitter {
         if (this._done) {
             return Promise.reject(new Error("Verification is already done"));
         }
+        const existingEvent = this.request.getEventFromOtherParty(type);
+        if (existingEvent) {
+            return Promise.resolve(existingEvent);
+        }
+
         this._expectedEvent = type;
         return new Promise((resolve, reject) => {
             this._resolveEvent = resolve;
@@ -287,6 +292,7 @@ export class VerificationBase extends EventEmitter {
         this._endTimer(); // always kill the activity timer
         if (!this._done) {
             this.cancelled = true;
+            this.request.onVerifierCancelled();
             if (this.userId && this.deviceId) {
                 // send a cancellation to the other user (if it wasn't
                 // cancelled by the other user)
@@ -369,7 +375,7 @@ export class VerificationBase extends EventEmitter {
 
         for (const [keyId, keyInfo] of Object.entries(keys)) {
             const deviceId = keyId.split(':', 2)[1];
-            const device = await this._baseApis.getStoredDevice(userId, deviceId);
+            const device = this._baseApis.getStoredDevice(userId, deviceId);
             if (device) {
                 await verifier(keyId, device, keyInfo);
                 verifiedDevices.push(deviceId);

src/crypto/verification/QRCode.js

@@ -65,20 +65,29 @@ export class ReciprocateQRCode extends Base {
             this.emit("show_reciprocate_qr", this.reciprocateQREvent);
         });
 
-        // 3. determine key to sign
+        // 3. determine key to sign / mark as trusted
         const keys = {};
-        if (qrCodeData.mode === MODE_VERIFY_OTHER_USER) {
-            // add master key to keys to be signed, only if we're not doing self-verification
-            const masterKey = qrCodeData.otherUserMasterKey;
-            keys[`ed25519:${masterKey}`] = masterKey;
-        } else if (qrCodeData.mode === MODE_VERIFY_SELF_TRUSTED) {
-            const deviceId = this.request.targetDevice.deviceId;
-            keys[`ed25519:${deviceId}`] = qrCodeData.otherDeviceKey;
-        } else {
-            // TODO: not sure if MODE_VERIFY_SELF_UNTRUSTED makes sense to sign anything here?
+
+        switch (qrCodeData.mode) {
+            case MODE_VERIFY_OTHER_USER: {
+                // add master key to keys to be signed, only if we're not doing self-verification
+                const masterKey = qrCodeData.otherUserMasterKey;
+                keys[`ed25519:${masterKey}`] = masterKey;
+                break;
+            }
+            case MODE_VERIFY_SELF_TRUSTED: {
+                const deviceId = this.request.targetDevice.deviceId;
+                keys[`ed25519:${deviceId}`] = qrCodeData.otherDeviceKey;
+                break;
+            }
+            case MODE_VERIFY_SELF_UNTRUSTED: {
+                const masterKey = qrCodeData.myMasterKey;
+                keys[`ed25519:${masterKey}`] = masterKey;
+                break;
+            }
         }
 
-        // 4. sign the key
+        // 4. sign the key (or mark own MSK as verified in case of MODE_VERIFY_SELF_TRUSTED)
         await this._verifyKeys(this.userId, keys, (keyId, device, keyInfo) => {
             // make sure the device has the expected keys
             const targetKey = keys[keyId];
@@ -108,11 +117,15 @@ const MODE_VERIFY_SELF_TRUSTED = 0x01; // We trust the master key
 const MODE_VERIFY_SELF_UNTRUSTED = 0x02; // We do not trust the master key
 
 export class QRCodeData {
-    constructor(mode, sharedSecret, otherUserMasterKey, otherDeviceKey, buffer) {
+    constructor(
+        mode, sharedSecret, otherUserMasterKey,
+        otherDeviceKey, myMasterKey, buffer,
+    ) {
         this._sharedSecret = sharedSecret;
         this._mode = mode;
         this._otherUserMasterKey = otherUserMasterKey;
         this._otherDeviceKey = otherDeviceKey;
+        this._myMasterKey = myMasterKey;
         this._buffer = buffer;
     }
 
@@ -121,22 +134,28 @@ export class QRCodeData {
         const mode = QRCodeData._determineMode(request, client);
         let otherUserMasterKey = null;
         let otherDeviceKey = null;
+        let myMasterKey = null;
         if (mode === MODE_VERIFY_OTHER_USER) {
             const otherUserCrossSigningInfo =
                 client.getStoredCrossSigningForUser(request.otherUserId);
             otherUserMasterKey = otherUserCrossSigningInfo.getId("master");
         } else if (mode === MODE_VERIFY_SELF_TRUSTED) {
             otherDeviceKey = await QRCodeData._getOtherDeviceKey(request, client);
+        } else if (mode === MODE_VERIFY_SELF_UNTRUSTED) {
+            const myUserId = client.getUserId();
+            const myCrossSigningInfo = client.getStoredCrossSigningForUser(myUserId);
+            myMasterKey = myCrossSigningInfo.getId("master");
         }
         const qrData = QRCodeData._generateQrData(
             request, client, mode,
             sharedSecret,
             otherUserMasterKey,
             otherDeviceKey,
+            myMasterKey,
         );
         const buffer = QRCodeData._generateBuffer(qrData);
         return new QRCodeData(mode, sharedSecret,
-            otherUserMasterKey, otherDeviceKey, buffer);
+            otherUserMasterKey, otherDeviceKey, myMasterKey, buffer);
     }
 
     get buffer() {
@@ -147,14 +166,30 @@ export class QRCodeData {
         return this._mode;
     }
 
+    /**
+     * only set when mode is MODE_VERIFY_SELF_TRUSTED
+     * @return {string} device key of other party at time of generating QR code
+     */
     get otherDeviceKey() {
         return this._otherDeviceKey;
     }
 
+    /**
+     * only set when mode is MODE_VERIFY_OTHER_USER
+     * @return {string} master key of other party at time of generating QR code
+     */
     get otherUserMasterKey() {
         return this._otherUserMasterKey;
     }
 
+    /**
+     * only set when mode is MODE_VERIFY_SELF_UNTRUSTED
+     * @return {string} own master key at time of generating QR code
+     */
+    get myMasterKey() {
+        return this._myMasterKey;
+    }
+
     /**
      * The unpadded base64 encoded shared secret.
      */
@@ -172,7 +207,7 @@ export class QRCodeData {
         const myUserId = client.getUserId();
         const otherDevice = request.targetDevice;
         const otherDeviceId = otherDevice ? otherDevice.deviceId : null;
-        const device = await client.getStoredDevice(myUserId, otherDeviceId);
+        const device = client.getStoredDevice(myUserId, otherDeviceId);
         if (!device) {
             throw new Error("could not find device " + otherDeviceId);
         }
@@ -198,7 +233,8 @@ export class QRCodeData {
     }
 
     static _generateQrData(request, client, mode,
-        encodedSharedSecret, otherUserMasterKey, otherDeviceKey,
+        encodedSharedSecret, otherUserMasterKey,
+        otherDeviceKey, myMasterKey,
     ) {
         const myUserId = client.getUserId();
         const transactionId = request.channel.transactionId;
@@ -213,16 +249,15 @@ export class QRCodeData {
         };
 
         const myCrossSigningInfo = client.getStoredCrossSigningForUser(myUserId);
-        const myMasterKey = myCrossSigningInfo.getId("master");
 
         if (mode === MODE_VERIFY_OTHER_USER) {
             // First key is our master cross signing key
-            qrData.firstKeyB64 = myMasterKey;
+            qrData.firstKeyB64 = myCrossSigningInfo.getId("master");
             // Second key is the other user's master cross signing key
             qrData.secondKeyB64 = otherUserMasterKey;
         } else if (mode === MODE_VERIFY_SELF_TRUSTED) {
             // First key is our master cross signing key
-            qrData.firstKeyB64 = myMasterKey;
+            qrData.firstKeyB64 = myCrossSigningInfo.getId("master");
             qrData.secondKeyB64 = otherDeviceKey;
         } else if (mode === MODE_VERIFY_SELF_UNTRUSTED) {
             // First key is our device's key

src/crypto/verification/SAS.js

@@ -295,7 +295,7 @@ export class SAS extends Base {
         const hashCommitment = content.commitment;
         const olmSAS = new global.Olm.SAS();
         try {
-            this._send("m.key.verification.key", {
+            await this._send("m.key.verification.key", {
                 key: olmSAS.get_pubkey(),
             });
 
@@ -318,9 +318,13 @@ export class SAS extends Base {
             const verifySAS = new Promise((resolve, reject) => {
                 this.sasEvent = {
                     sas: generateSas(sasBytes, sasMethods),
-                    confirm: () => {
-                        this._sendMAC(olmSAS, macMethod);
-                        resolve();
+                    confirm: async () => {
+                        try {
+                            await this._sendMAC(olmSAS, macMethod);
+                            resolve();
+                        } catch (err) {
+                            reject(err);
+                        }
                     },
                     cancel: () => reject(newUserCancelledError()),
                     mismatch: () => reject(newMismatchedSASError()),
@@ -377,7 +381,7 @@ export class SAS extends Base {
         const olmSAS = new global.Olm.SAS();
         try {
             const commitmentStr = olmSAS.get_pubkey() + anotherjson.stringify(content);
-            this._send("m.key.verification.accept", {
+            await this._send("m.key.verification.accept", {
                 key_agreement_protocol: keyAgreement,
                 hash: hashMethod,
                 message_authentication_code: macMethod,
@@ -391,7 +395,7 @@ export class SAS extends Base {
             // FIXME: make sure event is properly formed
             content = e.getContent();
             olmSAS.set_their_key(content.key);
-            this._send("m.key.verification.key", {
+            await this._send("m.key.verification.key", {
                 key: olmSAS.get_pubkey(),
             });
 
@@ -403,9 +407,13 @@ export class SAS extends Base {
             const verifySAS = new Promise((resolve, reject) => {
                 this.sasEvent = {
                     sas: generateSas(sasBytes, sasMethods),
-                    confirm: () => {
-                        this._sendMAC(olmSAS, macMethod);
-                        resolve();
+                    confirm: async () => {
+                        try {
+                            await this._sendMAC(olmSAS, macMethod);
+                            resolve();
+                        } catch(err) {
+                            reject(err);
+                        }
                     },
                     cancel: () => reject(newUserCancelledError()),
                     mismatch: () => reject(newMismatchedSASError()),
@@ -461,7 +469,7 @@ export class SAS extends Base {
             keyList.sort().join(","),
             baseInfo + "KEY_IDS",
         );
-        this._send("m.key.verification.mac", { mac, keys });
+        return this._send("m.key.verification.mac", { mac, keys });
     }
 
     async _checkMAC(olmSAS, content, method) {

src/crypto/verification/request/VerificationRequest.js

@@ -73,6 +73,7 @@ export class VerificationRequest extends EventEmitter {
         this._accepting = false;
         this._declining = false;
         this._verifierHasFinished = false;
+        this._cancelled = false;
         this._chosenMethod = null;
         // we keep a copy of the QR Code data (including other user master key) around
         // for QR reciprocate verification, to protect against
@@ -525,7 +526,7 @@ export class VerificationRequest extends EventEmitter {
         }
 
         const cancelEvent = this._getEventByEither(CANCEL_TYPE);
-        if (cancelEvent && phase() !== PHASE_DONE) {
+        if ((this._cancelled || cancelEvent) && phase() !== PHASE_DONE) {
             transitions.push({phase: PHASE_CANCELLED, event: cancelEvent});
             return transitions;
         }
@@ -858,6 +859,15 @@ export class VerificationRequest extends EventEmitter {
         return true;
     }
 
+    onVerifierCancelled() {
+        this._cancelled = true;
+        // move to cancelled phase
+        const newTransitions = this._applyPhaseTransitions();
+        if (newTransitions.length) {
+            this._setPhase(newTransitions[newTransitions.length - 1].phase);
+        }
+    }
+
     onVerifierFinished() {
         this.channel.send("m.key.verification.done", {});
         this._verifierHasFinished = true;
@@ -867,4 +877,8 @@ export class VerificationRequest extends EventEmitter {
             this._setPhase(newTransitions[newTransitions.length - 1].phase);
         }
     }
+
+    getEventFromOtherParty(type) {
+        return this._eventsByThem.get(type);
+    }
 }

src/filter.js

@@ -56,13 +56,6 @@ Filter.LAZY_LOADING_MESSAGES_FILTER = {
     lazy_load_members: true,
 };
 
-Filter.LAZY_LOADING_SYNC_FILTER = {
-    room: {
-        state: Filter.LAZY_LOADING_MESSAGES_FILTER,
-    },
-};
-
-
 /**
  * Get the ID of this filter on your homeserver (if known)
  * @return {?Number} The filter ID
@@ -96,6 +89,7 @@ Filter.prototype.setDefinition = function(definition) {
     //     "state": {
     //       "types": ["m.room.*"],
     //       "not_rooms": ["!726s6s6q:example.com"],
+    //       "lazy_load_members": true,
     //     },
     //     "timeline": {
     //       "limit": 10,
@@ -177,6 +171,10 @@ Filter.prototype.setTimelineLimit = function(limit) {
     setProp(this.definition, "room.timeline.limit", limit);
 };
 
+Filter.prototype.setLazyLoadMembers = function(enabled) {
+    setProp(this.definition, "room.state.lazy_load_members", !!enabled);
+};
+
 /**
  * Control whether left rooms should be included in responses.
  * @param {boolean} includeLeave True to make rooms the user has left appear

src/http-api.js

@@ -276,6 +276,9 @@ MatrixHttpApi.prototype = {
                         callbacks.clearTimeout(xhr.timeout_timer);
                         var resp;
                         try {
+                            if (xhr.status === 0) {
+                                throw new AbortError();
+                            }
                             if (!xhr.responseText) {
                                 throw new Error('No response body.');
                             }
@@ -789,6 +792,17 @@ const requestCallback = function(
     userDefinedCallback = userDefinedCallback || function() {};
 
     return function(err, response, body) {
+        if (err) {
+            // the unit tests use matrix-mock-request, which throw the string "aborted" when aborting a request.
+            // See https://github.com/matrix-org/matrix-mock-request/blob/3276d0263a561b5b8326b47bae720578a2c7473a/src/index.js#L48
+            const aborted = err.name === "AbortError" || err === "aborted";
+            if (!aborted && !(err instanceof MatrixError)) {
+                // browser-request just throws normal Error objects,
+                // not `TypeError`s like fetch does. So just assume any
+                // error is due to the connection.
+                err = new ConnectionError("request failed", err);
+            }
+        }
         if (!err) {
             try {
                 if (response.statusCode >= 400) {
@@ -892,12 +906,76 @@ function getResponseContentType(response) {
  * @prop {Object} data The raw Matrix error JSON used to construct this object.
  * @prop {integer} httpStatus The numeric HTTP status code given
  */
-export function MatrixError(errorJson) {
-    errorJson = errorJson || {};
-    this.errcode = errorJson.errcode;
-    this.name = errorJson.errcode || "Unknown error code";
-    this.message = errorJson.error || "Unknown message";
-    this.data = errorJson;
+export class MatrixError extends Error {
+    constructor(errorJson) {
+        errorJson = errorJson || {};
+        super(`MatrixError: ${errorJson.errcode}`);
+        this.errcode = errorJson.errcode;
+        this.name = errorJson.errcode || "Unknown error code";
+        this.message = errorJson.error || "Unknown message";
+        this.data = errorJson;
+    }
+}
+
+/**
+ * Construct a ConnectionError. This is a JavaScript Error indicating
+ * that a request failed because of some error with the connection, either
+ * CORS was not correctly configured on the server, the server didn't response,
+ * the request timed out, or the internet connection on the client side went down.
+ * @constructor
+ */
+export class ConnectionError extends Error {
+    constructor(message, cause = undefined) {
+        super(message + (cause ? `: ${cause.message}` : ""));
+        this._cause = cause;
+    }
+
+    get name() {
+        return "ConnectionError";
+    }
+
+    get cause() {
+        return this._cause;
+    }
+}
+
+export class AbortError extends Error {
+    constructor() {
+        super("Operation aborted");
+    }
+
+    get name() {
+        return "AbortError";
+    }
+}
+
+/**
+ * Retries a network operation run in a callback.
+ * @param  {number}   maxAttempts maximum attempts to try
+ * @param  {Function} callback    callback that returns a promise of the network operation. If rejected with ConnectionError, it will be retried by calling the callback again.
+ * @return {any} the result of the network operation
+ * @throws {ConnectionError} If after maxAttempts the callback still throws ConnectionError
+ */
+export async function retryNetworkOperation(maxAttempts, callback) {
+    let attempts = 0;
+    let lastConnectionError = null;
+    while (attempts < maxAttempts) {
+        try {
+            if (attempts > 0) {
+                const timeout = 1000 * Math.pow(2, attempts);
+                console.log(`network operation failed ${attempts} times,` +
+                    ` retrying in ${timeout}ms...`);
+                await new Promise(r => setTimeout(r, timeout));
+            }
+            return await callback();
+        } catch (err) {
+            if (err instanceof ConnectionError) {
+                attempts += 1;
+                lastConnectionError = err;
+            } else {
+                throw err;
+            }
+        }
+    }
+    throw lastConnectionError;
 }
-MatrixError.prototype = Object.create(Error.prototype);
-MatrixError.prototype.constructor = MatrixError;

src/interactive-auth.js

@@ -351,7 +351,13 @@ InteractiveAuth.prototype = {
                 error.data.session = this._data.session;
             }
             this._data = error.data;
-            this._startNextAuthStage();
+            try {
+                this._startNextAuthStage();
+            } catch (e) {
+                this._rejectFunc(e);
+                this._resolveFunc = null;
+                this._rejectFunc = null;
+            }
 
             if (
                 !this._emailSid &&

src/models/room.js

@@ -675,7 +675,7 @@ Room.prototype.hasUnverifiedDevices = async function() {
     }
     const e2eMembers = await this.getEncryptionTargetMembers();
     for (const member of e2eMembers) {
-        const devices = await this._client.getStoredDevicesForUser(member.userId);
+        const devices = this._client.getStoredDevicesForUser(member.userId);
         if (devices.some((device) => device.isUnverified())) {
             return true;
         }

src/store/memory.js

@@ -25,6 +25,15 @@ limitations under the License.
 import {User} from "../models/user";
 import * as utils from "../utils";
 
+function isValidFilterId(filterId) {
+    const isValidStr = typeof filterId === "string" &&
+        !!filterId &&
+        filterId !== "undefined" && // exclude these as we've serialized undefined in localStorage before
+        filterId !== "null";
+
+    return isValidStr || typeof filterId === "number";
+}
+
 /**
  * Construct a new in-memory data store for the Matrix Client.
  * @constructor
@@ -273,8 +282,17 @@ MemoryStore.prototype = {
         if (!this.localStorage) {
             return null;
         }
+        const key = "mxjssdk_memory_filter_" + filterName;
+        // XXX Storage.getItem doesn't throw ...
+        // or are we using something different
+        // than window.localStorage in some cases
+        // that does throw?
+        // that would be very naughty
         try {
-            return this.localStorage.getItem("mxjssdk_memory_filter_" + filterName);
+            const value = this.localStorage.getItem(key);
+            if (isValidFilterId(value)) {
+                return value;
+            }
         } catch (e) {}
         return null;
     },
@@ -288,8 +306,13 @@ MemoryStore.prototype = {
         if (!this.localStorage) {
             return;
         }
+        const key = "mxjssdk_memory_filter_" + filterName;
         try {
-            this.localStorage.setItem("mxjssdk_memory_filter_" + filterName, filterId);
+            if (isValidFilterId(filterId)) {
+                this.localStorage.setItem(key, filterId);
+            } else {
+                this.localStorage.removeItem(key);
+            }
         } catch (e) {}
     },
 

src/sync.js

@@ -511,6 +511,12 @@ SyncApi.prototype.sync = function() {
         checkLazyLoadStatus(); // advance to the next stage
     }
 
+    function buildDefaultFilter() {
+        const filter = new Filter(client.credentials.userId);
+        filter.setTimelineLimit(self.opts.initialSyncLimit);
+        return filter;
+    }
+
     const checkLazyLoadStatus = async () => {
         debuglog("Checking lazy load status...");
         if (this.opts.lazyLoadMembers && client.isGuest()) {
@@ -520,19 +526,11 @@ SyncApi.prototype.sync = function() {
             debuglog("Checking server lazy load support...");
             const supported = await client.doesServerSupportLazyLoading();
             if (supported) {
-                try {
-                    debuglog("Creating and storing lazy load sync filter...");
-                    this.opts.filter = await client.createFilter(
-                        Filter.LAZY_LOADING_SYNC_FILTER,
-                    );
-                    debuglog("Created and stored lazy load sync filter");
-                } catch (err) {
-                    logger.error(
-                        "Creating and storing lazy load sync filter failed",
-                        err,
-                    );
-                    throw err;
+                debuglog("Enabling lazy load on sync filter...");
+                if (!this.opts.filter) {
+                    this.opts.filter = buildDefaultFilter();
                 }
+                this.opts.filter.setLazyLoadMembers(true);
             } else {
                 debuglog("LL: lazy loading requested but not supported " +
                     "by server, so disabling");
@@ -575,8 +573,7 @@ SyncApi.prototype.sync = function() {
         if (self.opts.filter) {
             filter = self.opts.filter;
         } else {
-            filter = new Filter(client.credentials.userId);
-            filter.setTimelineLimit(self.opts.initialSyncLimit);
+            filter = buildDefaultFilter();
         }
 
         let filterId;

src/utils.ts

@@ -21,6 +21,7 @@ limitations under the License.
  */
 
 import unhomoglyph from 'unhomoglyph';
+import {ConnectionError} from "./http-api";
 
 /**
  * Encode a dictionary of query parameters.