v6.0.0-rc.1

Add initialFetch param to willUpdateDevices / devicesUpdated

CHANGELOG.md

@@ -1,3 +1,192 @@
+Changes in [6.0.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0-rc.1) (2020-04-30)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.4...v6.0.0-rc.1)
+
+BREAKING CHANGES
+---
+
+ * client.getStoredDevicesForUser and client.getStoredDevices are no longer async
+
+All Changes
+---
+
+ * Add initialFetch param to willUpdateDevices / devicesUpdated
+   [\#1362](https://github.com/matrix-org/matrix-js-sdk/pull/1362)
+ * Fix race between sending .request and receiving .ready over to_device
+   [\#1361](https://github.com/matrix-org/matrix-js-sdk/pull/1361)
+ * Handle race between sending and await next event from other party
+   [\#1358](https://github.com/matrix-org/matrix-js-sdk/pull/1358)
+ * Add crypto.willUpdateDevices event and make
+   getStoredDevices/getStoredDevicesForUser synchronous
+   [\#1356](https://github.com/matrix-org/matrix-js-sdk/pull/1356)
+ * Remove redundant key backup setup path
+   [\#1355](https://github.com/matrix-org/matrix-js-sdk/pull/1355)
+
+Changes in [5.3.1-rc.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.4) (2020-04-23)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.3...v5.3.1-rc.4)
+
+ * Retry account data upload requests
+   [\#1347](https://github.com/matrix-org/matrix-js-sdk/pull/1347)
+ * Fix: handle filter not found
+   [\#1341](https://github.com/matrix-org/matrix-js-sdk/pull/1341)
+ * Make getAccountDataFromServer return null if not found
+   [\#1339](https://github.com/matrix-org/matrix-js-sdk/pull/1339)
+ * Fix setDefaultKeyId to fail if the request fails
+   [\#1337](https://github.com/matrix-org/matrix-js-sdk/pull/1337)
+ * Fix: don't do extra /filter request when enabling lazy loading of members
+   [\#1333](https://github.com/matrix-org/matrix-js-sdk/pull/1333)
+ * Reject attemptAuth promise if no auth flow found
+   [\#1331](https://github.com/matrix-org/matrix-js-sdk/pull/1331)
+ * Serialise Olm prekey decryptions
+   [\#1330](https://github.com/matrix-org/matrix-js-sdk/pull/1330)
+
+Changes in [5.3.1-rc.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.3) (2020-04-17)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.2...v5.3.1-rc.3)
+
+ * Fix cross-signing/SSSS reset
+   [\#1323](https://github.com/matrix-org/matrix-js-sdk/pull/1323)
+ * Fix: crash when backup key needs fixing from corruption issue
+   [\#1325](https://github.com/matrix-org/matrix-js-sdk/pull/1325)
+
+Changes in [5.3.1-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.2) (2020-04-16)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.1...v5.3.1-rc.2)
+
+ * Implement QR code reciprocate for self-verification with untrusted MSK
+   [\#1321](https://github.com/matrix-org/matrix-js-sdk/pull/1321)
+
+Changes in [5.3.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.1) (2020-04-15)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.0-rc.1...v5.3.1-rc.1)
+
+ * Adapt release script for riot-desktop
+   [\#1319](https://github.com/matrix-org/matrix-js-sdk/pull/1319)
+ * Fix: prevent spurious notifications from indexer
+   [\#1318](https://github.com/matrix-org/matrix-js-sdk/pull/1318)
+ * Always create our own user object
+   [\#1317](https://github.com/matrix-org/matrix-js-sdk/pull/1317)
+ * Fix incorrect backup key format in SSSS
+   [\#1311](https://github.com/matrix-org/matrix-js-sdk/pull/1311)
+ * Fix e2ee crash after refreshing after having received a cross-singing key
+   reset
+   [\#1315](https://github.com/matrix-org/matrix-js-sdk/pull/1315)
+ * Fix: catch send errors in SAS verifier
+   [\#1314](https://github.com/matrix-org/matrix-js-sdk/pull/1314)
+ * Clear cross-signing keys when detecting the keys have changed
+   [\#1312](https://github.com/matrix-org/matrix-js-sdk/pull/1312)
+ * Upgrade deps
+   [\#1310](https://github.com/matrix-org/matrix-js-sdk/pull/1310)
+
+Changes in [5.3.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.0-rc.1) (2020-04-08)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.2.0...v5.3.0-rc.1)
+
+ * Store key backup key in cache as Uint8Array
+   [\#1308](https://github.com/matrix-org/matrix-js-sdk/pull/1308)
+ * Use the correct request body for the /keys/query endpoint.
+   [\#1307](https://github.com/matrix-org/matrix-js-sdk/pull/1307)
+ * Avoid creating two devices on registration
+   [\#1305](https://github.com/matrix-org/matrix-js-sdk/pull/1305)
+ * Lower max-warnings to 81
+   [\#1306](https://github.com/matrix-org/matrix-js-sdk/pull/1306)
+ * Move key backup key creation before caching
+   [\#1303](https://github.com/matrix-org/matrix-js-sdk/pull/1303)
+ * Expose function to force-reset outgoing room key requests
+   [\#1298](https://github.com/matrix-org/matrix-js-sdk/pull/1298)
+ * Add isSelfVerification property to VerificationRequest
+   [\#1302](https://github.com/matrix-org/matrix-js-sdk/pull/1302)
+ * QR code reciprocation
+   [\#1297](https://github.com/matrix-org/matrix-js-sdk/pull/1297)
+ * Add ability to check symmetric SSSS key before we try to use it
+   [\#1294](https://github.com/matrix-org/matrix-js-sdk/pull/1294)
+ * Add some debug logging for events stuck to bottom of timeline
+   [\#1296](https://github.com/matrix-org/matrix-js-sdk/pull/1296)
+ * Fix: spontanous verification request cancellation under some circumstances
+   [\#1295](https://github.com/matrix-org/matrix-js-sdk/pull/1295)
+ * Receive private key for caching from the app layer
+   [\#1293](https://github.com/matrix-org/matrix-js-sdk/pull/1293)
+ * Track whether we have verified a user before
+   [\#1292](https://github.com/matrix-org/matrix-js-sdk/pull/1292)
+ * Fix: error during tests
+   [\#1222](https://github.com/matrix-org/matrix-js-sdk/pull/1222)
+ * Send .done event for to_device verification
+   [\#1288](https://github.com/matrix-org/matrix-js-sdk/pull/1288)
+ * Request the key backup key & restore backup
+   [\#1291](https://github.com/matrix-org/matrix-js-sdk/pull/1291)
+ * Make screen sharing works on Chrome using getDisplayMedia()
+   [\#1276](https://github.com/matrix-org/matrix-js-sdk/pull/1276)
+ * Fix isVerified returning false
+   [\#1289](https://github.com/matrix-org/matrix-js-sdk/pull/1289)
+ * Fix: verification gets cancelled when event gets duplicated
+   [\#1286](https://github.com/matrix-org/matrix-js-sdk/pull/1286)
+ * Use requestSecret on the client to request secrets
+   [\#1287](https://github.com/matrix-org/matrix-js-sdk/pull/1287)
+ * Allow guests to fetch TURN servers
+   [\#1277](https://github.com/matrix-org/matrix-js-sdk/pull/1277)
+
+Changes in [5.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.2.0) (2020-03-30)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.2.0-rc.1...v5.2.0)
+
+ * Fix isVerified returning false
+   [\#1290](https://github.com/matrix-org/matrix-js-sdk/pull/1290)
+
+Changes in [5.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.2.0-rc.1) (2020-03-26)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.1.1...v5.2.0-rc.1)
+
+ * Add a flag for whether cross signing signatures are trusted
+   [\#1285](https://github.com/matrix-org/matrix-js-sdk/pull/1285)
+ * Cache user and self signing keys during bootstrap
+   [\#1282](https://github.com/matrix-org/matrix-js-sdk/pull/1282)
+ * remove unnecessary promise
+   [\#1283](https://github.com/matrix-org/matrix-js-sdk/pull/1283)
+ * Functions to cache session backups key automatically
+   [\#1281](https://github.com/matrix-org/matrix-js-sdk/pull/1281)
+ * Add function for checking cross-signing is ready
+   [\#1279](https://github.com/matrix-org/matrix-js-sdk/pull/1279)
+ * Use symmetric encryption for SSSS
+   [\#1228](https://github.com/matrix-org/matrix-js-sdk/pull/1228)
+ * Migrate SSSS to use symmetric algorithm
+   [\#1238](https://github.com/matrix-org/matrix-js-sdk/pull/1238)
+ * Migration to symmetric SSSS
+   [\#1272](https://github.com/matrix-org/matrix-js-sdk/pull/1272)
+ * Reduce number of one-time-key requests
+   [\#1280](https://github.com/matrix-org/matrix-js-sdk/pull/1280)
+ * Fix: assume the requested method is supported by other party with to_device
+   [\#1275](https://github.com/matrix-org/matrix-js-sdk/pull/1275)
+ * Use checkDeviceTrust when computing untrusted devices
+   [\#1278](https://github.com/matrix-org/matrix-js-sdk/pull/1278)
+ * Add a store for backup keys
+   [\#1271](https://github.com/matrix-org/matrix-js-sdk/pull/1271)
+ * Upload only new device signature of master key
+   [\#1268](https://github.com/matrix-org/matrix-js-sdk/pull/1268)
+ * Expose prepareToEncrypt in the client API
+   [\#1270](https://github.com/matrix-org/matrix-js-sdk/pull/1270)
+ * Don't kill the whole device download if one device gives an error
+   [\#1269](https://github.com/matrix-org/matrix-js-sdk/pull/1269)
+ * Handle racing .start event during self verification
+   [\#1267](https://github.com/matrix-org/matrix-js-sdk/pull/1267)
+ * A crypto.keySignatureUploadFailure event reported the wrong source
+   [\#1266](https://github.com/matrix-org/matrix-js-sdk/pull/1266)
+ * Fix editing of unsent messages by waiting for actual event id
+   [\#1263](https://github.com/matrix-org/matrix-js-sdk/pull/1263)
+ * Fix: ensureOlmSessionsForDevices parameter format
+   [\#1264](https://github.com/matrix-org/matrix-js-sdk/pull/1264)
+ * Remove stuff that yarn install doesn't think we need
+   [\#1261](https://github.com/matrix-org/matrix-js-sdk/pull/1261)
+ * Fix: prevent error being thrown during sync in some cases
+   [\#1258](https://github.com/matrix-org/matrix-js-sdk/pull/1258)
+ * Force `is_verified` for key backups to bool and fix computation
+   [\#1259](https://github.com/matrix-org/matrix-js-sdk/pull/1259)
+ * Add a method for legacy single device verification, returning a verification
+   request
+   [\#1257](https://github.com/matrix-org/matrix-js-sdk/pull/1257)
+ * yarn upgrade
+   [\#1256](https://github.com/matrix-org/matrix-js-sdk/pull/1256)
+
 Changes in [5.1.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.1.1) (2020-03-17)
 ================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.1.1-rc.1...v5.1.1)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-js-sdk",
-  "version": "5.1.1",
+  "version": "6.0.0-rc.1",
   "description": "Matrix Client-Server SDK for Javascript",
   "scripts": {
     "prepare": "yarn build",
@@ -14,7 +14,7 @@
     "build:minify-browser": "terser dist/browser-matrix.js --compress --mangle --source-map --output dist/browser-matrix.min.js",
     "gendoc": "jsdoc -c jsdoc.json -P package.json",
     "lint": "yarn lint:types && yarn lint:ts && yarn lint:js",
-    "lint:js": "eslint --max-warnings 93 src spec",
+    "lint:js": "eslint --max-warnings 81 src spec",
     "lint:types": "tsc --noEmit",
     "lint:ts": "tslint --project ./tsconfig.json -t stylish",
     "test": "jest spec/ --coverage --testEnvironment node",

release.sh

@@ -38,6 +38,7 @@ $USAGE
     -c changelog_file:  specify name of file containing changelog
     -x:                 skip updating the changelog
     -z:                 skip generating the jsdoc
+    -n:                 skip publish to NPM
 EOF
 }
 
@@ -60,9 +61,10 @@ fi
 
 skip_changelog=
 skip_jsdoc=
+skip_npm=
 changelog_file="CHANGELOG.md"
 expected_npm_user="matrixdotorg"
-while getopts hc:u:xz f; do
+while getopts hc:u:xzn f; do
     case $f in
         h)
             help
@@ -77,6 +79,9 @@ while getopts hc:u:xz f; do
         z)
             skip_jsdoc=1
             ;;
+        n)
+            skip_npm=1
+            ;;
         u)
             expected_npm_user="$OPTARG"
             ;;
@@ -96,10 +101,12 @@ fi
 
 # Login and publish continues to use `npm`, as it seems to have more clearly
 # defined options and semantics than `yarn` for writing to the registry.
-actual_npm_user=`npm whoami`;
-if [ $expected_npm_user != $actual_npm_user ]; then
-    echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
-    exit 1
+if [ -z "$skip_npm" ]; then
+    actual_npm_user=`npm whoami`;
+    if [ $expected_npm_user != $actual_npm_user ]; then
+        echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
+        exit 1
+    fi
 fi
 
 # ignore leading v on release
@@ -298,11 +305,13 @@ rm "${latest_changes}"
 # defined options and semantics than `yarn` for writing to the registry.
 # Tag both releases and prereleases as `next` so the last stable release remains
 # the default.
-npm publish --tag next
-if [ $prerelease -eq 0 ]; then
-    # For a release, also add the default `latest` tag.
-    package=$(cat package.json | jq -er .name)
-    npm dist-tag add "$package@$release" latest
+if [ -z "$skip_npm" ]; then
+    npm publish --tag next
+    if [ $prerelease -eq 0 ]; then
+        # For a release, also add the default `latest` tag.
+        package=$(cat package.json | jq -er .name)
+        npm dist-tag add "$package@$release" latest
+    fi
 fi
 
 if [ -z "$skip_jsdoc" ]; then
@@ -338,8 +347,10 @@ if [ -z "$skip_jsdoc" ]; then
     git push origin gh-pages
 fi
 
-# finally, merge master back onto develop
-git checkout develop
-git pull
-git merge master
-git push origin develop
+# finally, merge master back onto develop (if it exists)
+if [ $(git branch -lr | grep origin/develop -c) -ge 1 ]; then
+    git checkout develop
+    git pull
+    git merge master
+    git push origin develop
+fi

spec/TestClient.js

@@ -185,7 +185,7 @@ TestClient.prototype.expectKeyQuery = function(response) {
         200, (path, content) => {
             Object.keys(response.device_keys).forEach((userId) => {
                 expect(content.device_keys[userId]).toEqual(
-                    {},
+                    [],
                     "Expected key query for " + userId + ", got " +
                     Object.keys(content.device_keys),
                 );

spec/integ/matrix-client-crypto.spec.js

@@ -70,7 +70,7 @@ function expectAliQueryKeys() {
     aliTestClient.httpBackend.when("POST", "/keys/query")
             .respond(200, function(path, content) {
         expect(content.device_keys[bobUserId]).toEqual(
-            {},
+            [],
             "Expected Alice to key query for " + bobUserId + ", got " +
             Object.keys(content.device_keys),
         );
@@ -98,7 +98,7 @@ function expectBobQueryKeys() {
         "POST", "/keys/query",
     ).respond(200, function(path, content) {
         expect(content.device_keys[aliUserId]).toEqual(
-            {},
+            [],
             "Expected Bob to key query for " + aliUserId + ", got " +
             Object.keys(content.device_keys),
         );

spec/integ/matrix-client-methods.spec.js

@@ -347,8 +347,8 @@ describe("MatrixClient", function() {
 
             httpBackend.when("POST", "/keys/query").check(function(req) {
                 expect(req.data).toEqual({device_keys: {
-                    'boris': {},
-                    'chaz': {},
+                    'boris': [],
+                    'chaz': [],
                 }});
             }).respond(200, {
                 device_keys: {

spec/integ/matrix-client-opts.spec.js

@@ -3,6 +3,7 @@ import HttpBackend from "matrix-mock-request";
 import {MatrixClient} from "../../src/matrix";
 import {MatrixScheduler} from "../../src/scheduler";
 import {MemoryStore} from "../../src/store/memory";
+import {MatrixError} from "../../src/http-api";
 
 describe("MatrixClient opts", function() {
     const baseUrl = "http://localhost.or.something";
@@ -132,10 +133,10 @@ describe("MatrixClient opts", function() {
         });
 
         it("shouldn't retry sending events", function(done) {
-            httpBackend.when("PUT", "/txn1").fail(500, {
+            httpBackend.when("PUT", "/txn1").fail(500, new MatrixError({
                 errcode: "M_SOMETHING",
                 error: "Ruh roh",
-            });
+            }));
             client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                 expect(false).toBe(true, "sendTextMessage resolved but shouldn't");
             }, function(err) {

spec/unit/crypto/CrossSigningInfo.spec.js

@@ -217,6 +217,7 @@ describe.each([
     ["MemoryCryptoStore", () => {
         const store = new IndexedDBCryptoStore(undefined, "tests");
         store._backend = new MemoryCryptoStore();
+        store._backendPromise = Promise.resolve(store._backend);
         return store;
     }],
 ])("CrossSigning > createCryptoStoreCacheCallbacks [%s]", function(name, dbFactory) {

spec/unit/crypto/algorithms/megolm.spec.js

@@ -297,6 +297,10 @@ describe("MegolmDecryption", function() {
                 },
             }));
 
+            mockCrypto.checkDeviceTrust.mockReturnValue({
+                isVerified: () => false,
+            });
+
             const megolmEncryption = new MegolmEncryption({
                 userId: '@user:id',
                 crypto: mockCrypto,

spec/unit/crypto/backup.spec.js

@@ -541,5 +541,31 @@ describe("MegolmBackup", function() {
                 expect(res.clearEvent.content).toEqual('testytest');
             });
         });
+
+        it('has working cache functions', async function() {
+            const key = Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]);
+            await client._crypto.storeSessionBackupPrivateKey(key);
+            const result = await client._crypto.getSessionBackupPrivateKey();
+            expect(result).toEqual(key);
+        });
+
+        it('caches session backup keys as it encounters them', async function() {
+            const cachedNull = await client._crypto.getSessionBackupPrivateKey();
+            expect(cachedNull).toBeNull();
+            client._http.authedRequest = function() {
+                return Promise.resolve(KEY_BACKUP_DATA);
+            };
+            await new Promise((resolve) => {
+                client.restoreKeyBackupWithRecoveryKey(
+                    "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d",
+                    ROOM_ID,
+                    SESSION_ID,
+                    BACKUP_INFO,
+                    { cacheCompleteCallback: resolve },
+                );
+            });
+            const cachedKey = await client._crypto.getSessionBackupPrivateKey();
+            expect(cachedKey).not.toBeNull();
+        });
     });
 });

spec/unit/crypto/outgoing-room-key-requests.spec.js

@@ -0,0 +1,87 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {
+    IndexedDBCryptoStore,
+} from '../../../src/crypto/store/indexeddb-crypto-store';
+import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
+import 'fake-indexeddb/auto';
+import 'jest-localstorage-mock';
+
+import {
+    ROOM_KEY_REQUEST_STATES,
+} from '../../../src/crypto/OutgoingRoomKeyRequestManager';
+
+const requests = [
+    {
+        requestId: "A",
+        requestBody: { session_id: "A", room_id: "A" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "B",
+        requestBody: { session_id: "B", room_id: "B" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "C",
+        requestBody: { session_id: "C", room_id: "C" },
+        state: ROOM_KEY_REQUEST_STATES.UNSENT,
+    },
+];
+
+describe.each([
+    ["IndexedDBCryptoStore",
+     () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+    ["LocalStorageCryptoStore",
+     () => new IndexedDBCryptoStore(undefined, "tests")],
+    ["MemoryCryptoStore", () => {
+        const store = new IndexedDBCryptoStore(undefined, "tests");
+        store._backend = new MemoryCryptoStore();
+        store._backendPromise = Promise.resolve(store._backend);
+        return store;
+    }],
+])("Outgoing room key requests [%s]", function(name, dbFactory) {
+    let store;
+
+    beforeAll(async () => {
+        store = dbFactory();
+        await store.startup();
+        await Promise.all(requests.map((request) =>
+            store.getOrAddOutgoingRoomKeyRequest(request),
+        ));
+    });
+
+    it("getAllOutgoingRoomKeyRequestsByState retrieves all entries in a given state",
+    async () => {
+        const r = await
+            store.getAllOutgoingRoomKeyRequestsByState(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(r).toHaveLength(2);
+        requests.filter((e) => e.state == ROOM_KEY_REQUEST_STATES.SENT).forEach((e) => {
+            expect(r).toContainEqual(e);
+        });
+    });
+
+    test("getOutgoingRoomKeyRequestByState retrieves any entry in a given state",
+    async () => {
+        const r =
+            await store.getOutgoingRoomKeyRequestByState([ROOM_KEY_REQUEST_STATES.SENT]);
+        expect(r).not.toBeNull();
+        expect(r).not.toBeUndefined();
+        expect(r.state).toEqual(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(requests).toContainEqual(r);
+    });
+});

spec/unit/crypto/secrets.spec.js

@@ -16,10 +16,20 @@ limitations under the License.
 
 import '../../olm-loader';
 import * as olmlib from "../../../src/crypto/olmlib";
-import {SECRET_STORAGE_ALGORITHM_V1} from "../../../src/crypto/SecretStorage";
+import {SECRET_STORAGE_ALGORITHM_V1_AES} from "../../../src/crypto/SecretStorage";
 import {MatrixEvent} from "../../../src/models/event";
 import {TestClient} from '../../TestClient';
 import {makeTestClients} from './verification/util';
+import {encryptAES} from "../../../src/crypto/aes";
+
+import * as utils from "../../../src/utils";
+
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    console.log('nodejs was compiled without crypto support');
+}
 
 async function makeTestClient(userInfo, options) {
     const client = (new TestClient(
@@ -40,6 +50,13 @@ async function makeTestClient(userInfo, options) {
     return client;
 }
 
+// Wrapper around pkSign to return a signed object. pkSign returns the
+// signature, rather than the signed object.
+function sign(obj, key, userId) {
+    olmlib.pkSign(obj, key, userId);
+    return obj;
+}
+
 describe("Secrets", function() {
     if (!global.Olm) {
         console.warn('Not running megolm backup unit tests: libolm not present');
@@ -51,9 +68,8 @@ describe("Secrets", function() {
     });
 
     it("should store and retrieve a secret", async function() {
-        const decryption = new global.Olm.PkDecryption();
-        const pubkey = decryption.generate_key();
-        const privkey = decryption.get_private_key();
+        const key = new Uint8Array(16);
+        for (let i = 0; i < 16; i++) key[i] = i;
 
         const signing = new global.Olm.PkSigning();
         const signingKey = signing.generate_seed();
@@ -69,7 +85,7 @@ describe("Secrets", function() {
 
         const getKey = jest.fn(e => {
             expect(Object.keys(e.keys)).toEqual(["abc"]);
-            return ['abc', privkey];
+            return ['abc', key];
         });
 
         const alice = await makeTestClient(
@@ -100,8 +116,7 @@ describe("Secrets", function() {
         };
 
         const keyAccountData = {
-            algorithm: SECRET_STORAGE_ALGORITHM_V1,
-            pubkey: pubkey,
+            algorithm: SECRET_STORAGE_ALGORITHM_V1_AES,
         };
         await alice._crypto._crossSigningInfo.signObject(keyAccountData, 'master');
 
@@ -112,11 +127,11 @@ describe("Secrets", function() {
             }),
         ]);
 
-        expect(await secretStorage.isStored("foo")).toBe(false);
+        expect(await secretStorage.isStored("foo")).toBeFalsy();
 
         await secretStorage.store("foo", "bar", ["abc"]);
 
-        expect(await secretStorage.isStored("foo")).toBe(true);
+        expect(await secretStorage.isStored("foo")).toBeTruthy();
         expect(await secretStorage.get("foo")).toBe("bar");
 
         expect(getKey).toHaveBeenCalled();
@@ -149,6 +164,13 @@ describe("Secrets", function() {
     });
 
     it("should encrypt with default key if keys is null", async function() {
+        const key = new Uint8Array(16);
+        for (let i = 0; i < 16; i++) key[i] = i;
+        const getKey = jest.fn(e => {
+            expect(Object.keys(e.keys)).toEqual([newKeyId]);
+            return [newKeyId, key];
+        });
+
         let keys = {};
         const alice = await makeTestClient(
             {userId: "@alice:example.com", deviceId: "Osborne2"},
@@ -156,6 +178,7 @@ describe("Secrets", function() {
                 cryptoCallbacks: {
                     getCrossSigningKey: t => keys[t],
                     saveCrossSigningKeys: k => keys = k,
+                    getSecretStorageKey: getKey,
                 },
             },
         );
@@ -170,7 +193,7 @@ describe("Secrets", function() {
         alice.resetCrossSigningKeys();
 
         const newKeyId = await alice.addSecretStorageKey(
-            SECRET_STORAGE_ALGORITHM_V1,
+            SECRET_STORAGE_ALGORITHM_V1_AES,
         );
         // we don't await on this because it waits for the event to come down the sync
         // which won't happen in the test setup
@@ -251,89 +274,708 @@ describe("Secrets", function() {
         expect(secret).toBe("bar");
     });
 
-    it("bootstraps when no storage or cross-signing keys locally", async function() {
-        const bob = await makeTestClient(
-            {
-                userId: "@bob:example.com",
-                deviceId: "bob1",
-            },
+    describe("bootstrap", function() {
+        // keys used in some of the tests
+        const XSK = new Uint8Array(
+            olmlib.decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="),
         );
-        bob.uploadDeviceSigningKeys = async () => {};
-        bob.uploadKeySignatures = async () => {};
-        bob.setAccountData = async function(eventType, contents, callback) {
-            const event = new MatrixEvent({
-                type: eventType,
-                content: contents,
+        const XSPubKey = "DRb8pFVJyEJ9OWvXeUoM0jq/C2Wt+NxzBZVuk2nRb+0";
+        const USK = new Uint8Array(
+            olmlib.decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="),
+        );
+        const USPubKey = "CUpoiTtHiyXpUmd+3ohb7JVxAlUaOG1NYs9Jlx8soQU";
+        const SSK = new Uint8Array(
+            olmlib.decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="),
+        );
+        const SSPubKey = "0DfNsRDzEvkCLA0gD3m7VAGJ5VClhjEsewI35xq873Q";
+        const backupKey = new Uint8Array(
+            olmlib.decodeBase64(
+                "OTmOKrPsWP1qxmt0s2qat4vYxULG+U9cmFAWFco3A5U=",
+            ),
+        );
+        const SSSSKey = new Uint8Array(
+            olmlib.decodeBase64(
+                "XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0=",
+            ),
+        );
+        const SSSSPubKey = "v3A8HTypbccUm6jaRCRw5l+7lSdzQUACeY9xpQ5BVmE";
+
+        it("bootstraps when no storage or cross-signing keys locally", async function() {
+            const key = new Uint8Array(16);
+            for (let i = 0; i < 16; i++) key[i] = i;
+            const getKey = jest.fn(e => {
+                return [Object.keys(e.keys)[0], key];
             });
-            this.store.storeAccountDataEvents([
-                event,
-            ]);
-            this.emit("accountData", event);
-        };
 
-        await bob.bootstrapSecretStorage();
-
-        const crossSigning = bob._crypto._crossSigningInfo;
-        const secretStorage = bob._crypto._secretStorage;
-
-        expect(crossSigning.getId()).toBeTruthy();
-        expect(await crossSigning.isStoredInSecretStorage(secretStorage)).toBeTruthy();
-        expect(await secretStorage.hasKey()).toBeTruthy();
-    });
-
-    it("bootstraps when cross-signing keys in secret storage", async function() {
-        const decryption = new global.Olm.PkDecryption();
-        const storagePublicKey = decryption.generate_key();
-        const storagePrivateKey = decryption.get_private_key();
-
-        const bob = await makeTestClient(
-            {
-                userId: "@bob:example.com",
-                deviceId: "bob1",
-            },
-            {
-                cryptoCallbacks: {
-                    getSecretStorageKey: async request => {
-                        const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
-                        expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
-                        return [defaultKeyId, storagePrivateKey];
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: getKey,
                     },
                 },
-            },
-        );
+            );
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
 
-        bob.uploadDeviceSigningKeys = async () => {};
-        bob.uploadKeySignatures = async () => {};
-        bob.setAccountData = async function(eventType, contents, callback) {
-            const event = new MatrixEvent({
-                type: eventType,
-                content: contents,
-            });
-            this.store.storeAccountDataEvents([
-                event,
-            ]);
-            this.emit("accountData", event);
-        };
-        bob._crypto.checkKeyBackup = async () => {};
+            await bob.bootstrapSecretStorage();
 
-        const crossSigning = bob._crypto._crossSigningInfo;
-        const secretStorage = bob._crypto._secretStorage;
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;
 
-        // Set up cross-signing keys from scratch with specific storage key
-        await bob.bootstrapSecretStorage({
-            createSecretStorageKey: async () => ({ pubkey: storagePublicKey }),
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
         });
 
-        // Clear local cross-signing keys and read from secret storage
-        bob._crypto._deviceList.storeCrossSigningForUser(
-            "@bob:example.com",
-            crossSigning.toStorage(),
-        );
-        crossSigning.keys = {};
-        await bob.bootstrapSecretStorage();
+        it("bootstraps when cross-signing keys in secret storage", async function() {
+            const decryption = new global.Olm.PkDecryption();
+            const storagePublicKey = decryption.generate_key();
+            const storagePrivateKey = decryption.get_private_key();
 
-        expect(crossSigning.getId()).toBeTruthy();
-        expect(await crossSigning.isStoredInSecretStorage(secretStorage)).toBeTruthy();
-        expect(await secretStorage.hasKey()).toBeTruthy();
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: async request => {
+                            const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
+                            expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
+                            return [defaultKeyId, storagePrivateKey];
+                        },
+                    },
+                },
+            );
+
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
+            bob._crypto.checkKeyBackup = async () => {};
+
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;
+
+            // Set up cross-signing keys from scratch with specific storage key
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey: async () => ({
+                    // `pubkey` not used anymore with symmetric 4S
+                    keyInfo: { pubkey: storagePublicKey },
+                    privateKey: storagePrivateKey,
+                }),
+            });
+
+            // Clear local cross-signing keys and read from secret storage
+            bob._crypto._deviceList.storeCrossSigningForUser(
+                "@bob:example.com",
+                crossSigning.toStorage(),
+            );
+            crossSigning.keys = {};
+            await bob.bootstrapSecretStorage();
+
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
+        });
+
+        it("converts asymmetric SSSS to symmetric SSSS", async function() {
+            let crossSigningKeys = {};
+            const secretStorageKeys = {
+                "old_key_id": SSSSKey,
+                "new_key_id": SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            const encryption = new global.Olm.PkEncryption();
+            encryption.set_recipient_key(SSSSPubKey);
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "old_key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.old_key_id",
+                    content: sign({
+                        algorithm: "m.secret_storage.v1.curve25519-aes-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                        pubkey: SSSSPubKey,
+                    }, XSK, "@alice:example.com"),
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(olmlib.encodeBase64(XSK)),
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(olmlib.encodeBase64(SSK)),
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(olmlib.encodeBase64(USK)),
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.megolm_backup.v1",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(
+                                olmlib.encodeBase64(backupKey),
+                            ),
+                        },
+                    },
+                }),
+            ]);
+            encryption.free();
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            const origAddSecretStorageKey = alice._crypto.addSecretStorageKey;
+            alice._crypto.addSecretStorageKey = async function(algorithm, opts, keyId) {
+                return await origAddSecretStorageKey.call(
+                    alice._crypto, algorithm, opts, keyId || "new_key_id",
+                );
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            // it should create a new key with the same parameters
+            expect(alice.getAccountData("m.secret_storage.default_key").getContent())
+                .toEqual({key: "new_key_id"});
+            const newKeyInfo = alice.getAccountData("m.secret_storage.key.new_key_id")
+                .getContent();
+            expect(newKeyInfo.algorithm)
+                .toEqual("m.secret_storage.v1.aes-hmac-sha2");
+            expect(newKeyInfo.passphrase).toEqual({
+                algorithm: "m.pbkdf2",
+                iterations: 500000,
+                salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+            });
+            expect(newKeyInfo).toHaveProperty("iv");
+            expect(newKeyInfo).toHaveProperty("mac");
+            expect(alice.checkSecretStorageKey(secretStorageKeys.new_key_id, newKeyInfo))
+                .toBeTruthy();
+
+            const crossSigningMaster = alice.getAccountData("m.cross_signing.master")
+                .getContent();
+            expect(crossSigningMaster.encrypted).toHaveProperty("new_key_id");
+            expect(crossSigningMaster.encrypted).not.toHaveProperty("old_key_id");
+            expect(await alice.getSecret("m.cross_signing.master"))
+                .toEqual(olmlib.encodeBase64(XSK));
+
+            const USKInfo = alice.getAccountData("m.cross_signing.self_signing")
+                .getContent();
+            expect(USKInfo.encrypted).toHaveProperty("new_key_id");
+            expect(USKInfo.encrypted).not.toHaveProperty("old_key_id");
+            expect(await alice.getSecret("m.cross_signing.user_signing"))
+                .toEqual(olmlib.encodeBase64(USK));
+
+            const SSKInfo = alice.getAccountData("m.cross_signing.self_signing")
+                .getContent();
+            expect(SSKInfo.encrypted).toHaveProperty("new_key_id");
+            expect(SSKInfo.encrypted).not.toHaveProperty("old_key_id");
+            expect(await alice.getSecret("m.cross_signing.self_signing"))
+                .toEqual(olmlib.encodeBase64(SSK));
+
+            const backupKeyInfo = alice.getAccountData("m.megolm_backup.v1")
+                .getContent();
+            expect(backupKeyInfo.encrypted).toHaveProperty("new_key_id");
+            expect(backupKeyInfo.encrypted).not.toHaveProperty("old_key_id");
+            expect(await alice.getSecret("m.megolm_backup.v1"))
+                .toEqual(olmlib.encodeBase64(backupKey));
+        });
+        it("converts asymmetric SSSS with passthrough", async function() {
+            let crossSigningKeys = {};
+            const secretStorageKeys = {
+                "old_key_id": SSSSKey,
+                "new_key_id": SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            const encryption = new global.Olm.PkEncryption();
+            encryption.set_recipient_key(SSSSPubKey);
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "old_key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.old_key_id",
+                    content: sign({
+                        algorithm: "m.secret_storage.v1.curve25519-aes-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                        pubkey: "v3A8HTypbccUm6jaRCRw5l+7lSdzQUACeY9xpQ5BVmE",
+                    }, XSK, "@alice:example.com"),
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(olmlib.encodeBase64(XSK)),
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(olmlib.encodeBase64(SSK)),
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            old_key_id: encryption.encrypt(olmlib.encodeBase64(USK)),
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.megolm_backup.v1",
+                    content: {
+                        encrypted: {
+                            old_key_id: {
+                                passthrough: true,
+                            },
+                        },
+                    },
+                }),
+            ]);
+            encryption.free();
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "v3A8HTypbccUm6jaRCRw5l+7lSdzQUACeY9xpQ5BVmE",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            const origAddSecretStorageKey = alice._crypto.addSecretStorageKey;
+            alice._crypto.addSecretStorageKey = async function(algorithm, opts, keyId) {
+                return await origAddSecretStorageKey.call(
+                    alice._crypto, algorithm, opts, keyId || "new_key_id",
+                );
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            // the new backup key should be the encoded SSSS key
+            const backupKeyInfo = alice.getAccountData("m.megolm_backup.v1")
+                .getContent();
+            expect(backupKeyInfo.encrypted).toHaveProperty("new_key_id");
+            expect(backupKeyInfo.encrypted).not.toHaveProperty("old_key_id");
+            expect(await alice.getSecret("m.megolm_backup.v1"))
+                .toEqual(olmlib.encodeBase64(SSSSKey));
+        });
+        it("adds passphrase checking if it's lacking", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                // we never use these values, other than checking that they
+                // exist, so just use dummy values
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            expect(alice.getAccountData("m.secret_storage.default_key").getContent())
+                .toEqual({key: "key_id"});
+            const keyInfo = alice.getAccountData("m.secret_storage.key.key_id")
+                .getContent();
+            expect(keyInfo.algorithm)
+                .toEqual("m.secret_storage.v1.aes-hmac-sha2");
+            expect(keyInfo.passphrase).toEqual({
+                algorithm: "m.pbkdf2",
+                iterations: 500000,
+                salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+            });
+            expect(keyInfo).toHaveProperty("iv");
+            expect(keyInfo).toHaveProperty("mac");
+            expect(alice.checkSecretStorageKey(secretStorageKeys.key_id, keyInfo))
+                .toBeTruthy();
+        });
+        it("fixes backup keys in the wrong format", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.megolm_backup.v1",
+                    content: {
+                        encrypted: {
+                            key_id: await encryptAES(
+                                "123,45,6,7,89,1,234,56,78,90,12,34,5,67,8,90",
+                                secretStorageKeys.key_id, "m.megolm_backup.v1",
+                            ),
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            const backupKey = alice.getAccountData("m.megolm_backup.v1")
+                .getContent();
+            expect(backupKey.encrypted).toHaveProperty("key_id");
+            expect(await alice.getSecret("m.megolm_backup.v1"))
+                .toEqual("ey0GB1kB6jhOWgwiBUMIWg==");
+        });
     });
 });

spec/unit/crypto/verification/sas.spec.js

@@ -44,7 +44,16 @@ describe("SAS verification", function() {
     });
 
     it("should error on an unexpected event", async function() {
-        const sas = new SAS({}, "@alice:example.com", "ABCDEFG");
+        //channel, baseApis, userId, deviceId, startEvent, request
+        const request = {
+            onVerifierCancelled: function() {},
+        };
+        const channel = {
+            send: function() {
+                return Promise.resolve();
+            },
+        };
+        const sas = new SAS(channel, {}, "@alice:example.com", "ABCDEFG", null, request);
         sas.handleEvent(new MatrixEvent({
             sender: "@alice:example.com",
             type: "es.inquisition",

spec/unit/crypto/verification/secret_request.spec.js

@@ -64,12 +64,20 @@ describe("self-verifications", () => {
             }),
         };
 
+        const storeSessionBackupPrivateKey = jest.fn();
+        const restoreKeyBackupWithCache = jest.fn(() => Promise.resolve());
+
         const client = {
             _crypto: {
                 _crossSigningInfo,
                 _secretStorage,
+                storeSessionBackupPrivateKey,
+                getSessionBackupPrivateKey: () => null,
             },
+            requestSecret: _secretStorage.request.bind(_secretStorage),
             getUserId: () => userId,
+            getKeyBackupVersion: () => Promise.resolve({}),
+            restoreKeyBackupWithCache,
         };
 
         const request = {
@@ -88,15 +96,20 @@ describe("self-verifications", () => {
 
         const result = await verification.done();
 
-        /* We should request, and store, two keys */
+        /* We should request, and store, two cross signing key and the key backup key */
         expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls.length).toBe(2);
-        expect(_secretStorage.request.mock.calls.length).toBe(2);
+        expect(_secretStorage.request.mock.calls.length).toBe(3);
 
         expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[0][1])
           .toEqual(testKey);
         expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[1][1])
           .toEqual(testKey);
 
+        expect(storeSessionBackupPrivateKey.mock.calls[0][0])
+          .toEqual(testKey);
+
+        expect(restoreKeyBackupWithCache).toHaveBeenCalled();
+
         expect(result).toBeInstanceOf(Array);
         expect(result[0][0]).toBe(testKeyPub);
         expect(result[1][0]).toBe(testKeyPub);

spec/unit/interactive-auth.spec.js

@@ -143,4 +143,33 @@ describe("InteractiveAuth", function() {
             expect(stateUpdated).toBeCalledTimes(1);
         });
     });
+
+    it("should start an auth stage and reject if no auth flow", function() {
+        const doRequest = jest.fn();
+        const stateUpdated = jest.fn();
+
+        const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
+            doRequest: doRequest,
+            stateUpdated: stateUpdated,
+        });
+
+        doRequest.mockImplementation(function(authData) {
+            logger.log("request1", authData);
+            expect(authData).toEqual({});
+            const err = new MatrixError({
+                session: "sessionId",
+                flows: [],
+                params: {
+                    "logintype": { param: "aa" },
+                },
+            });
+            err.httpStatus = 401;
+            throw err;
+        });
+
+        return ia.attemptAuth().catch(function(error) {
+            expect(error.message).toBe('No appropriate authentication flow found');
+        });
+    });
 });

src/autodiscovery.js

@@ -502,10 +502,12 @@ export class AutoDiscovery {
             request(
                 { method: "GET", uri: url, timeout: 5000 },
                 (err, response, body) => {
-                    if (err || response.statusCode < 200 || response.statusCode >= 300) {
+                    if (err || response &&
+                        (response.statusCode < 200 || response.statusCode >= 300)
+                    ) {
                         let action = "FAIL_PROMPT";
                         let reason = (err ? err.message : null) || "General failure";
-                        if (response.statusCode === 404) {
+                        if (response && response.statusCode === 404) {
                             action = "IGNORE";
                             reason = AutoDiscovery.ERROR_MISSING_WELLKNOWN;
                         }

src/base-apis.js

@@ -1757,7 +1757,7 @@ MatrixBaseApis.prototype.downloadKeysForUsers = function(userIds, opts) {
         content.token = opts.token;
     }
     userIds.forEach((u) => {
-        content.device_keys[u] = {};
+        content.device_keys[u] = [];
     });
 
     return this._http.authedRequest(undefined, "POST", "/keys/query", undefined, content);

src/client.js

@@ -35,19 +35,25 @@ import {StubStore} from "./store/stub";
 import {createNewMatrixCall} from "./webrtc/call";
 import * as utils from './utils';
 import {sleep} from './utils';
-import {MatrixError, PREFIX_MEDIA_R0, PREFIX_UNSTABLE} from "./http-api";
+import {
+    MatrixError,
+    PREFIX_MEDIA_R0,
+    PREFIX_UNSTABLE,
+    retryNetworkOperation,
+} from "./http-api";
 import {getHttpUriForMxc} from "./content-repo";
 import * as ContentHelpers from "./content-helpers";
 import * as olmlib from "./crypto/olmlib";
 import {ReEmitter} from './ReEmitter';
 import {RoomList} from './crypto/RoomList';
 import {logger} from './logger';
-import {Crypto, isCryptoAvailable} from './crypto';
+import {Crypto, isCryptoAvailable, fixBackupKey} from './crypto';
 import {decodeRecoveryKey} from './crypto/recoverykey';
 import {keyFromAuthData} from './crypto/key_passphrase';
 import {randomString} from './randomstring';
 import {PushProcessor} from "./pushprocessor";
 import {encodeBase64, decodeBase64} from "./crypto/olmlib";
+import { User } from "./models/user";
 
 const SCROLLBACK_DELAY_MS = 3000;
 export const CRYPTO_ENABLED = isCryptoAvailable();
@@ -217,8 +223,10 @@ function keyFromRecoverySession(session, decryptionKey) {
  * Args:
  *   {object} keys Information about the keys:
  *       {
- *           <key name>: {
- *               pubkey: {UInt8Array}
+ *           keys: {
+ *               <key name>: {
+ *                   pubkey: {UInt8Array}
+ *               }, ...
  *           }
  *       }
  *   {string} name the name of the value we want to read out of SSSS, for UI purposes.
@@ -281,13 +289,19 @@ export function MatrixClient(opts) {
     this.scheduler = opts.scheduler;
     if (this.scheduler) {
         const self = this;
-        this.scheduler.setProcessFunction(function(eventToSend) {
+        this.scheduler.setProcessFunction(async function(eventToSend) {
             const room = self.getRoom(eventToSend.getRoomId());
             if (eventToSend.status !== EventStatus.SENDING) {
                 _updatePendingEventStatus(room, eventToSend,
                                           EventStatus.SENDING);
             }
-            return _sendEventHttpRequest(self, eventToSend);
+            const res = await _sendEventHttpRequest(self, eventToSend);
+            if (room) {
+                // ensure we update pending event before the next scheduler run so that any listeners to event id
+                // updates on the synchronous event emitter get a chance to run first.
+                room.updatePendingEvent(eventToSend, EventStatus.SENT, res.event_id);
+            }
+            return res;
         });
     }
     this.clientRunning = false;
@@ -723,6 +737,7 @@ MatrixClient.prototype.initCrypto = async function() {
         "crypto.roomKeyRequestCancellation",
         "crypto.warning",
         "crypto.devicesUpdated",
+        "crypto.willUpdateDevices",
         "deviceVerificationChanged",
         "userTrustStatusChanged",
         "crossSigning.keysChanged",
@@ -752,7 +767,6 @@ MatrixClient.prototype.isCryptoEnabled = function() {
     return this._crypto !== null;
 };
 
-
 /**
  * Get the Ed25519 key for this device
  *
@@ -812,9 +826,9 @@ MatrixClient.prototype.downloadKeys = function(userIds, forceDownload) {
  *
  * @param {string} userId the user to list keys for.
  *
- * @return {Promise<module:crypto/deviceinfo[]>} list of devices
+ * @return {module:crypto/deviceinfo[]} list of devices
  */
-MatrixClient.prototype.getStoredDevicesForUser = async function(userId) {
+MatrixClient.prototype.getStoredDevicesForUser = function(userId) {
     if (this._crypto === null) {
         throw new Error("End-to-end encryption disabled");
     }
@@ -827,9 +841,9 @@ MatrixClient.prototype.getStoredDevicesForUser = async function(userId) {
  * @param {string} userId the user to list keys for.
  * @param {string} deviceId unique identifier for the device
  *
- * @return {Promise<?module:crypto/deviceinfo>} device or null
+ * @return {module:crypto/deviceinfo} device or null
  */
-MatrixClient.prototype.getStoredDevice = async function(userId, deviceId) {
+MatrixClient.prototype.getStoredDevice = function(userId, deviceId) {
     if (this._crypto === null) {
         throw new Error("End-to-end encryption disabled");
     }
@@ -1127,6 +1141,12 @@ function wrapCryptoFuncs(MatrixClient, names) {
  * @returns {boolean} true if the key matches, otherwise false
  */
 
+/**
+ * Perform any background tasks that can be done before a message is ready to
+ * send, in order to speed up sending of the message.
+ *
+ * @param {module:models/room} room the room the event is in
+ */
 wrapCryptoFuncs(MatrixClient, [
     "resetCrossSigningKeys",
     "getCrossSigningId",
@@ -1135,6 +1155,11 @@ wrapCryptoFuncs(MatrixClient, [
     "checkDeviceTrust",
     "checkOwnCrossSigningTrust",
     "checkCrossSigningPrivateKey",
+    "legacyDeviceVerification",
+    "prepareToEncrypt",
+    "isCrossSigningReady",
+    "getCryptoTrustCrossSignedDevices",
+    "setCryptoTrustCrossSignedDevices",
 ]);
 
 /**
@@ -1161,7 +1186,9 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {
  * @param {string} password Passphrase string that can be entered by the user
  *     when restoring the backup as an alternative to entering the recovery key.
  *     Optional.
- * @returns {Promise<String>} The user-facing recovery key string.
+ * @returns {Promise<Object>} Object with public key metadata, encoded private
+ *     recovery key which should be disposed of after displaying to the user,
+ *     and raw private key to avoid round tripping if needed.
  */
 
 /**
@@ -1233,7 +1260,9 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {
  * @param {boolean} checkKey check if the secret is encrypted by a trusted
  *     key
  *
- * @return {boolean} whether or not the secret is stored
+ * @return {object?} map of key name to key info the secret is encrypted
+ *     with, or null if it is not present or not encrypted with a trusted
+ *     key
  */
 
 /**
@@ -1281,12 +1310,14 @@ wrapCryptoFuncs(MatrixClient, [
     "bootstrapSecretStorage",
     "addSecretStorageKey",
     "hasSecretStorageKey",
+    "secretStorageKeyNeedsUpgrade",
     "storeSecret",
     "getSecret",
     "isSecretStored",
     "requestSecret",
     "getDefaultSecretStorageKeyId",
     "setDefaultSecretStorageKeyId",
+    "checkSecretStorageKey",
     "checkSecretStoragePrivateKey",
 ]);
 
@@ -1542,7 +1573,7 @@ MatrixClient.prototype.prepareKeyBackupVersion = async function(
         throw new Error("End-to-end encryption disabled");
     }
 
-    const [keyInfo, encodedPrivateKey, privateKey] =
+    const { keyInfo, encodedPrivateKey, privateKey } =
         await this.createRecoveryKeyFromPassphrase(password);
 
     if (secureSecretStorage) {
@@ -1567,7 +1598,9 @@ MatrixClient.prototype.prepareKeyBackupVersion = async function(
 
 /**
  * Check whether the key backup private key is stored in secret storage.
- * @return {Promise<boolean>} Whether the backup key is stored.
+ * @return {Promise<object?>} map of key name to key info the secret is
+ *     encrypted with, or null if it is not present or not encrypted with a
+ *     trusted key
  */
 MatrixClient.prototype.isKeyBackupKeyStored = async function() {
     return this.isSecretStored("m.megolm_backup.v1", false /* checkKey */);
@@ -1720,6 +1753,35 @@ MatrixClient.prototype.isValidRecoveryKey = function(recoveryKey) {
     }
 };
 
+/**
+ * Get the raw key for a key backup from the password
+ * Used when migrating key backups into SSSS
+ *
+ * The cross-signing API is currently UNSTABLE and may change without notice.
+ *
+ * @param {string} password Passphrase
+ * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @return {Promise<Buffer>} key backup key
+ */
+MatrixClient.prototype.keyBackupKeyFromPassword = function(
+    password, backupInfo,
+) {
+    return keyFromAuthData(backupInfo.auth_data, password);
+};
+
+/**
+ * Get the raw key for a key backup from the recovery key
+ * Used when migrating key backups into SSSS
+ *
+ * The cross-signing API is currently UNSTABLE and may change without notice.
+ *
+ * @param {string} recoveryKey The recovery key
+ * @return {Buffer} key backup key
+ */
+MatrixClient.prototype.keyBackupKeyFromRecoveryKey = function(recoveryKey) {
+    return decodeRecoveryKey(recoveryKey);
+};
+
 MatrixClient.RESTORE_BACKUP_ERROR_BAD_KEY = 'RESTORE_BACKUP_ERROR_BAD_KEY';
 
 /**
@@ -1731,15 +1793,16 @@ MatrixClient.RESTORE_BACKUP_ERROR_BAD_KEY = 'RESTORE_BACKUP_ERROR_BAD_KEY';
  * @param {string} [targetSessionId] Session ID to target a specific session.
  * Restores all sessions if omitted.
  * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @param {object} opts Optional params such as callbacks
  * @return {Promise<object>} Status of restoration with `total` and `imported`
  * key counts.
  */
 MatrixClient.prototype.restoreKeyBackupWithPassword = async function(
-    password, targetRoomId, targetSessionId, backupInfo,
+    password, targetRoomId, targetSessionId, backupInfo, opts,
 ) {
     const privKey = await keyFromAuthData(backupInfo.auth_data, password);
     return this._restoreKeyBackup(
-        privKey, targetRoomId, targetSessionId, backupInfo,
+        privKey, targetRoomId, targetSessionId, backupInfo, opts,
     );
 };
 
@@ -1752,15 +1815,26 @@ MatrixClient.prototype.restoreKeyBackupWithPassword = async function(
  * Restores all rooms if omitted.
  * @param {string} [targetSessionId] Session ID to target a specific session.
  * Restores all sessions if omitted.
+ * @param {object} opts Optional params such as callbacks
  * @return {Promise<object>} Status of restoration with `total` and `imported`
  * key counts.
  */
 MatrixClient.prototype.restoreKeyBackupWithSecretStorage = async function(
-    backupInfo, targetRoomId, targetSessionId,
+    backupInfo, targetRoomId, targetSessionId, opts,
 ) {
-    const privKey = decodeBase64(await this.getSecret("m.megolm_backup.v1"));
+    const storedKey = await this.getSecret("m.megolm_backup.v1");
+
+    // ensure that the key is in the right format.  If not, fix the key and
+    // store the fixed version
+    const fixedKey = fixBackupKey(storedKey);
+    if (fixedKey) {
+        const [keyId] = await this._crypto.getSecretStorageKey();
+        await this.storeSecret("m.megolm_backup.v1", fixedKey, [keyId]);
+    }
+
+    const privKey = decodeBase64(fixedKey || storedKey);
     return this._restoreKeyBackup(
-        privKey, targetRoomId, targetSessionId, backupInfo,
+        privKey, targetRoomId, targetSessionId, backupInfo, opts,
     );
 };
 
@@ -1773,20 +1847,47 @@ MatrixClient.prototype.restoreKeyBackupWithSecretStorage = async function(
  * @param {string} [targetSessionId] Session ID to target a specific session.
  * Restores all sessions if omitted.
  * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @param {object} opts Optional params such as callbacks
+
  * @return {Promise<object>} Status of restoration with `total` and `imported`
  * key counts.
  */
 MatrixClient.prototype.restoreKeyBackupWithRecoveryKey = function(
-    recoveryKey, targetRoomId, targetSessionId, backupInfo,
+    recoveryKey, targetRoomId, targetSessionId, backupInfo, opts,
 ) {
     const privKey = decodeRecoveryKey(recoveryKey);
     return this._restoreKeyBackup(
-        privKey, targetRoomId, targetSessionId, backupInfo,
+        privKey, targetRoomId, targetSessionId, backupInfo, opts,
+    );
+};
+
+/**
+ * Restore from an existing key backup using a cached key, or fail
+ *
+ * @param {string} [targetRoomId] Room ID to target a specific room.
+ * Restores all rooms if omitted.
+ * @param {string} [targetSessionId] Session ID to target a specific session.
+ * Restores all sessions if omitted.
+ * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @param {object} opts Optional params such as callbacks
+ * @return {Promise<object>} Status of restoration with `total` and `imported`
+ * key counts.
+ */
+MatrixClient.prototype.restoreKeyBackupWithCache = async function(
+    targetRoomId, targetSessionId, backupInfo, opts,
+) {
+    const privKey = await this._crypto.getSessionBackupPrivateKey();
+    if (!privKey) {
+        throw new Error("Couldn't get key");
+    }
+    return this._restoreKeyBackup(
+        privKey, targetRoomId, targetSessionId, backupInfo, opts,
     );
 };
 
 MatrixClient.prototype._restoreKeyBackup = function(
     privKey, targetRoomId, targetSessionId, backupInfo,
+    { cacheCompleteCallback }={}, // For sequencing during tests
 ) {
     if (this._crypto === null) {
         throw new Error("End-to-end encryption disabled");
@@ -1814,6 +1915,13 @@ MatrixClient.prototype._restoreKeyBackup = function(
         return Promise.reject({errcode: MatrixClient.RESTORE_BACKUP_ERROR_BAD_KEY});
     }
 
+    // Cache the key, if possible.
+    // This is async.
+    this._crypto.storeSessionBackupPrivateKey(privKey)
+    .catch((e) => {
+        console.warn("Error caching session backup key:", e);
+    }).then(cacheCompleteCallback);
+
     return this._http.authedRequest(
         undefined, "GET", path.path, path.queryData, undefined,
         {prefix: PREFIX_UNSTABLE},
@@ -1984,6 +2092,7 @@ MatrixClient.prototype.getUsers = function() {
 
 /**
  * Set account data event for the current user.
+ * It will retry the request up to 5 times.
  * @param {string} eventType The event type
  * @param {Object} contents the contents object for the event
  * @param {module:client.callback} callback Optional.
@@ -1995,9 +2104,13 @@ MatrixClient.prototype.setAccountData = function(eventType, contents, callback)
         $userId: this.credentials.userId,
         $type: eventType,
     });
-    return this._http.authedRequest(
-        callback, "PUT", path, undefined, contents,
-    );
+    const promise = retryNetworkOperation(5, () => {
+        return this._http.authedRequest(undefined, "PUT", path, undefined, contents);
+    });
+    if (callback) {
+        promise.then(result => callback(null, result), callback);
+    }
+    return promise;
 };
 
 /**
@@ -2032,9 +2145,17 @@ MatrixClient.prototype.getAccountDataFromServer = async function(eventType) {
         $userId: this.credentials.userId,
         $type: eventType,
     });
-    return this._http.authedRequest(
-        undefined, "GET", path, undefined,
-    );
+    try {
+        const result = await this._http.authedRequest(
+            undefined, "GET", path, undefined,
+        );
+        return result;
+    } catch (e) {
+        if (e.data && e.data.errcode === 'M_NOT_FOUND') {
+            return null;
+        }
+        throw e;
+    }
 };
 
 /**
@@ -2425,12 +2546,15 @@ function _sendEvent(client, room, event, callback) {
 
         if (!promise) {
             promise = _sendEventHttpRequest(client, event);
+            if (room) {
+                promise = promise.then(res => {
+                    room.updatePendingEvent(event, EventStatus.SENT, res.event_id);
+                    return res;
+                });
+            }
         }
         return promise;
     }).then(function(res) {  // the request was sent OK
-        if (room) {
-            room.updatePendingEvent(event, EventStatus.SENT, res.event_id);
-        }
         if (callback) {
             callback(null, res);
         }
@@ -4405,64 +4529,54 @@ MatrixClient.prototype.getFilter = function(userId, filterId, allowCached) {
  * @param {Filter} filter
  * @return {Promise<String>} Filter ID
  */
-MatrixClient.prototype.getOrCreateFilter = function(filterName, filter) {
+MatrixClient.prototype.getOrCreateFilter = async function(filterName, filter) {
     const filterId = this.store.getFilterIdByName(filterName);
-    let promise = Promise.resolve();
-    const self = this;
+    let existingId = undefined;
 
     if (filterId) {
         // check that the existing filter matches our expectations
-        promise = self.getFilter(self.credentials.userId,
-                         filterId, true,
-        ).then(function(existingFilter) {
-            const oldDef = existingFilter.getDefinition();
-            const newDef = filter.getDefinition();
+        try {
+            const existingFilter =
+                await this.getFilter(this.credentials.userId, filterId, true);
+            if (existingFilter) {
+                const oldDef = existingFilter.getDefinition();
+                const newDef = filter.getDefinition();
 
-            if (utils.deepCompare(oldDef, newDef)) {
-                // super, just use that.
-                // debuglog("Using existing filter ID %s: %s", filterId,
-                //          JSON.stringify(oldDef));
-                return Promise.resolve(filterId);
+                if (utils.deepCompare(oldDef, newDef)) {
+                    // super, just use that.
+                    // debuglog("Using existing filter ID %s: %s", filterId,
+                    //          JSON.stringify(oldDef));
+                    existingId = filterId;
+                }
             }
-            // debuglog("Existing filter ID %s: %s; new filter: %s",
-            //          filterId, JSON.stringify(oldDef), JSON.stringify(newDef));
-            self.store.setFilterIdByName(filterName, undefined);
-            return undefined;
-        }, function(error) {
+        } catch (error) {
             // Synapse currently returns the following when the filter cannot be found:
             // {
             //     errcode: "M_UNKNOWN",
             //     name: "M_UNKNOWN",
             //     message: "No row found",
-            //     data: Object, httpStatus: 404
             // }
-            if (error.httpStatus === 404 &&
-                (error.errcode === "M_UNKNOWN" || error.errcode === "M_NOT_FOUND")) {
-                // Clear existing filterId from localStorage
-                // if it no longer exists on the server
-                self.store.setFilterIdByName(filterName, undefined);
-                // Return a undefined value for existingId further down the promise chain
-                return undefined;
-            } else {
+            if (error.errcode !== "M_UNKNOWN" && error.errcode !== "M_NOT_FOUND") {
                 throw error;
             }
-        });
+        }
+        // if the filter doesn't exist anymore on the server, remove from store
+        if (!existingId) {
+            this.store.setFilterIdByName(filterName, undefined);
+        }
     }
 
-    return promise.then(function(existingId) {
-        if (existingId) {
-            return existingId;
-        }
+    if (existingId) {
+        return existingId;
+    }
 
-        // create a new filter
-        return self.createFilter(filter.getDefinition(),
-        ).then(function(createdFilter) {
-            // debuglog("Created new filter ID %s: %s", createdFilter.filterId,
-            //          JSON.stringify(createdFilter.getDefinition()));
-            self.store.setFilterIdByName(filterName, createdFilter.filterId);
-            return createdFilter.filterId;
-        });
-    });
+    // create a new filter
+    const createdFilter = await this.createFilter(filter.getDefinition());
+
+    // debuglog("Created new filter ID %s: %s", createdFilter.filterId,
+    //          JSON.stringify(createdFilter.getDefinition()));
+    this.store.setFilterIdByName(filterName, createdFilter.filterId);
+    return createdFilter.filterId;
 };
 
 
@@ -4640,6 +4754,13 @@ MatrixClient.prototype.startClient = async function(opts) {
         };
     }
 
+    // Create our own user object artificially (instead of waiting for sync)
+    // so it's always available, even if the user is not in any rooms etc.
+    const userId = this.getUserId();
+    if (userId) {
+        this.store.storeUser(new User(userId));
+    }
+
     if (this._crypto) {
         this._crypto.uploadDeviceKeys();
         this._crypto.start();
@@ -5113,9 +5234,6 @@ function checkTurnServers(client) {
     if (!client._supportsVoip) {
         return;
     }
-    if (client.isGuest()) {
-        return; // guests can't access TURN servers
-    }
 
     client.turnServer().then(function(res) {
         if (res.uris) {
@@ -5156,17 +5274,20 @@ function _resolve(callback, resolve, res) {
     resolve(res);
 }
 
-function _PojoToMatrixEventMapper(client) {
+function _PojoToMatrixEventMapper(client, options) {
+    const preventReEmit = Boolean(options && options.preventReEmit);
     function mapper(plainOldJsObject) {
         const event = new MatrixEvent(plainOldJsObject);
         if (event.isEncrypted()) {
-            client.reEmitter.reEmit(event, [
-                "Event.decrypted",
-            ]);
+            if (!preventReEmit) {
+                client.reEmitter.reEmit(event, [
+                    "Event.decrypted",
+                ]);
+            }
             event.attemptDecryption(client._crypto);
         }
         const room = client.getRoom(event.getRoomId());
-        if (room) {
+        if (room && !preventReEmit) {
             room.reEmitter.reEmit(event, ["Event.replaced"]);
         }
         return event;
@@ -5175,10 +5296,12 @@ function _PojoToMatrixEventMapper(client) {
 }
 
 /**
+ * @param {object} [options]
+ * @param {bool} options.preventReEmit don't reemit events emitted on an event mapped by this mapper on the client
  * @return {Function}
  */
-MatrixClient.prototype.getEventMapper = function() {
-    return _PojoToMatrixEventMapper(this);
+MatrixClient.prototype.getEventMapper = function(options = undefined) {
+    return _PojoToMatrixEventMapper(this, options);
 };
 
 /**
@@ -5464,6 +5587,22 @@ MatrixClient.prototype.generateClientSecret = function() {
  * });
  */
 
+/**
+ * Fires whenever the stored devices for a user have changed
+ * @event module:client~MatrixClient#"crypto.devicesUpdated"
+ * @param {String[]} users A list of user IDs that were updated
+ * @param {bool} initialFetch If true, the store was empty (apart
+ *     from our own device) and has been seeded.
+ */
+
+/**
+ * Fires whenever the stored devices for a user will be updated
+ * @event module:client~MatrixClient#"crypto.willUpdateDevices"
+ * @param {String[]} users A list of user IDs that will be updated
+ * @param {bool} initialFetch If true, the store is empty (apart
+ *     from our own device) and is being seeded.
+ */
+
 /**
  * Fires whenever the status of e2e key backup changes, as returned by getKeyBackupEnabled()
  * @event module:client~MatrixClient#"crypto.keyBackupStatus"

src/crypto/CrossSigning.js

@@ -55,6 +55,12 @@ export class CrossSigningInfo extends EventEmitter {
         this._cacheCallbacks = cacheCallbacks || {};
         this.keys = {};
         this.firstUse = true;
+        // This tracks whether we've ever verified this user with any identity.
+        // When you verify a user, any devices online at the time that receive
+        // the verifying signature via the homeserver will latch this to true
+        // and can use it in the future to detect cases where the user has
+        // become unverifed later for any reason.
+        this.crossSigningVerifiedBefore = false;
     }
 
     /**
@@ -132,6 +138,7 @@ export class CrossSigningInfo extends EventEmitter {
         return {
             keys: this.keys,
             firstUse: this.firstUse,
+            crossSigningVerifiedBefore: this.crossSigningVerifiedBefore,
         };
     }
 
@@ -141,14 +148,28 @@ export class CrossSigningInfo extends EventEmitter {
      * want to know this anyway...
      *
      * @param {SecretStorage} secretStorage The secret store using account data
-     * @returns {boolean} Whether all private keys were found in storage
+     * @returns {object} map of key name to key info the secret is encrypted
+     *     with, or null if it is not present or not encrypted with a trusted
+     *     key
      */
     async isStoredInSecretStorage(secretStorage) {
-        let stored = true;
-        for (const type of ["master", "self_signing", "user_signing"]) {
-            stored &= await secretStorage.isStored(`m.cross_signing.${type}`, false);
+        // check what SSSS keys have encrypted the master key (if any)
+        const stored =
+              await secretStorage.isStored("m.cross_signing.master", false) || {};
+        // then check which of those SSSS keys have also encrypted the SSK and USK
+        function intersect(s) {
+            for (const k of Object.keys(stored)) {
+                if (!s[k]) {
+                    delete stored[k];
+                }
+            }
         }
-        return stored;
+        for (const type of ["self_signing", "user_signing"]) {
+            intersect(
+                await secretStorage.isStored(`m.cross_signing.${type}`, false) || {},
+            );
+        }
+        return Object.keys(stored).length ? stored : null;
     }
 
     /**
@@ -289,6 +310,13 @@ export class CrossSigningInfo extends EventEmitter {
         }
     }
 
+    /**
+     * unsets the keys, used when another session has reset the keys, to disable cross-signing
+     */
+    clearKeys() {
+        this.keys = {};
+    }
+
     setKeys(keys) {
         const signingKeys = {};
         if (keys.master) {
@@ -361,6 +389,14 @@ export class CrossSigningInfo extends EventEmitter {
         }
     }
 
+    updateCrossSigningVerifiedBefore(isCrossSigningVerified) {
+        // It is critical that this value latches forward from false to true but
+        // never back to false to avoid a downgrade attack.
+        if (!this.crossSigningVerifiedBefore && isCrossSigningVerified) {
+            this.crossSigningVerifiedBefore = true;
+        }
+    }
+
     async signObject(data, type) {
         if (!this.keys[type]) {
             throw new Error(
@@ -419,13 +455,13 @@ export class CrossSigningInfo extends EventEmitter {
             && this.getId("self_signing")
             && this.getId("self_signing") === userCrossSigning.getId("self_signing")
         ) {
-            return new UserTrustLevel(true, this.firstUse);
+            return new UserTrustLevel(true, true, this.firstUse);
         }
 
         if (!this.keys.user_signing) {
             // If there's no user signing key, they can't possibly be verified.
             // They may be TOFU trusted though.
-            return new UserTrustLevel(false, userCrossSigning.firstUse);
+            return new UserTrustLevel(false, false, userCrossSigning.firstUse);
         }
 
         let userTrusted;
@@ -437,7 +473,11 @@ export class CrossSigningInfo extends EventEmitter {
         } catch (e) {
             userTrusted = false;
         }
-        return new UserTrustLevel(userTrusted, userCrossSigning.firstUse);
+        return new UserTrustLevel(
+            userTrusted,
+            userCrossSigning.crossSigningVerifiedBefore,
+            userCrossSigning.firstUse,
+        );
     }
 
     /**
@@ -446,17 +486,20 @@ export class CrossSigningInfo extends EventEmitter {
      * @param {CrossSigningInfo} userCrossSigning Cross signing info for user
      * @param {module:crypto/deviceinfo} device The device to check
      * @param {bool} localTrust Whether the device is trusted locally
+     * @param {bool} trustCrossSignedDevices Whether we trust cross signed devices
      *
      * @returns {DeviceTrustLevel}
      */
-    checkDeviceTrust(userCrossSigning, device, localTrust) {
+    checkDeviceTrust(userCrossSigning, device, localTrust, trustCrossSignedDevices) {
         const userTrust = this.checkUserTrust(userCrossSigning);
 
         const userSSK = userCrossSigning.keys.self_signing;
         if (!userSSK) {
             // if the user has no self-signing key then we cannot make any
             // trust assertions about this device from cross-signing
-            return new DeviceTrustLevel(false, false, localTrust);
+            return new DeviceTrustLevel(
+                false, false, localTrust, trustCrossSignedDevices,
+            );
         }
 
         const deviceObj = deviceToObject(device, userCrossSigning.userId);
@@ -468,9 +511,13 @@ export class CrossSigningInfo extends EventEmitter {
                 deviceObj, publicKeyFromKeyInfo(userSSK), userCrossSigning.userId,
             );
             // ...then we trust this device as much as far as we trust the user
-            return DeviceTrustLevel.fromUserTrustLevel(userTrust, localTrust);
+            return DeviceTrustLevel.fromUserTrustLevel(
+                userTrust, localTrust, trustCrossSignedDevices,
+            );
         } catch (e) {
-            return new DeviceTrustLevel(false, false, localTrust);
+            return new DeviceTrustLevel(
+                false, false, localTrust, trustCrossSignedDevices,
+            );
         }
     }
 
@@ -502,8 +549,9 @@ export const CrossSigningLevel = {
  * Represents the ways in which we trust a user
  */
 export class UserTrustLevel {
-    constructor(crossSigningVerified, tofu) {
+    constructor(crossSigningVerified, crossSigningVerifiedBefore, tofu) {
         this._crossSigningVerified = crossSigningVerified;
+        this._crossSigningVerifiedBefore = crossSigningVerifiedBefore;
         this._tofu = tofu;
     }
 
@@ -521,6 +569,14 @@ export class UserTrustLevel {
         return this._crossSigningVerified;
     }
 
+    /**
+     * @returns {bool} true if we ever verified this user before (at least for
+     * the history of verifications observed by this device).
+     */
+    wasCrossSigningVerified() {
+        return this._crossSigningVerifiedBefore;
+    }
+
     /**
      * @returns {bool} true if this user's key is trusted on first use
      */
@@ -533,17 +589,19 @@ export class UserTrustLevel {
  * Represents the ways in which we trust a device
  */
 export class DeviceTrustLevel {
-    constructor(crossSigningVerified, tofu, localVerified) {
+    constructor(crossSigningVerified, tofu, localVerified, trustCrossSignedDevices) {
         this._crossSigningVerified = crossSigningVerified;
         this._tofu = tofu;
         this._localVerified = localVerified;
+        this._trustCrossSignedDevices = trustCrossSignedDevices;
     }
 
-    static fromUserTrustLevel(userTrustLevel, localVerified) {
+    static fromUserTrustLevel(userTrustLevel, localVerified, trustCrossSignedDevices) {
         return new DeviceTrustLevel(
             userTrustLevel._crossSigningVerified,
             userTrustLevel._tofu,
             localVerified,
+            trustCrossSignedDevices,
         );
     }
 
@@ -551,7 +609,9 @@ export class DeviceTrustLevel {
      * @returns {bool} true if this device is verified via any means
      */
     isVerified() {
-        return this.isCrossSigningVerified() || this.isLocallyVerified();
+        return Boolean(this.isLocallyVerified() || (
+            this._trustCrossSignedDevices && this.isCrossSigningVerified()
+        ));
     }
 
     /**
@@ -585,17 +645,22 @@ export function createCryptoStoreCacheCallbacks(store) {
                     'readonly',
                     [IndexedDBCryptoStore.STORE_ACCOUNT],
                     (txn) => {
-                        store.getCrossSigningPrivateKey(txn, resolve, type);
+                        store.getSecretStorePrivateKey(txn, resolve, type);
                     },
                 );
             });
         },
         storeCrossSigningKeyCache: function(type, key) {
+            if (!(key instanceof Uint8Array)) {
+                throw new Error(
+                    `storeCrossSigningKeyCache expects Uint8Array, got ${key}`,
+                );
+            }
             return store.doTxn(
                 'readwrite',
                 [IndexedDBCryptoStore.STORE_ACCOUNT],
                 (txn) => {
-                    store.storeCrossSigningPrivateKey(txn, type, key);
+                    store.storeSecretStorePrivateKey(txn, type, key);
                 },
             );
         },

src/crypto/DeviceList.js

@@ -109,6 +109,9 @@ export class DeviceList extends EventEmitter {
         this._savePromiseTime = null;
         // The timer used to delay the save
         this._saveTimer = null;
+        // True if we have fetched data from the server or loaded a non-empty
+        // set of device data from the store
+        this._hasFetched = null;
     }
 
     /**
@@ -118,6 +121,7 @@ export class DeviceList extends EventEmitter {
         await this._cryptoStore.doTxn(
             'readonly', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
                 this._cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
+                    this._hasFetched = Boolean(deviceData && deviceData.devices);
                     this._devices = deviceData ? deviceData.devices : {},
                     this._crossSigningInfo = deviceData ?
                         deviceData.crossSigningInfo || {} : {};
@@ -317,6 +321,15 @@ export class DeviceList extends EventEmitter {
         return stored;
     }
 
+    /**
+     * Returns a list of all user IDs the DeviceList knows about
+     *
+     * @return {array} All known user IDs
+     */
+    getKnownUserIds() {
+        return Object.keys(this._devices);
+    }
+
     /**
      * Get the stored device keys for a user id
      *
@@ -643,6 +656,7 @@ export class DeviceList extends EventEmitter {
         });
 
         const finished = (success) => {
+            this.emit("crypto.willUpdateDevices", users, !this._hasFetched);
             users.forEach((u) => {
                 this._dirty = true;
 
@@ -668,7 +682,8 @@ export class DeviceList extends EventEmitter {
                 }
             });
             this.saveIfDirty();
-            this.emit("crypto.devicesUpdated", users);
+            this.emit("crypto.devicesUpdated", users, !this._hasFetched);
+            this._hasFetched = true;
         };
 
         return prom;
@@ -767,31 +782,34 @@ class DeviceListUpdateSerialiser {
 
         this._baseApis.downloadKeysForUsers(
             downloadUsers, opts,
-        ).then((res) => {
+        ).then(async (res) => {
             const dk = res.device_keys || {};
             const masterKeys = res.master_keys || {};
             const ssks = res.self_signing_keys || {};
             const usks = res.user_signing_keys || {};
 
-            // do each user in a separate promise, to avoid wedging the CPU
+            // yield to other things that want to execute in between users, to
+            // avoid wedging the CPU
             // (https://github.com/vector-im/riot-web/issues/3158)
             //
             // of course we ought to do this in a web worker or similar, but
             // this serves as an easy solution for now.
-            let prom = Promise.resolve();
             for (const userId of downloadUsers) {
-                prom = prom.then(sleep(5)).then(() => {
-                    return this._processQueryResponseForUser(
+                await sleep(5);
+                try {
+                    await this._processQueryResponseForUser(
                         userId, dk[userId], {
                             master: masterKeys[userId],
                             self_signing: ssks[userId],
                             user_signing: usks[userId],
                         },
                     );
-                });
+                } catch (e) {
+                    // log the error but continue, so that one bad key
+                    // doesn't kill the whole process
+                    logger.error(`Error processing keys for ${userId}:`, e);
+                }
             }
-
-            return prom;
         }).then(() => {
             logger.log('Completed key download for ' + downloadUsers);
 
@@ -812,7 +830,7 @@ class DeviceListUpdateSerialiser {
     }
 
     async _processQueryResponseForUser(
-        userId, dkResponse, crossSigningResponse, sskResponse,
+        userId, dkResponse, crossSigningResponse,
     ) {
         logger.log('got device keys for ' + userId + ':', dkResponse);
         logger.log('got cross-signing keys for ' + userId + ':', crossSigningResponse);

src/crypto/OlmDevice.js

@@ -105,6 +105,9 @@ export function OlmDevice(cryptoStore) {
     // Keep track of sessions that we're starting, so that we don't start
     // multiple sessions for the same device at the same time.
     this._sessionsInProgress = {};
+
+    // Used by olm to serialise prekey message decryptions
+    this._olmPrekeyPromise = Promise.resolve();
 }
 
 /**

src/crypto/OutgoingRoomKeyRequestManager.js

@@ -58,7 +58,7 @@ const SEND_KEY_REQUESTS_DELAY_MS = 500;
  *
  * @enum {number}
  */
-const ROOM_KEY_REQUEST_STATES = {
+export const ROOM_KEY_REQUEST_STATES = {
     /** request not yet sent */
     UNSENT: 0,
 
@@ -327,6 +327,21 @@ export class OutgoingRoomKeyRequestManager {
         );
     }
 
+    /**
+     * Find anything in `sent` state, and kick it around the loop again.
+     * This is intended for situations where something substantial has changed, and we
+     * don't really expect the other end to even care about the cancellation.
+     * For example, after initialization or self-verification.
+     * @return {Promise} An array of `sendRoomKeyRequest` outputs.
+     */
+    async cancelAndResendAllOutgoingRequests() {
+        const outgoings = await this._cryptoStore.getAllOutgoingRoomKeyRequestsByState(
+            ROOM_KEY_REQUEST_STATES.SENT,
+        );
+        return Promise.all(outgoings.map(({ requestBody, recipients }) =>
+            this.sendRoomKeyRequest(requestBody, recipients, true)));
+    }
+
     // start the background timer to send queued requests, if the timer isn't
     // already running
     _startTimer() {

src/crypto/SecretStorage.js

@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,8 +19,16 @@ import {logger} from '../logger';
 import * as olmlib from './olmlib';
 import {pkVerify} from './olmlib';
 import {randomString} from '../randomstring';
+import {encryptAES, decryptAES} from './aes';
+import {encodeBase64} from "./olmlib";
 
-export const SECRET_STORAGE_ALGORITHM_V1 = "m.secret_storage.v1.curve25519-aes-sha2";
+export const SECRET_STORAGE_ALGORITHM_V1_AES
+    = "m.secret_storage.v1.aes-hmac-sha2";
+// don't use curve25519 for writing data.
+export const SECRET_STORAGE_ALGORITHM_V1_CURVE25519
+    = "m.secret_storage.v1.curve25519-aes-sha2";
+
+const ZERO_STR = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
 
 /**
  * Implements Secure Secret Storage and Sharing (MSC1946)
@@ -45,7 +53,7 @@ export class SecretStorage extends EventEmitter {
     }
 
     setDefaultKeyId(keyId) {
-        return new Promise((resolve) => {
+        return new Promise(async (resolve, reject) => {
             const listener = (ev) => {
                 if (
                     ev.getType() === 'm.secret_storage.default_key' &&
@@ -57,10 +65,15 @@ export class SecretStorage extends EventEmitter {
             };
             this._baseApis.on('accountData', listener);
 
-            this._baseApis.setAccountData(
-                'm.secret_storage.default_key',
-                { key: keyId },
-            );
+            try {
+                await this._baseApis.setAccountData(
+                    'm.secret_storage.default_key',
+                    { key: keyId },
+                );
+            } catch (e) {
+                this._baseApis.removeListener('accountData', listener);
+                reject(e);
+            }
         });
     }
 
@@ -85,23 +98,15 @@ export class SecretStorage extends EventEmitter {
         }
 
         switch (algorithm) {
-        case SECRET_STORAGE_ALGORITHM_V1:
+        case SECRET_STORAGE_ALGORITHM_V1_AES:
         {
-            const decryption = new global.Olm.PkDecryption();
-            try {
-                const { passphrase, pubkey } = opts;
-                // Copies in public key details of the form generated by
-                // the Crypto module's `createRecoveryKeyFromPassphrase`.
-                if (passphrase && pubkey) {
-                    keyData.passphrase = passphrase;
-                    keyData.pubkey = pubkey;
-                } else if (pubkey) {
-                    keyData.pubkey = pubkey;
-                } else {
-                    keyData.pubkey = decryption.generate_key();
-                }
-            } finally {
-                decryption.free();
+            if (opts.passphrase) {
+                keyData.passphrase = opts.passphrase;
+            }
+            if (opts.key) {
+                const {iv, mac} = await SecretStorage._calculateKeyCheck(opts.key);
+                keyData.iv = iv;
+                keyData.mac = mac;
             }
             break;
         }
@@ -155,6 +160,28 @@ export class SecretStorage extends EventEmitter {
         );
     }
 
+    /**
+     * Get the key information for a given ID.
+     *
+     * @param {string} [keyId = default key's ID] The ID of the key to check
+     *     for. Defaults to the default key ID if not provided.
+     * @returns {Array?} If the key was found, the return value is an array of
+     *     the form [keyId, keyInfo].  Otherwise, null is returned.
+     */
+    async getKey(keyId) {
+        if (!keyId) {
+            keyId = await this.getDefaultKeyId();
+        }
+        if (!keyId) {
+            return null;
+        }
+
+        const keyInfo = await this._baseApis.getAccountDataFromServer(
+            "m.secret_storage.key." + keyId,
+        );
+        return keyInfo ? [keyId, keyInfo] : null;
+    }
+
     /**
      * Check whether we have a key with a given ID.
      *
@@ -163,16 +190,59 @@ export class SecretStorage extends EventEmitter {
      * @return {boolean} Whether we have the key.
      */
     async hasKey(keyId) {
-        if (!keyId) {
-            keyId = await this.getDefaultKeyId();
-        }
-        if (!keyId) {
+        return !!(await this.getKey(keyId));
+    }
+
+    async keyNeedsUpgrade(keyId) {
+        const keyInfo = await this.getKey(keyId);
+        if (keyInfo && keyInfo[1].algorithm === SECRET_STORAGE_ALGORITHM_V1_CURVE25519) {
+            return true;
+        } else {
             return false;
         }
+    }
 
-        return !!this._baseApis.getAccountDataFromServer(
-            "m.secret_storage.key." + keyId,
-        );
+    /**
+     * Check whether a key matches what we expect based on the key info
+     *
+     * @param {Uint8Array} key the key to check
+     * @param {object} info the key info
+     *
+     * @return {boolean} whether or not the key matches
+     */
+    async checkKey(key, info) {
+        switch (info.algorithm) {
+        case SECRET_STORAGE_ALGORITHM_V1_AES:
+        {
+            if (info.mac) {
+                const {mac} = await SecretStorage._calculateKeyCheck(key, info.iv);
+                return info.mac === mac;
+            } else {
+                // if we have no information, we have to assume the key is right
+                return true;
+            }
+        }
+        case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
+        {
+            let decryption = null;
+            try {
+                decryption = new global.Olm.PkDecryption();
+                const gotPubkey = decryption.init_with_private_key(key);
+                // make sure it agrees with the given pubkey
+                return gotPubkey === info.pubkey;
+            } catch (e) {
+                return false;
+            } finally {
+                if (decryption) decryption.free();
+            }
+        }
+        default:
+            throw new Error("Unknown algorithm");
+        }
+    }
+
+    static async _calculateKeyCheck(key, iv) {
+        return await encryptAES(ZERO_STR, key, "", iv);
     }
 
     /**
@@ -207,24 +277,13 @@ export class SecretStorage extends EventEmitter {
                 throw new Error("Unknown key: " + keyId);
             }
 
-            // check signature of key info
-            pkVerify(
-                keyInfo,
-                this._crossSigningInfo.getId('master'),
-                this._crossSigningInfo.userId,
-            );
-
             // encrypt secret, based on the algorithm
             switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
+            case SECRET_STORAGE_ALGORITHM_V1_AES:
             {
-                const encryption = new global.Olm.PkEncryption();
-                try {
-                    encryption.set_recipient_key(keyInfo.pubkey);
-                    encrypted[keyId] = encryption.encrypt(secret);
-                } finally {
-                    encryption.free();
-                }
+                const keys = {[keyId]: keyInfo};
+                const [, encryption] = await this._getSecretStorageKey(keys, name);
+                encrypted[keyId] = await encryption.encrypt(secret);
                 break;
             }
             default:
@@ -238,29 +297,6 @@ export class SecretStorage extends EventEmitter {
         await this._baseApis.setAccountData(name, {encrypted});
     }
 
-    /**
-     * Store a secret defined to be the same as the given key.
-     * No secret information will be stored, instead the secret will
-     * be stored with a marker to say that the contents of the secret is
-     * the value of the given key.
-     * This is useful for migration from systems that predate SSSS such as
-     * key backup.
-     *
-     * @param {string} name The name of the secret
-     * @param {string} keyId The ID of the key whose value will be the
-     *     value of the secret
-     * @returns {Promise} resolved when account data is saved
-     */
-    storePassthrough(name, keyId) {
-        return this._baseApis.setAccountData(name, {
-            encrypted: {
-                [keyId]: {
-                    passthrough: true,
-                },
-            },
-        });
-    }
-
     /**
      * Temporary method to fix up existing accounts where secrets
      * are incorrectly stored without the 'encrypted' level
@@ -317,7 +353,12 @@ export class SecretStorage extends EventEmitter {
             );
             const encInfo = secretInfo.encrypted[keyId];
             switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
+            case SECRET_STORAGE_ALGORITHM_V1_AES:
+                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
+                    keys[keyId] = keyInfo;
+                }
+                break;
+            case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
                 if (
                     keyInfo.pubkey && (
                         (encInfo.ciphertext && encInfo.mac && encInfo.ephemeral) ||
@@ -341,18 +382,13 @@ export class SecretStorage extends EventEmitter {
             const encInfo = secretInfo.encrypted[keyId];
 
             // We don't actually need the decryption object if it's a passthrough
-            // since we just want to return the key itself.
-            if (encInfo.passthrough) return decryption.get_private_key();
+            // since we just want to return the key itself. It must be base64
+            // encoded, since this is how a key would normally be stored.
+            if (encInfo.passthrough) return encodeBase64(decryption.get_private_key());
 
-            // decrypt secret
-            switch (keys[keyId].algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-                return decryption.decrypt(
-                    encInfo.ephemeral, encInfo.mac, encInfo.ciphertext,
-                );
-            }
+            return await decryption.decrypt(encInfo);
         } finally {
-            if (decryption) decryption.free();
+            if (decryption && decryption.free) decryption.free();
         }
     }
 
@@ -362,22 +398,26 @@ export class SecretStorage extends EventEmitter {
      * @param {string} name the name of the secret
      * @param {boolean} checkKey check if the secret is encrypted by a trusted key
      *
-     * @return {boolean} whether or not the secret is stored
+     * @return {object?} map of key name to key info the secret is encrypted
+     *     with, or null if it is not present or not encrypted with a trusted
+     *     key
      */
     async isStored(name, checkKey) {
         // check if secret exists
         let secretInfo = await this._baseApis.getAccountDataFromServer(name);
-        if (!secretInfo) return false;
+        if (!secretInfo) return null;
         if (!secretInfo.encrypted) {
             // try to fix it up
             secretInfo = await this._fixupStoredSecret(name, secretInfo);
             if (!secretInfo || !secretInfo.encrypted) {
-                return false;
+                return null;
             }
         }
 
         if (checkKey === undefined) checkKey = true;
 
+        const ret = {};
+
         // check if secret is encrypted by a known/trusted secret and
         // encryption looks sane
         for (const keyId of Object.keys(secretInfo.encrypted)) {
@@ -385,32 +425,55 @@ export class SecretStorage extends EventEmitter {
             const keyInfo = await this._baseApis.getAccountDataFromServer(
                 "m.secret_storage.key." + keyId,
             );
-            if (!keyInfo) return false;
+            if (!keyInfo) continue;
             const encInfo = secretInfo.encrypted[keyId];
-            if (checkKey) {
-                pkVerify(
-                    keyInfo,
-                    this._crossSigningInfo.getId('master'),
-                    this._crossSigningInfo.userId,
-                );
-            }
 
             // We don't actually need the decryption object if it's a passthrough
             // since we just want to return the key itself.
-            if (encInfo.passthrough) return true;
+            if (encInfo.passthrough) {
+                try {
+                    pkVerify(
+                        keyInfo,
+                        this._crossSigningInfo.getId('master'),
+                        this._crossSigningInfo.userId,
+                    );
+                } catch (e) {
+                    // not trusted, so move on to the next key
+                    continue;
+                }
+                ret[keyId] = keyInfo;
+                continue;
+            }
 
             switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
+            case SECRET_STORAGE_ALGORITHM_V1_AES:
+                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
+                    ret[keyId] = keyInfo;
+                }
+                break;
+            case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
                 if (keyInfo.pubkey && encInfo.ciphertext && encInfo.mac
                     && encInfo.ephemeral) {
-                    return true;
+                    if (checkKey) {
+                        try {
+                            pkVerify(
+                                keyInfo,
+                                this._crossSigningInfo.getId('master'),
+                                this._crossSigningInfo.userId,
+                            );
+                        } catch (e) {
+                            // not trusted, so move on to the next key
+                            continue;
+                        }
+                    }
+                    ret[keyId] = keyInfo;
                 }
                 break;
             default:
                 // do nothing if we don't understand the encryption algorithm
             }
         }
-        return false;
+        return Object.keys(ret).length ? ret : null;
     }
 
     /**
@@ -462,6 +525,7 @@ export class SecretStorage extends EventEmitter {
         for (const device of devices) {
             toDevice[device] = requestData;
         }
+        logger.info(`Request secret ${name} from ${devices}, id ${requestId}`);
         this._baseApis.sendToDevice("m.secret.request", {
             [this._baseApis.getUserId()]: toDevice,
         });
@@ -515,6 +579,7 @@ export class SecretStorage extends EventEmitter {
                 device_trust: this._baseApis.checkDeviceTrust(sender, deviceId),
             });
             if (secret) {
+                logger.info(`Preparing ${content.name} secret for ${deviceId}`);
                 const payload = {
                     type: "m.secret.send",
                     content: {
@@ -532,7 +597,7 @@ export class SecretStorage extends EventEmitter {
                     this._baseApis,
                     {
                         [sender]: [
-                            await this._baseApis.getStoredDevice(sender, deviceId),
+                            this._baseApis.getStoredDevice(sender, deviceId),
                         ],
                     },
                 );
@@ -542,7 +607,7 @@ export class SecretStorage extends EventEmitter {
                     this._baseApis.deviceId,
                     this._baseApis._crypto._olmDevice,
                     sender,
-                    this._baseApis._crypto.getStoredDevice(sender, deviceId),
+                    this._baseApis.getStoredDevice(sender, deviceId),
                     payload,
                 );
                 const contentMap = {
@@ -551,7 +616,10 @@ export class SecretStorage extends EventEmitter {
                     },
                 };
 
+                logger.info(`Sending ${content.name} secret for ${deviceId}`);
                 this._baseApis.sendToDevice("m.room.encrypted", contentMap);
+            } else {
+                logger.info(`Request denied for ${content.name} secret for ${deviceId}`);
             }
         }
     }
@@ -563,7 +631,7 @@ export class SecretStorage extends EventEmitter {
             return;
         }
         const content = event.getContent();
-        logger.log("got secret share for request ", content.request_id);
+        logger.log("got secret share for request", content.request_id);
         const requestControl = this._requests[content.request_id];
         if (requestControl) {
             // make sure that the device that sent it is one of the devices that
@@ -607,26 +675,48 @@ export class SecretStorage extends EventEmitter {
         }
 
         switch (keys[keyId].algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-            {
-                const decryption = new global.Olm.PkDecryption();
-                let pubkey;
-                try {
-                    pubkey = decryption.init_with_private_key(privateKey);
-                } catch (e) {
-                    decryption.free();
-                    throw new Error("getSecretStorageKey callback returned invalid key");
-                }
-                if (pubkey !== keys[keyId].pubkey) {
-                    decryption.free();
-                    throw new Error(
-                        "getSecretStorageKey callback returned incorrect key",
-                    );
-                }
-                return [keyId, decryption];
+        case SECRET_STORAGE_ALGORITHM_V1_AES:
+        {
+            const decryption = {
+                encrypt: async function(secret) {
+                    return await encryptAES(secret, privateKey, name);
+                },
+                decrypt: async function(encInfo) {
+                    return await decryptAES(encInfo, privateKey, name);
+                },
+            };
+            return [keyId, decryption];
+        }
+        case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
+        {
+            const pkDecryption = new global.Olm.PkDecryption();
+            let pubkey;
+            try {
+                pubkey = pkDecryption.init_with_private_key(privateKey);
+            } catch (e) {
+                pkDecryption.free();
+                throw new Error("getSecretStorageKey callback returned invalid key");
             }
-            default:
-                throw new Error("Unknown key type: " + keys[keyId].algorithm);
+            if (pubkey !== keys[keyId].pubkey) {
+                pkDecryption.free();
+                throw new Error(
+                    "getSecretStorageKey callback returned incorrect key",
+                );
+            }
+            const decryption = {
+                free: pkDecryption.free.bind(pkDecryption),
+                decrypt: async function(encInfo) {
+                    return pkDecryption.decrypt(
+                        encInfo.ephemeral, encInfo.mac, encInfo.ciphertext,
+                    );
+                },
+                // needed for passthrough
+                get_private_key: pkDecryption.get_private_key.bind(pkDecryption),
+            };
+            return [keyId, decryption];
+        }
+        default:
+            throw new Error("Unknown key type: " + keys[keyId].algorithm);
         }
     }
 }

src/crypto/aes.js

@@ -0,0 +1,251 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {getCrypto} from '../utils';
+import {decodeBase64, encodeBase64} from './olmlib';
+
+const subtleCrypto = (typeof window !== "undefined" && window.crypto) ?
+    (window.crypto.subtle || window.crypto.webkitSubtle) : null;
+
+// salt for HKDF, with 8 bytes of zeros
+const zerosalt = new Uint8Array(8);
+
+/**
+ * encrypt a string in Node.js
+ *
+ * @param {string} data the plaintext to encrypt
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
+ */
+async function encryptNode(data, key, name, ivStr) {
+    const crypto = getCrypto();
+    if (!crypto) {
+        throw new Error("No usable crypto implementation");
+    }
+
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = crypto.randomBytes(16);
+    }
+
+    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
+    // (which would mean we wouldn't be able to decrypt on Android). The loss
+    // of a single bit of iv is a price we have to pay.
+    iv[8] &= 0x7f;
+
+    const [aesKey, hmacKey] = deriveKeysNode(key, name);
+
+    const cipher = crypto.createCipheriv("aes-256-ctr", aesKey, iv);
+    const ciphertext = cipher.update(data, "utf-8", "base64")
+          + cipher.final("base64");
+
+    const hmac = crypto.createHmac("sha256", hmacKey)
+        .update(ciphertext, "base64").digest("base64");
+
+    return {
+        iv: encodeBase64(iv),
+        ciphertext: ciphertext,
+        mac: hmac,
+    };
+}
+
+/**
+ * decrypt a string in Node.js
+ *
+ * @param {object} data the encrypted data
+ * @param {string} data.ciphertext the ciphertext in base64
+ * @param {string} data.iv the initialization vector in base64
+ * @param {string} data.mac the HMAC in base64
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function decryptNode(data, key, name) {
+    const crypto = getCrypto();
+    if (!crypto) {
+        throw new Error("No usable crypto implementation");
+    }
+
+    const [aesKey, hmacKey] = deriveKeysNode(key, name);
+
+    const hmac = crypto.createHmac("sha256", hmacKey)
+        .update(data.ciphertext, "base64").digest("base64");
+
+    if (hmac !== data.mac) {
+        throw new Error(`Error decrypting secret ${name}: bad MAC`);
+    }
+
+    const decipher = crypto.createDecipheriv(
+        "aes-256-ctr", aesKey, decodeBase64(data.iv),
+    );
+    return decipher.update(data.ciphertext, "base64", "utf-8")
+          + decipher.final("utf-8");
+}
+
+function deriveKeysNode(key, name) {
+    const crypto = getCrypto();
+    const prk = crypto.createHmac("sha256", zerosalt)
+        .update(key).digest();
+
+    const b = Buffer.alloc(1, 1);
+    const aesKey = crypto.createHmac("sha256", prk)
+        .update(name, "utf-8").update(b).digest();
+    b[0] = 2;
+    const hmacKey = crypto.createHmac("sha256", prk)
+        .update(aesKey).update(name, "utf-8").update(b).digest();
+
+    return [aesKey, hmacKey];
+}
+
+/**
+ * encrypt a string in Node.js
+ *
+ * @param {string} data the plaintext to encrypt
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
+ */
+async function encryptBrowser(data, key, name, ivStr) {
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = new Uint8Array(16);
+        window.crypto.getRandomValues(iv);
+    }
+
+    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
+    // (which would mean we wouldn't be able to decrypt on Android). The loss
+    // of a single bit of iv is a price we have to pay.
+    iv[8] &= 0x7f;
+
+    const [aesKey, hmacKey] = await deriveKeysBrowser(key, name);
+    const encodedData = new TextEncoder().encode(data);
+
+    const ciphertext = await subtleCrypto.encrypt(
+        {
+            name: "AES-CTR",
+            counter: iv,
+            length: 64,
+        },
+        aesKey,
+        encodedData,
+    );
+
+    const hmac = await subtleCrypto.sign(
+        {name: 'HMAC'},
+        hmacKey,
+        ciphertext,
+    );
+
+    return {
+        iv: encodeBase64(iv),
+        ciphertext: encodeBase64(ciphertext),
+        mac: encodeBase64(hmac),
+    };
+}
+
+/**
+ * decrypt a string in the browser
+ *
+ * @param {object} data the encrypted data
+ * @param {string} data.ciphertext the ciphertext in base64
+ * @param {string} data.iv the initialization vector in base64
+ * @param {string} data.mac the HMAC in base64
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function decryptBrowser(data, key, name) {
+    const [aesKey, hmacKey] = await deriveKeysBrowser(key, name);
+
+    const ciphertext = decodeBase64(data.ciphertext);
+
+    if (!await subtleCrypto.verify(
+        {name: "HMAC"},
+        hmacKey,
+        decodeBase64(data.mac),
+        ciphertext,
+    )) {
+        throw new Error(`Error decrypting secret ${name}: bad MAC`);
+    }
+
+    const plaintext = await subtleCrypto.decrypt(
+        {
+            name: "AES-CTR",
+            counter: decodeBase64(data.iv),
+            length: 64,
+        },
+        aesKey,
+        ciphertext,
+    );
+
+    return new TextDecoder().decode(new Uint8Array(plaintext));
+}
+
+async function deriveKeysBrowser(key, name) {
+    const hkdfkey = await subtleCrypto.importKey(
+        'raw',
+        key,
+        {name: "HKDF"},
+        false,
+        ["deriveBits"],
+    );
+    const keybits = await subtleCrypto.deriveBits(
+        {
+            name: "HKDF",
+            salt: zerosalt,
+            info: (new TextEncoder().encode(name)),
+            hash: "SHA-256",
+        },
+        hkdfkey,
+        512,
+    );
+
+    const aesKey = keybits.slice(0, 32);
+    const hmacKey = keybits.slice(32);
+
+    const aesProm = subtleCrypto.importKey(
+        'raw',
+        aesKey,
+        {name: 'AES-CTR'},
+        false,
+        ['encrypt', 'decrypt'],
+    );
+
+    const hmacProm = subtleCrypto.importKey(
+        'raw',
+        hmacKey,
+        {
+            name: 'HMAC',
+            hash: {name: 'SHA-256'},
+        },
+        false,
+        ['sign', 'verify'],
+    );
+
+    return await Promise.all([aesProm, hmacProm]);
+}
+
+export function encryptAES(...args) {
+    return subtleCrypto ? encryptBrowser(...args) : encryptNode(...args);
+}
+
+export function decryptAES(...args) {
+    return subtleCrypto ? decryptBrowser(...args) : decryptNode(...args);
+}
+

src/crypto/algorithms/megolm.js

@@ -185,12 +185,14 @@ utils.inherits(MegolmEncryption, EncryptionAlgorithm);
  *
  * @param {Object} devicesInRoom The devices in this room, indexed by user ID
  * @param {Object} blocked The devices that are blocked, indexed by user ID
+ * @param {boolean} [singleOlmCreationPhase] Only perform one round of olm
+ *     session creation
  *
  * @return {Promise} Promise which resolves to the
  *    OutboundSessionInfo when setup is complete.
  */
 MegolmEncryption.prototype._ensureOutboundSession = async function(
-    devicesInRoom, blocked,
+    devicesInRoom, blocked, singleOlmCreationPhase,
 ) {
     let session;
 
@@ -270,56 +272,53 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
                 const errorDevices = [];
 
                 // meanwhile, establish olm sessions for devices that we don't
-                // already have a session for, and share keys with them.  Use a
-                // shorter timeout when fetching one-time keys.
+                // already have a session for, and share keys with them.  If
+                // we're doing two phases of olm session creation, use a
+                // shorter timeout when fetching one-time keys for the first
+                // phase.
+                const start = Date.now();
+                const failedServers = [];
                 await this._shareKeyWithDevices(
-                    session, key, payload, devicesWithoutSession, errorDevices, 2000,
+                    session, key, payload, devicesWithoutSession, errorDevices,
+                    singleOlmCreationPhase ? 10000 : 2000, failedServers,
                 );
 
-                (async () => {
-                    // Retry sending keys to devices that we were unable to establish
-                    // an olm session for.  This time, we use a longer timeout, but we
-                    // do this in the background and don't block anything else while we
-                    // do this.
-                    const retryDevices = {};
-                    for (const {userId, deviceInfo} of errorDevices) {
-                        retryDevices[userId] = retryDevices[userId] || [];
-                        retryDevices[userId].push(deviceInfo);
-                    }
+                if (!singleOlmCreationPhase && (Date.now() - start < 10000)) {
+                    // perform the second phase of olm session creation if requested,
+                    // and if the first phase didn't take too long
+                    (async () => {
+                        // Retry sending keys to devices that we were unable to establish
+                        // an olm session for.  This time, we use a longer timeout, but we
+                        // do this in the background and don't block anything else while we
+                        // do this.  We only need to retry users from servers that didn't
+                        // respond the first time.
+                        const retryDevices = {};
+                        const failedServerMap = new Set;
+                        for (const server of failedServers) {
+                            failedServerMap.add(server);
+                        }
+                        const failedDevices = [];
+                        for (const {userId, deviceInfo} of errorDevices) {
+                            const userHS = userId.slice(userId.indexOf(":") + 1);
+                            if (failedServerMap.has(userHS)) {
+                                retryDevices[userId] = retryDevices[userId] || [];
+                                retryDevices[userId].push(deviceInfo);
+                            } else {
+                                // if we aren't going to retry, then handle it
+                                // as a failed device
+                                failedDevices.push({userId, deviceInfo});
+                            }
+                        }
 
-                    const failedDevices = [];
-                    await this._shareKeyWithDevices(
-                        session, key, payload, retryDevices, failedDevices,
-                    );
-
-                    const blockedMap = {};
-                    const filteredFailedDevices =
-                        await this._olmDevice.filterOutNotifiedErrorDevices(
-                            failedDevices,
+                        await this._shareKeyWithDevices(
+                            session, key, payload, retryDevices, failedDevices,
                         );
-                    for (const {userId, deviceInfo} of filteredFailedDevices) {
-                        blockedMap[userId] = blockedMap[userId] || {};
-                        // we use a similar format to what
-                        // olmlib.ensureOlmSessionsForDevices returns, so that
-                        // we can use the same function to split
-                        blockedMap[userId][deviceInfo.deviceId] = {
-                            device: {
-                                code: "m.no_olm",
-                                reason: WITHHELD_MESSAGES["m.no_olm"],
-                                deviceInfo,
-                            },
-                        };
 
-                        const deviceId = deviceInfo.deviceId;
-
-                        // mark this device as "handled" because we don't want to try
-                        // to claim a one-time-key for dead devices on every message.
-                        session.markSharedWithDevice(userId, deviceId, key.chain_index);
-                    }
-
-                    // notify devices that we couldn't get an olm session
-                    await this._notifyBlockedDevices(session, blockedMap);
-                })();
+                        await this._notifyFailedOlmDevices(session, key, failedDevices);
+                    })();
+                } else {
+                    await this._notifyFailedOlmDevices(session, key, errorDevices);
+                }
             })(),
             (async () => {
                 // also, notify blocked devices that they're blocked
@@ -690,12 +689,15 @@ MegolmEncryption.prototype.reshareKeyWithDevice = async function(
  *
  * @param {Number} [otkTimeout] The timeout in milliseconds when requesting
  *     one-time keys for establishing new olm sessions.
+ *
+ * @param {Array} [failedServers] An array to fill with remote servers that
+ *     failed to respond to one-time-key requests.
  */
 MegolmEncryption.prototype._shareKeyWithDevices = async function(
-    session, key, payload, devicesByUser, errorDevices, otkTimeout,
+    session, key, payload, devicesByUser, errorDevices, otkTimeout, failedServers,
 ) {
     const devicemap = await olmlib.ensureOlmSessionsForDevices(
-        this._olmDevice, this._baseApis, devicesByUser, otkTimeout,
+        this._olmDevice, this._baseApis, devicesByUser, otkTimeout, failedServers,
     );
 
     this._getDevicesWithoutSessions(devicemap, devicesByUser, errorDevices);
@@ -724,6 +726,52 @@ MegolmEncryption.prototype._shareKeyWithOlmSessions = async function(
     }
 };
 
+/**
+ * Notify devices that we weren't able to create olm sessions.
+ *
+ * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ *
+ * @param {object} key
+ *
+ * @param {Array<object>} failedDevices the devices that we were unable to
+ *     create olm sessions for, as returned by _shareKeyWithDevices
+ */
+MegolmEncryption.prototype._notifyFailedOlmDevices = async function(
+    session, key, failedDevices,
+) {
+    // mark the devices that failed as "handled" because we don't want to try
+    // to claim a one-time-key for dead devices on every message.
+    for (const {userId, deviceInfo} of failedDevices) {
+        const deviceId = deviceInfo.deviceId;
+
+        session.markSharedWithDevice(
+            userId, deviceId, key.chain_index,
+        );
+    }
+
+    const filteredFailedDevices =
+          await this._olmDevice.filterOutNotifiedErrorDevices(
+              failedDevices,
+          );
+    const blockedMap = {};
+    for (const {userId, deviceInfo} of filteredFailedDevices) {
+        blockedMap[userId] = blockedMap[userId] || {};
+        // we use a similar format to what
+        // olmlib.ensureOlmSessionsForDevices returns, so that
+        // we can use the same function to split
+        blockedMap[userId][deviceInfo.deviceId] = {
+            device: {
+                code: "m.no_olm",
+                reason: WITHHELD_MESSAGES["m.no_olm"],
+                deviceInfo,
+            },
+        };
+    }
+
+    // send the notifications
+    await this._notifyBlockedDevices(session, blockedMap);
+};
+
 /**
  * Notify blocked devices that they have been blocked.
  *
@@ -767,8 +815,6 @@ MegolmEncryption.prototype._notifyBlockedDevices = async function(
  * @param {module:models/room} room the room the event is in
  */
 MegolmEncryption.prototype.prepareToEncrypt = function(room) {
-    logger.debug(`Preparing to encrypt events for ${this._roomId}`);
-
     if (this.encryptionPreparation) {
         // We're already preparing something, so don't do anything else.
         // FIXME: check if we need to restart
@@ -776,6 +822,8 @@ MegolmEncryption.prototype.prepareToEncrypt = function(room) {
         return;
     }
 
+    logger.debug(`Preparing to encrypt events for ${this._roomId}`);
+
     this.encryptionPreparation = (async () => {
         const [devicesInRoom, blocked] = await this._getDevicesInRoom(room);
 
@@ -786,7 +834,7 @@ MegolmEncryption.prototype.prepareToEncrypt = function(room) {
             this._removeUnknownDevices(devicesInRoom);
         }
 
-        await this._ensureOutboundSession(devicesInRoom, blocked);
+        await this._ensureOutboundSession(devicesInRoom, blocked, true);
 
         delete this.encryptionPreparation;
     })();
@@ -955,8 +1003,10 @@ MegolmEncryption.prototype._getDevicesInRoom = async function(room) {
                 continue;
             }
 
+            const deviceTrust = this._crypto.checkDeviceTrust(userId, deviceId);
+
             if (userDevices[deviceId].isBlocked() ||
-                (userDevices[deviceId].isUnverified() && isBlacklisting)
+                (!deviceTrust.isVerified() && isBlacklisting)
             ) {
                 if (!blocked[userId]) {
                     blocked[userId] = {};

src/crypto/algorithms/olm.js

@@ -264,6 +264,25 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
  */
 OlmDecryption.prototype._decryptMessage = async function(
     theirDeviceIdentityKey, message,
+) {
+    // This is a wrapper that serialises decryptions of prekey messages, because
+    // otherwise we race between deciding we have no active sessions for the message
+    // and creating a new one, which we can only do once because it removes the OTK.
+    if (message.type !== 0) {
+        // not a prekey message: we can safely just try & decrypt it
+        return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+    } else {
+        const myPromise = this._olmDevice._olmPrekeyPromise.then(() => {
+            return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+        });
+        // we want the error, but don't propagate it to the next decryption
+        this._olmDevice._olmPrekeyPromise = myPromise.catch(() => {});
+        return await myPromise;
+    }
+};
+
+OlmDecryption.prototype._reallyDecryptMessage = async function(
+    theirDeviceIdentityKey, message,
 ) {
     const sessionIds = await this._olmDevice.getSessionIdsForDevice(
         theirDeviceIdentityKey,

src/crypto/index.js

@@ -39,7 +39,7 @@ import {
     UserTrustLevel,
     createCryptoStoreCacheCallbacks,
 } from './CrossSigning';
-import {SECRET_STORAGE_ALGORITHM_V1, SecretStorage} from './SecretStorage';
+import {SECRET_STORAGE_ALGORITHM_V1_AES, SecretStorage} from './SecretStorage';
 import {OutgoingRoomKeyRequestManager} from './OutgoingRoomKeyRequestManager';
 import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
 import {
@@ -55,6 +55,7 @@ import {InRoomChannel, InRoomRequests} from "./verification/request/InRoomChanne
 import {ToDeviceChannel, ToDeviceRequests} from "./verification/request/ToDeviceChannel";
 import * as httpApi from "../http-api";
 import {IllegalMethod} from "./verification/IllegalMethod";
+import {KeySignatureUploadError} from "../errors";
 
 const DeviceVerification = DeviceInfo.DeviceVerification;
 
@@ -119,6 +120,8 @@ export function Crypto(baseApis, sessionStore, userId, deviceId,
     this._onDeviceListUserCrossSigningUpdated =
         this._onDeviceListUserCrossSigningUpdated.bind(this);
 
+    this._trustCrossSignedDevices = true;
+
     this._reEmitter = new ReEmitter(this);
     this._baseApis = baseApis;
     this._sessionStore = sessionStore;
@@ -166,7 +169,9 @@ export function Crypto(baseApis, sessionStore, userId, deviceId,
     this._deviceList.on(
         'userCrossSigningUpdated', this._onDeviceListUserCrossSigningUpdated,
     );
-    this._reEmitter.reEmit(this._deviceList, ["crypto.devicesUpdated"]);
+    this._reEmitter.reEmit(this._deviceList, [
+        "crypto.devicesUpdated", "crypto.willUpdateDevices",
+    ]);
 
     // the last time we did a check for the number of one-time-keys on the
     // server.
@@ -303,7 +308,8 @@ Crypto.prototype.init = async function(opts) {
         'readonly', [IndexedDBCryptoStore.STORE_ACCOUNT],
         (txn) => {
             this._cryptoStore.getCrossSigningKeys(txn, (keys) => {
-                if (keys) {
+                // can be an empty object after resetting cross-signing keys, see _storeTrustedSelfKeys
+                if (keys && Object.keys(keys).length !== 0) {
                     logger.log("Loaded cross-signing public keys from crypto store");
                     this._crossSigningInfo.setKeys(keys);
                 }
@@ -318,13 +324,54 @@ Crypto.prototype.init = async function(opts) {
     this._checkAndStartKeyBackup();
 };
 
+/**
+ * Whether to trust a others users signatures of their devices.
+ * If false, devices will only be considered 'verified' if we have
+ * verified that device individually (effectively disabling cross-signing).
+ *
+ * Default: true
+ *
+ * @return {bool} True if trusting cross-signed devices
+ */
+Crypto.prototype.getCryptoTrustCrossSignedDevices = function() {
+    return this._trustCrossSignedDevices;
+};
+
+/**
+ * See getCryptoTrustCrossSignedDevices
+
+ * This may be set before initCrypto() is called to ensure no races occur.
+ *
+ * @param {bool} val True to trust cross-signed devices
+ */
+Crypto.prototype.setCryptoTrustCrossSignedDevices = function(val) {
+    this._trustCrossSignedDevices = val;
+
+    for (const userId of this._deviceList.getKnownUserIds()) {
+        const devices = this._deviceList.getRawStoredDevicesForUser(userId);
+        for (const deviceId of Object.keys(devices)) {
+            const deviceTrust = this.checkDeviceTrust(userId, deviceId);
+            // If the device is locally verified then isVerified() is always true,
+            // so this will only have caused the value to change if the device is
+            // cross-signing verified but not locally verified
+            if (
+                !deviceTrust.isLocallyVerified() &&
+                deviceTrust.isCrossSigningVerified()
+            ) {
+                const deviceObj = this._deviceList.getStoredDevice(userId, deviceId);
+                this.emit("deviceVerificationChanged", userId, deviceId, deviceObj);
+            }
+        }
+    }
+};
+
 /**
  * Create a recovery key from a user-supplied passphrase.
  *
  * @param {string} password Passphrase string that can be entered by the user
  *     when restoring the backup as an alternative to entering the recovery key.
  *     Optional.
- * @returns {Promise<Array>} Array with public key metadata, encoded private
+ * @returns {Promise<Object>} Object with public key metadata, encoded private
  *     recovery key which should be disposed of after displaying to the user,
  *     and raw private key to avoid round tripping if needed.
  */
@@ -345,18 +392,57 @@ Crypto.prototype.createRecoveryKeyFromPassphrase = async function(password) {
         }
         const privateKey = decryption.get_private_key();
         const encodedPrivateKey = encodeRecoveryKey(privateKey);
-        return [keyInfo, encodedPrivateKey, privateKey];
+        return { keyInfo, encodedPrivateKey, privateKey };
     } finally {
         if (decryption) decryption.free();
     }
 };
 
+/**
+ * Checks whether cross signing:
+ * - is enabled on this account
+ * - is trusted by this device
+ * - has private keys stored in secret storage
+ * and that the account has a secret storage key
+ *
+ * If this function returns false, bootstrapSecretStorage() can be used
+ * to fix things such that it returns true. That is to say, after
+ * bootstrapSecretStorage() completes sucessfully, this function should
+ * return true.
+ *
+ * The cross-signing API is currently UNSTABLE and may change without notice.
+ *
+ * @return {bool} True if cross-signing is ready to be used on this device
+ */
+Crypto.prototype.isCrossSigningReady = async function() {
+    const publicKeysOnDevice = this._crossSigningInfo.getId();
+    const privateKeysInStorage = await this._crossSigningInfo.isStoredInSecretStorage(
+        this._secretStorage,
+    );
+    const secretStorageKeyInAccount = await this._secretStorage.hasKey();
+
+    return (
+        publicKeysOnDevice &&
+        privateKeysInStorage &&
+        secretStorageKeyInAccount
+    );
+};
+
+
 /**
  * Bootstrap Secure Secret Storage if needed by creating a default key and
  * signing it with the cross-signing master key. If everything is already set
  * up, then no changes are made, so this is safe to run to ensure secret storage
  * is ready for use.
  *
+ * This function
+ * - creates a new Secure Secret Storage key if no default key exists
+ *   - if a key backup exists, it is migrated to store the key in the Secret
+ *     Storage
+ * - creates a backup if none exists, and one is requested
+ * - migrates Secure Secret Storage to use the latest algorithm, if an outdated
+ *   algorithm is found
+ *
  * @param {function} [opts.authUploadDeviceSigningKeys] Optional. Function
  * called to await an interactive auth flow when uploading device signing keys.
  * Args:
@@ -364,22 +450,30 @@ Crypto.prototype.createRecoveryKeyFromPassphrase = async function(password) {
  *     auth data as an object.
  * @param {function} [opts.createSecretStorageKey] Optional. Function
  * called to await a secret storage key creation flow.
+ * Returns:
+ *     {Promise<Object>} Object with public key metadata, encoded private
+ *     recovery key which should be disposed of after displaying to the user,
+ *     and raw private key to avoid round tripping if needed.
  * @param {object} [opts.keyBackupInfo] The current key backup object. If passed,
  * the passphrase and recovery key from this backup will be used.
  * @param {bool} [opts.setupNewKeyBackup] If true, a new key backup version will be
  * created and the private key stored in the new SSSS store. Ignored if keyBackupInfo
  * is supplied.
  * @param {bool} [opts.setupNewSecretStorage] Optional. Reset even if keys already exist.
+ * @param {func} [opts.getKeyBackupPassphrase] Optional. Function called to get the user's
+ *     current key backup passphrase. Should return a promise that resolves with a Buffer
+ *     containing the key, or rejects if the key cannot be obtained.
  * Returns:
  *     {Promise} A promise which resolves to key creation data for
  *     SecretStorage#addKey: an object with `passphrase` and/or `pubkey` fields.
  */
 Crypto.prototype.bootstrapSecretStorage = async function({
     authUploadDeviceSigningKeys,
-    createSecretStorageKey = async () => { },
+    createSecretStorageKey = async () => ({ }),
     keyBackupInfo,
     setupNewKeyBackup,
     setupNewSecretStorage,
+    getKeyBackupPassphrase,
 } = {}) {
     logger.log("Bootstrapping Secure Secret Storage");
 
@@ -394,93 +488,244 @@ Crypto.prototype.bootstrapSecretStorage = async function({
     // use temporary callbacks to weave them through the various APIs.
     const appCallbacks = Object.assign({}, this._baseApis._cryptoCallbacks);
 
-    try {
-        const inStorage = !setupNewSecretStorage &&
-            await this._crossSigningInfo.isStoredInSecretStorage(this._secretStorage);
-        if (!this._crossSigningInfo.getId() || !inStorage) {
-            logger.log(
-                "Cross-signing public and/or private keys not found, " +
-                "checking secret storage for private keys",
-            );
-            if (inStorage) {
-                logger.log("Cross-signing private keys found in secret storage");
-                await this.checkOwnCrossSigningTrust();
-            } else {
-                logger.log(
-                    "Cross-signing private keys not found in secret storage, " +
-                    "creating new keys",
-                );
-                this._baseApis._cryptoCallbacks.saveCrossSigningKeys =
-                    keys => Object.assign(crossSigningPrivateKeys, keys);
-                this._baseApis._cryptoCallbacks.getCrossSigningKey =
-                    name => crossSigningPrivateKeys[name];
-                await this.resetCrossSigningKeys(
-                    CrossSigningLevel.MASTER,
-                    { authUploadDeviceSigningKeys },
-                );
+    // the ID of the new SSSS key, if we create one
+    let newKeyId = null;
+
+    // cache SSSS keys so that we don't need to constantly pester the user about it
+    const ssssKeys = {};
+
+    this._baseApis._cryptoCallbacks.getSecretStorageKey =
+        async ({keys}, name) => {
+            // if we already have a key that works, return it
+            for (const keyId of Object.keys(keys)) {
+                if (ssssKeys[keyId]) {
+                    return [keyId, ssssKeys[keyId]];
+                }
             }
-        } else {
-            logger.log("Cross signing keys are present in secret storage");
+
+            // otherwise, prompt the user and cache it
+            const key = await appCallbacks.getSecretStorageKey({keys}, name);
+            if (key) {
+                const [keyId, keyData] = key;
+                ssssKeys[keyId] = keyData;
+            }
+            return key;
+        };
+
+    // create a new SSSS key and set it as default
+    const createSSSS = async (opts, privateKey) => {
+        opts = opts || {};
+        if (privateKey) {
+            opts.key = privateKey;
         }
 
-        // Check if Secure Secret Storage has a default key. If we don't have one, create
-        // the default key (which will also be signed by the cross-signing master key).
-        if (setupNewSecretStorage || !await this.hasSecretStorageKey()) {
-            let newKeyId;
-            if (keyBackupInfo) {
-                logger.log("Secret storage default key not found, using key backup key");
-                const opts = {
-                    pubkey: keyBackupInfo.auth_data.public_key,
-                };
+        const keyId = await this.addSecretStorageKey(
+            SECRET_STORAGE_ALGORITHM_V1_AES, opts,
+        );
+        await this.setDefaultSecretStorageKeyId(keyId);
 
-                if (
-                    keyBackupInfo.auth_data.private_key_salt &&
-                    keyBackupInfo.auth_data.private_key_iterations
-                ) {
-                    opts.passphrase = {
-                        algorithm: "m.pbkdf2",
-                        iterations: keyBackupInfo.auth_data.private_key_iterations,
-                        salt: keyBackupInfo.auth_data.private_key_salt,
-                    };
-                }
+        if (privateKey) {
+            // cache the private key so that we can access it again
+            ssssKeys[keyId] = privateKey;
+        }
+        return keyId;
+    };
 
-                newKeyId = await this.addSecretStorageKey(
-                    SECRET_STORAGE_ALGORITHM_V1, opts,
-                );
+    // reset the cross-signing keys
+    const resetCrossSigning = async () => {
+        this._baseApis._cryptoCallbacks.saveCrossSigningKeys =
+            keys => Object.assign(crossSigningPrivateKeys, keys);
+        this._baseApis._cryptoCallbacks.getCrossSigningKey =
+            name => crossSigningPrivateKeys[name];
+        await this.resetCrossSigningKeys(
+            CrossSigningLevel.MASTER,
+            { authUploadDeviceSigningKeys },
+        );
+    };
 
-                // Add an entry for the backup key in SSSS as a 'passthrough' key
-                // (ie. the secret is the key itself).
-                this._secretStorage.storePassthrough('m.megolm_backup.v1', newKeyId);
+    const ensureCanCheckPassphrase = async (keyId, keyInfo) => {
+        if (!keyInfo.mac) {
+            const key = await this._baseApis._cryptoCallbacks.getSecretStorageKey(
+                {keys: {[keyId]: keyInfo}}, "",
+            );
+            if (key) {
+                const keyData = key[1];
+                ssssKeys[keyId] = keyData;
+                const {iv, mac} = await SecretStorage._calculateKeyCheck(keyData);
+                keyInfo.iv = iv;
+                keyInfo.mac = mac;
 
-                // if this key backup is trusted, sign it with the cross signing key
-                // so the key backup can be trusted via cross-signing.
-                const backupSigStatus = await this.checkKeyBackup(keyBackupInfo);
-                if (backupSigStatus.trustInfo.usable) {
-                    console.log("Adding cross signing signature to key backup");
-                    await this._crossSigningInfo.signObject(
-                        keyBackupInfo.auth_data, "master",
-                    );
-                    await this._baseApis._http.authedRequest(
-                        undefined, "PUT", "/room_keys/version/" + keyBackupInfo.version,
-                        undefined, keyBackupInfo,
-                        {prefix: httpApi.PREFIX_UNSTABLE},
-                    );
-                } else {
-                    console.log(
-                        "Key backup is NOT TRUSTED: NOT adding cross signing signature",
-                    );
-                }
-            } else {
-                logger.log("Secret storage default key not found, creating new key");
-                const keyOptions = await createSecretStorageKey();
-                newKeyId = await this.addSecretStorageKey(
-                    SECRET_STORAGE_ALGORITHM_V1,
-                    keyOptions,
+                await this._crossSigningInfo.signObject(keyInfo, 'master');
+
+                await this._baseApis.setAccountData(
+                    `m.secret_storage.key.${keyId}`, keyInfo,
                 );
             }
-            await this.setDefaultSecretStorageKeyId(newKeyId);
+        }
+    };
+
+    try {
+        const oldSSSSKey = await this.getSecretStorageKey();
+        const [oldKeyId, oldKeyInfo] = oldSSSSKey || [null, null];
+        const decryptionKeys =
+              await this._crossSigningInfo.isStoredInSecretStorage(this._secretStorage);
+        const inStorage = !setupNewSecretStorage && decryptionKeys;
+
+        if (!inStorage && !keyBackupInfo) {
+            // either we don't have anything, or we've been asked to restart
+            // from scratch
+            logger.log(
+                "Cross-signing private keys not found in secret storage, " +
+                    "creating new keys",
+            );
+
+            await resetCrossSigning();
+
+            if (
+                setupNewSecretStorage ||
+                !oldKeyInfo ||
+                oldKeyInfo.algorithm !== SECRET_STORAGE_ALGORITHM_V1_AES
+            ) {
+                // if we already have a usable default SSSS key and aren't resetting SSSS just use it.
+                // otherwise, create a new one
+                // Note: we leave the old SSSS key in place: there could be other secrets using it, in theory.
+                // We could move them to the new key but a) that would mean we'd need to prompt for the old
+                // passphrase, and b) it's not clear that would be the right thing to do anyway.
+                const { keyInfo, privateKey } = await createSecretStorageKey();
+                newKeyId = await createSSSS(keyInfo, privateKey);
+            }
+
+            if (oldKeyInfo && oldKeyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                await ensureCanCheckPassphrase(oldKeyId, oldKeyInfo);
+            }
+        } else if (!inStorage && keyBackupInfo) {
+            // we have an existing backup, but no SSSS
+
+            logger.log("Secret storage default key not found, using key backup key");
+
+            // if we have the backup key already cached, use it; otherwise use the
+            // callback to prompt for the key
+            const backupKey = await this.getSessionBackupPrivateKey() ||
+                              await getKeyBackupPassphrase();
+
+            // create new cross-signing keys
+            await resetCrossSigning();
+
+            // create a new SSSS key and use the backup key as the new SSSS key
+            const opts = {};
+
+            if (
+                keyBackupInfo.auth_data.private_key_salt &&
+                keyBackupInfo.auth_data.private_key_iterations
+            ) {
+                opts.passphrase = {
+                    algorithm: "m.pbkdf2",
+                    iterations: keyBackupInfo.auth_data.private_key_iterations,
+                    salt: keyBackupInfo.auth_data.private_key_salt,
+                    bits: 256,
+                };
+            }
+
+            newKeyId = await createSSSS(opts, backupKey);
+
+            // store the backup key in secret storage
+            await this.storeSecret(
+                "m.megolm_backup.v1", olmlib.encodeBase64(backupKey), [newKeyId],
+            );
+
+            // The backup is trusted because the user provided the private key.
+            // Sign the backup with the cross signing key so the key backup can
+            // be trusted via cross-signing.
+            logger.log("Adding cross signing signature to key backup");
+            await this._crossSigningInfo.signObject(
+                keyBackupInfo.auth_data, "master",
+            );
+            await this._baseApis._http.authedRequest(
+                undefined, "PUT", "/room_keys/version/" + keyBackupInfo.version,
+                undefined, keyBackupInfo,
+                {prefix: httpApi.PREFIX_UNSTABLE},
+            );
+        } else if (!(Object.values(decryptionKeys).some(
+            info => info.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES,
+        ))) {
+            // we have an asymmetric SSSS
+            logger.log("Asymmetric SSSS found. Switching SSSS to symmetric");
+            const keys = {};
+            // fetch the cross-signing private keys (needed to sign the new
+            // SSSS key, and so that we can re-encrypt them with the new key).
+            // We store the cross-signing keys, and temporarily set a callback
+            // so that when the private keys are needed while setting things
+            // up, we can provide it.
+            this._baseApis._cryptoCallbacks.getCrossSigningKey =
+                name => crossSigningPrivateKeys[name];
+            for (const type of ["master", "self_signing", "user_signing"]) {
+                const secretName = `m.cross_signing.${type}`;
+                const secret = await this.getSecret(secretName);
+                keys[type] = secret;
+                crossSigningPrivateKeys[type]
+                    = new Uint8Array(olmlib.decodeBase64(secret));
+            }
+
+            await this.checkOwnCrossSigningTrust();
+
+            const opts = {};
+
+            let oldKey = null;
+            for (const [keyId, keyInfo] of Object.entries(decryptionKeys)) {
+                // See if the old key was generated from a passphrase.  If
+                // yes, use the same settings.
+                if (keyId in ssssKeys) {
+                    oldKey = ssssKeys[keyId];
+                    if (keyInfo.passphrase) {
+                        opts.passphrase = keyInfo.passphrase;
+                    }
+                    break;
+                }
+            }
+
+            // create new symmetric SSSS key
+            newKeyId = await createSSSS(opts, oldKey);
+
+            logger.log("re-encrypting cross-signing keys");
+            // re-encrypt all the cross-signing keys with the new key
+            for (const type of ["master", "self_signing", "user_signing"]) {
+                const secretName = `m.cross_signing.${type}`;
+                await this.storeSecret(secretName, keys[type], [newKeyId]);
+            }
+
+            if (await this.isSecretStored("m.megolm_backup.v1", false)) {
+                // re-encrypt the backup key if we had one too
+                logger.log("re-encrypting backup key");
+                const backupKey = await this.getSecret("m.megolm_backup.v1");
+
+                await this.storeSecret(
+                    "m.megolm_backup.v1", fixBackupKey(backupKey) || backupKey,
+                    [newKeyId],
+                );
+            }
+        } else if (!this._crossSigningInfo.getId()) {
+            // we have SSSS, but we don't know if the server's cross-signing
+            // keys should be trusted
+            logger.log("Cross-signing private keys found in secret storage");
+
+            // fetch the private keys and set up our local copy of the keys for
+            // use
+            await this.checkOwnCrossSigningTrust();
+
+            if (oldKeyInfo && oldKeyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                // make sure that the default key has the information needed to
+                // check the passphrase
+                await ensureCanCheckPassphrase(oldKeyId, oldKeyInfo);
+            }
         } else {
-            logger.log("Have secret storage key");
+            // we have SSSS and we cross-signing is already set up
+            logger.log("Cross signing keys are present in secret storage");
+
+            if (oldKeyInfo && oldKeyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                // make sure that the default key has the information needed to
+                // check the passphrase
+                await ensureCanCheckPassphrase(oldKeyId, oldKeyInfo);
+            }
         }
 
         // If cross-signing keys were reset, store them in Secure Secret Storage.
@@ -511,6 +756,33 @@ Crypto.prototype.bootstrapSecretStorage = async function({
             );
             await this._baseApis.createKeyBackupVersion(info);
         }
+
+        // Call `getCrossSigningKey` for side effect of caching private keys for
+        // future gossiping to other devices if enabled via app level callbacks.
+        if (this._crossSigningInfo._cacheCallbacks) {
+            for (const type of ["self_signing", "user_signing"]) {
+                logger.log(`Cache ${type} cross-signing private key locally`);
+                await this._crossSigningInfo.getCrossSigningKey(type);
+            }
+        }
+
+        // and likewise for the session backup key
+        const sessionBackupKey = await this.getSecret('m.megolm_backup.v1');
+        if (sessionBackupKey) {
+            logger.info("Got session backup key from secret storage: caching");
+            // fix up the backup key if it's in the wrong format, and replace
+            // in secret storage
+            const fixedBackupKey = fixBackupKey(sessionBackupKey);
+            if (fixedBackupKey) {
+                await this.storeSecret(
+                    "m.megolm_backup.v1", fixedBackupKey, [newKeyId || oldKeyId],
+                );
+            }
+            const decodedBackupKey = new Uint8Array(olmlib.decodeBase64(
+                fixedBackupKey || sessionBackupKey,
+            ));
+            await this.storeSessionBackupPrivateKey(decodedBackupKey);
+        }
     } finally {
         // Restore the original callbacks. NB. we must do this by manipulating
         // the same object since the CrossSigning class has a reference to the
@@ -526,6 +798,26 @@ Crypto.prototype.bootstrapSecretStorage = async function({
     logger.log("Secure Secret Storage ready");
 };
 
+/**
+ * Fix up the backup key, that may be in the wrong format due to a bug in a
+ * migration step.  Some backup keys were stored as a comma-separated list of
+ * integers, rather than a base64-encoded byte array.  If this function is
+ * passed a string that looks like a list of integers rather than a base64
+ * string, it will attempt to convert it to the right format.
+ *
+ * @param {string} key the key to check
+ * @returns {null | string} If the key is in the wrong format, then the fixed
+ * key will be returned. Otherwise null will be returned.
+ *
+ */
+export function fixBackupKey(key) {
+    if (key.indexOf(",") < 0) {
+        return null;
+    }
+    const fixedKey = Uint8Array.from(key.split(","), x => parseInt(x));
+    return olmlib.encodeBase64(fixedKey);
+}
+
 Crypto.prototype.addSecretStorageKey = function(algorithm, opts, keyID) {
     return this._secretStorage.addKey(algorithm, opts, keyID);
 };
@@ -534,6 +826,14 @@ Crypto.prototype.hasSecretStorageKey = function(keyID) {
     return this._secretStorage.hasKey(keyID);
 };
 
+Crypto.prototype.secretStorageKeyNeedsUpgrade = function(keyID) {
+    return this._secretStorage.keyNeedsUpgrade(keyID);
+};
+
+Crypto.prototype.getSecretStorageKey = function(keyID) {
+    return this._secretStorage.getKey(keyID);
+};
+
 Crypto.prototype.storeSecret = function(name, secret, keys) {
     return this._secretStorage.store(name, secret, keys);
 };
@@ -561,6 +861,10 @@ Crypto.prototype.setDefaultSecretStorageKeyId = function(k) {
     return this._secretStorage.setDefaultKeyId(k);
 };
 
+Crypto.prototype.checkSecretStorageKey = function(key, info) {
+    return this._secretStorage.checkKey(key, info);
+};
+
 /**
  * Checks that a given secret storage private key matches a given public key.
  * This can be used by the getSecretStorageKey callback to verify that the
@@ -582,6 +886,51 @@ Crypto.prototype.checkSecretStoragePrivateKey = function(privateKey, expectedPub
     }
 };
 
+/**
+ * Fetches the backup private key, if cached
+ * @returns {Promise} the key, if any, or null
+ */
+Crypto.prototype.getSessionBackupPrivateKey = async function() {
+    let key = await new Promise((resolve) => {
+        this._cryptoStore.doTxn(
+            'readonly',
+            [IndexedDBCryptoStore.STORE_ACCOUNT],
+            (txn) => {
+                this._cryptoStore.getSecretStorePrivateKey(
+                    txn,
+                    resolve,
+                    "m.megolm_backup.v1",
+                );
+            },
+        );
+    });
+
+    // make sure we have a Uint8Array, rather than a string
+    if (key && typeof(key === "string")) {
+        key = new Uint8Array(olmlib.decodeBase64(fixBackupKey(key) || key));
+        await this.storeSessionBackupPrivateKey(key);
+    }
+    return key;
+};
+
+/**
+ * Stores the session backup key to the cache
+ * @param {Uint8Array} key the private key
+ * @returns {Promise} so you can catch failures
+ */
+Crypto.prototype.storeSessionBackupPrivateKey = async function(key) {
+    if (!(key instanceof Uint8Array)) {
+        throw new Error(`storeSessionBackupPrivateKey expects Uint8Array, got ${key}`);
+    }
+    return this._cryptoStore.doTxn(
+        'readwrite',
+        [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._cryptoStore.storeSecretStorePrivateKey(txn, "m.megolm_backup.v1", key);
+        },
+    );
+};
+
 /**
  * Checks that a given cross-signing private key matches a given public key.
  * This can be used by the getCrossSigningKey callback to verify that the
@@ -667,15 +1016,34 @@ Crypto.prototype._afterCrossSigningLocalKeyChange = async function() {
     const device = this._deviceList.getStoredDevice(this._userId, this._deviceId);
     const signedDevice = await this._crossSigningInfo.signDevice(this._userId, device);
     logger.info(`Starting background key sig upload for ${this._deviceId}`);
-    this._baseApis.uploadKeySignatures({
-        [this._userId]: {
-            [this._deviceId]: signedDevice,
-        },
-    }).then(() => {
-        logger.info(`Finished background key sig upload for ${this._deviceId}`);
-    }).catch(e => {
-        logger.error(`Error during background key sig upload for ${this._deviceId}`, e);
-    });
+
+    const upload = ({ shouldEmit }) => {
+        return this._baseApis.uploadKeySignatures({
+            [this._userId]: {
+                [this._deviceId]: signedDevice,
+            },
+        }).then((response) => {
+            const { failures } = response || {};
+            if (Object.keys(failures || []).length > 0) {
+                if (shouldEmit) {
+                    this._baseApis.emit(
+                        "crypto.keySignatureUploadFailure",
+                        failures,
+                        "_afterCrossSigningLocalKeyChange",
+                        upload, // continuation
+                    );
+                }
+                throw new KeySignatureUploadError("Key upload failed", { failures });
+            }
+            logger.info(`Finished background key sig upload for ${this._deviceId}`);
+        }).catch(e => {
+            logger.error(
+                `Error during background key sig upload for ${this._deviceId}`,
+                e,
+            );
+        });
+    };
+    upload({ shouldEmit: true });
 
     const shouldUpgradeCb = (
         this._baseApis._cryptoCallbacks.shouldUpgradeDeviceVerifications
@@ -815,7 +1183,7 @@ Crypto.prototype.getStoredCrossSigningForUser = function(userId) {
 Crypto.prototype.checkUserTrust = function(userId) {
     const userCrossSigning = this._deviceList.getStoredCrossSigningForUser(userId);
     if (!userCrossSigning) {
-        return new UserTrustLevel(false, false);
+        return new UserTrustLevel(false, false, false);
     }
     return this._crossSigningInfo.checkUserTrust(userCrossSigning);
 };
@@ -846,11 +1214,14 @@ Crypto.prototype._checkDeviceInfoTrust = function(userId, device) {
 
     const userCrossSigning = this._deviceList.getStoredCrossSigningForUser(userId);
     if (device && userCrossSigning) {
+        // The _trustCrossSignedDevices only affects trust of other people's cross-signing
+        // signatures
+        const trustCrossSig = this._trustCrossSignedDevices || userId === this._userId;
         return this._crossSigningInfo.checkDeviceTrust(
-            userCrossSigning, device, trustedLocally,
+            userCrossSigning, device, trustedLocally, trustCrossSig,
         );
     } else {
-        return new DeviceTrustLevel(false, false, trustedLocally);
+        return new DeviceTrustLevel(false, false, trustedLocally, false);
     }
 };
 
@@ -870,16 +1241,34 @@ Crypto.prototype._onDeviceListUserCrossSigningUpdated = async function(userId) {
             // If it's not changed, just make sure everything is up to date
             await this.checkOwnCrossSigningTrust();
         } else {
-            this.emit("crossSigning.keysChanged", {});
             // We'll now be in a state where cross-signing on the account is not trusted
             // because our locally stored cross-signing keys will not match the ones
-            // on the server for our account. The app must call checkOwnCrossSigningTrust()
-            // to fix this.
-            // XXX: Do we need to do something to emit events saying every device has become
-            // untrusted?
+            // on the server for our account. So we clear our own stored cross-signing keys,
+            // effectively disabling cross-signing until the user gets verified by the device
+            // that reset the keys
+            this._storeTrustedSelfKeys(null);
+            // emit cross-signing has been disabled
+            this.emit("crossSigning.keysChanged", {});
+            // as the trust for our own user has changed,
+            // also emit an event for this
+            this.emit("userTrustStatusChanged",
+                this._userId, this.checkUserTrust(userId));
         }
     } else {
         await this._checkDeviceVerifications(userId);
+
+        // Update verified before latch using the current state and save the new
+        // latch value in the device list store.
+        const crossSigning = this._deviceList.getStoredCrossSigningForUser(userId);
+        if (crossSigning) {
+            crossSigning.updateCrossSigningVerifiedBefore(
+                this.checkUserTrust(userId).isCrossSigningVerified(),
+            );
+            this._deviceList.setRawStoredCrossSigningForUser(
+                userId, crossSigning.toStorage(),
+            );
+        }
+
         this.emit("userTrustStatusChanged", userId, this.checkUserTrust(userId));
     }
 };
@@ -952,19 +1341,52 @@ Crypto.prototype.checkOwnCrossSigningTrust = async function() {
     }
 
     if (masterChanged) {
-        await this._signObject(this._crossSigningInfo.keys.master);
-        keySignatures[this._crossSigningInfo.getId()]
-            = this._crossSigningInfo.keys.master;
+        const masterKey = this._crossSigningInfo.keys.master;
+        await this._signObject(masterKey);
+        const deviceSig = masterKey.signatures[this._userId]["ed25519:" + this._deviceId];
+        // Include only the _new_ device signature in the upload.
+        // We may have existing signatures from deleted devices, which will cause
+        // the entire upload to fail.
+        keySignatures[this._crossSigningInfo.getId()] = Object.assign(
+            {},
+            masterKey,
+            {
+                signatures: {
+                    [this._userId]: {
+                        ["ed25519:" + this._deviceId]: deviceSig,
+                    },
+                },
+            },
+        );
     }
 
     const keysToUpload = Object.keys(keySignatures);
     if (keysToUpload.length) {
-        logger.info(`Starting background key sig upload for ${keysToUpload}`);
-        this._baseApis.uploadKeySignatures({ [this._userId]: keySignatures }).then(() => {
-            logger.info(`Finished background key sig upload for ${keysToUpload}`);
-        }).catch(e => {
-            logger.error(`Error during background key sig upload for ${keysToUpload}`, e);
-        });
+        const upload = ({ shouldEmit }) => {
+            logger.info(`Starting background key sig upload for ${keysToUpload}`);
+            return this._baseApis.uploadKeySignatures({ [this._userId]: keySignatures })
+            .then((response) => {
+                const { failures } = response || {};
+                logger.info(`Finished background key sig upload for ${keysToUpload}`);
+                if (Object.keys(failures || []).length > 0) {
+                    if (shouldEmit) {
+                        this._baseApis.emit(
+                            "crypto.keySignatureUploadFailure",
+                            failures,
+                            "checkOwnCrossSigningTrust",
+                            upload,
+                        );
+                    }
+                    throw new KeySignatureUploadError("Key upload failed", { failures });
+                }
+            }).catch(e => {
+                logger.error(
+                    `Error during background key sig upload for ${keysToUpload}`,
+                    e,
+                );
+            });
+        };
+        upload({ shouldEmit: true });
     }
 
     this.emit("userTrustStatusChanged", userId, this.checkUserTrust(userId));
@@ -986,7 +1408,11 @@ Crypto.prototype.checkOwnCrossSigningTrust = async function() {
  * @param {object} keys The new trusted set of keys
  */
 Crypto.prototype._storeTrustedSelfKeys = async function(keys) {
-    this._crossSigningInfo.setKeys(keys);
+    if (keys) {
+        this._crossSigningInfo.setKeys(keys);
+    } else {
+        this._crossSigningInfo.clearKeys();
+    }
     await this._cryptoStore.doTxn(
         'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
         (txn) => {
@@ -1605,12 +2031,33 @@ Crypto.prototype.setDeviceVerification = async function(
             );
             const device = await this._crossSigningInfo.signUser(xsk);
             if (device) {
-                logger.info("Uploading signature for " + userId + "...");
-                await this._baseApis.uploadKeySignatures({
-                    [userId]: {
-                        [deviceId]: device,
-                    },
-                });
+                const upload = async ({ shouldEmit }) => {
+                    logger.info("Uploading signature for " + userId + "...");
+                    const response = await this._baseApis.uploadKeySignatures({
+                        [userId]: {
+                            [deviceId]: device,
+                        },
+                    });
+                    const { failures } = response || {};
+                    if (Object.keys(failures || []).length > 0) {
+                        if (shouldEmit) {
+                            this._baseApis.emit(
+                                "crypto.keySignatureUploadFailure",
+                                failures,
+                                "setDeviceVerification",
+                                upload,
+                            );
+                        }
+                        /* Throwing here causes the process to be cancelled and the other
+                        * user to be notified */
+                        throw new KeySignatureUploadError(
+                            "Key upload failed",
+                            { failures },
+                        );
+                    }
+                };
+                await upload({ shouldEmit: true });
+
                 // This will emit events when it comes back down the sync
                 // (we could do local echo to speed things up)
             }
@@ -1659,12 +2106,27 @@ Crypto.prototype.setDeviceVerification = async function(
             userId, DeviceInfo.fromStorage(dev, deviceId),
         );
         if (device) {
-            logger.info("Uploading signature for " + deviceId);
-            await this._baseApis.uploadKeySignatures({
-                [userId]: {
-                    [deviceId]: device,
-                },
-            });
+            const upload = async ({shouldEmit}) => {
+                logger.info("Uploading signature for " + deviceId);
+                const response = await this._baseApis.uploadKeySignatures({
+                    [userId]: {
+                        [deviceId]: device,
+                    },
+                });
+                const { failures } = response || {};
+                if (Object.keys(failures || []).length > 0) {
+                    if (shouldEmit) {
+                        this._baseApis.emit(
+                            "crypto.keySignatureUploadFailure",
+                            failures,
+                            "setDeviceVerification",
+                            upload, // continuation
+                        );
+                    }
+                    throw new KeySignatureUploadError("Key upload failed", { failures });
+                }
+            };
+            await upload({shouldEmit: true});
             // XXX: we'll need to wait for the device list to be updated
         }
     }
@@ -1701,7 +2163,8 @@ Crypto.prototype.requestVerification = function(userId, devices) {
     if (existingRequest) {
         return Promise.resolve(existingRequest);
     }
-    const channel = new ToDeviceChannel(this._baseApis, userId, devices);
+    const channel = new ToDeviceChannel(this._baseApis, userId, devices,
+        ToDeviceChannel.makeTransactionId());
     return this._requestVerificationWithChannel(
         userId,
         channel,
@@ -1714,6 +2177,10 @@ Crypto.prototype._requestVerificationWithChannel = async function(
 ) {
     let request = new VerificationRequest(
         channel, this._verificationMethods, this._baseApis);
+    // if transaction id is already known, add request
+    if (channel.transactionId) {
+        requestsMap.setRequestByChannel(channel, request);
+    }
     await request.sendRequest();
     // don't replace the request created by a racing remote echo
     const racingRequest = requestsMap.getRequestByChannel(channel);
@@ -1751,6 +2218,27 @@ Crypto.prototype.beginKeyVerification = function(
     return request.beginKeyVerification(method, {userId, deviceId});
 };
 
+Crypto.prototype.legacyDeviceVerification = async function(
+    userId, deviceId, method,
+) {
+    const transactionId = ToDeviceChannel.makeTransactionId();
+    const channel = new ToDeviceChannel(
+        this._baseApis, userId, [deviceId], transactionId, deviceId);
+    const request = new VerificationRequest(
+        channel, this._verificationMethods, this._baseApis);
+    this._toDeviceVerificationRequests.setRequestBySenderAndTxnId(
+        userId, transactionId, request);
+    const verifier = request.beginKeyVerification(method, {userId, deviceId});
+    // either reject by an error from verify() while sending .start
+    // or resolve when the request receives the
+    // local (fake remote) echo for sending the .start event
+    await Promise.race([
+        verifier.verify(),
+        request.waitFor(r => r.started),
+    ]);
+    return request;
+};
+
 
 /**
  * Get information on the active olm sessions with a user
@@ -2252,6 +2740,20 @@ Crypto.prototype.flagAllGroupSessionsForBackup = async function() {
     return remaining;
 };
 
+/**
+ * Perform any background tasks that can be done before a message is ready to
+ * send, in order to speed up sending of the message.
+ *
+ * @param {module:models/room} room the room the event is in
+ */
+Crypto.prototype.prepareToEncrypt = function(room) {
+    const roomId = room.roomId;
+    const alg = this._roomEncryptors[roomId];
+    if (alg) {
+        alg.prepareToEncrypt(room);
+    }
+};
+
 /* eslint-disable valid-jsdoc */    //https://github.com/eslint/eslint/issues/7307
 /**
  * Encrypt an event according to the configuration of the room.
@@ -2395,6 +2897,14 @@ Crypto.prototype.cancelRoomKeyRequest = function(requestBody) {
     });
 };
 
+/**
+ * Re-send any outgoing key requests, eg after verification
+ * @returns {Promise}
+ */
+Crypto.prototype.cancelAndResendAllOutgoingKeyRequests = function() {
+    return this._outgoingRoomKeyRequestManager.cancelAndResendAllOutgoingRequests();
+};
+
 /**
  * handle an m.room.encryption event
  *

src/crypto/key_passphrase.js

@@ -19,6 +19,8 @@ import {randomString} from '../randomstring';
 
 const DEFAULT_ITERATIONS = 500000;
 
+const DEFAULT_BITSIZE = 256;
+
 export async function keyFromAuthData(authData, password) {
     if (!global.Olm) {
         throw new Error("Olm is not available");
@@ -34,6 +36,7 @@ export async function keyFromAuthData(authData, password) {
     return await deriveKey(
         password, authData.private_key_salt,
         authData.private_key_iterations,
+        authData.private_key_bits || DEFAULT_BITSIZE,
     );
 }
 
@@ -44,12 +47,12 @@ export async function keyFromPassphrase(password) {
 
     const salt = randomString(32);
 
-    const key = await deriveKey(password, salt, DEFAULT_ITERATIONS);
+    const key = await deriveKey(password, salt, DEFAULT_ITERATIONS, DEFAULT_BITSIZE);
 
     return { key, salt, iterations: DEFAULT_ITERATIONS };
 }
 
-export async function deriveKey(password, salt, iterations) {
+export async function deriveKey(password, salt, iterations, numBits = DEFAULT_BITSIZE) {
     const subtleCrypto = global.crypto.subtle;
     const TextEncoder = global.TextEncoder;
     if (!subtleCrypto || !TextEncoder) {
@@ -73,7 +76,7 @@ export async function deriveKey(password, salt, iterations) {
             hash: 'SHA-512',
         },
         key,
-        global.Olm.PRIVATE_KEY_LENGTH * 8,
+        numBits,
     );
 
     return new Uint8Array(keybits);

src/crypto/olmlib.js

@@ -180,14 +180,18 @@ export async function getExistingOlmSessions(
  * @param {Number} [otkTimeout] The timeout in milliseconds when requesting
  *     one-time keys for establishing new olm sessions.
  *
+ * @param {Array} [failedServers] An array to fill with remote servers that
+ *     failed to respond to one-time-key requests.
+ *
  * @return {Promise} resolves once the sessions are complete, to
  *    an Object mapping from userId to deviceId to
  *    {@link module:crypto~OlmSessionResult}
  */
 export async function ensureOlmSessionsForDevices(
-    olmDevice, baseApis, devicesByUser, force, otkTimeout,
+    olmDevice, baseApis, devicesByUser, force, otkTimeout, failedServers,
 ) {
     if (typeof force === "number") {
+        failedServers = otkTimeout;
         otkTimeout = force;
         force = false;
     }
@@ -261,6 +265,10 @@ export async function ensureOlmSessionsForDevices(
         throw e;
     }
 
+    if (failedServers && "failures" in res) {
+        failedServers.push(...Object.keys(res.failures));
+    }
+
     const otk_res = res.one_time_keys || {};
     const promises = [];
     for (const [userId, devices] of Object.entries(devicesByUser)) {

src/crypto/recoverykey.js

@@ -59,8 +59,8 @@ export function decodeRecoveryKey(recoverykey) {
         throw new Error("Incorrect length");
     }
 
-    return result.slice(
+    return Uint8Array.from(result.slice(
         OLM_RECOVERY_KEY_PREFIX.length,
         OLM_RECOVERY_KEY_PREFIX.length + global.Olm.PRIVATE_KEY_LENGTH,
-    );
+    ));
 }

src/crypto/store/indexeddb-crypto-store-backend.js

@@ -203,6 +203,23 @@ export class Backend {
         return promiseifyTxn(txn).then(() => result);
     }
 
+    /**
+     *
+     * @param {Number} wantedState
+     * @return {Promise<Array<*>>} All elements in a given state
+     */
+    getAllOutgoingRoomKeyRequestsByState(wantedState) {
+        return new Promise((resolve, reject) => {
+            const txn = this._db.transaction("outgoingRoomKeyRequests", "readonly");
+            const store = txn.objectStore("outgoingRoomKeyRequests");
+            const index = store.index("state");
+            const request = index.getAll(wantedState);
+
+            request.onsuccess = (ev) => resolve(ev.target.result);
+            request.onerror = (ev) => reject(ev.target.error);
+        });
+    }
+
     getOutgoingRoomKeyRequestsByTarget(userId, deviceId, wantedStates) {
         let stateIndex = 0;
         const results = [];
@@ -341,7 +358,7 @@ export class Backend {
         };
     }
 
-    getCrossSigningPrivateKey(txn, func, type) {
+    getSecretStorePrivateKey(txn, func, type) {
         const objectStore = txn.objectStore("account");
         const getReq = objectStore.get(`ssss_cache:${type}`);
         getReq.onsuccess = function() {
@@ -358,7 +375,7 @@ export class Backend {
         objectStore.put(keys, "crossSigningKeys");
     }
 
-    storeCrossSigningPrivateKey(txn, type, key) {
+    storeSecretStorePrivateKey(txn, type, key) {
         const objectStore = txn.objectStore("account");
         objectStore.put(key, `ssss_cache:${type}`);
     }

src/crypto/store/indexeddb-crypto-store.js

@@ -225,6 +225,17 @@ export class IndexedDBCryptoStore {
         return this._backend.getOutgoingRoomKeyRequestByState(wantedStates);
     }
 
+    /**
+     * Look for room key requests by state –
+     * unlike above, return a list of all entries in one state.
+     *
+     * @param {Number} wantedState
+     * @return {Promise<Array<*>>} Returns an array of requests in the given state
+     */
+    getAllOutgoingRoomKeyRequestsByState(wantedState) {
+        return this._backend.getAllOutgoingRoomKeyRequestsByState(wantedState);
+    }
+
     /**
      * Look for room key requests by target device and state
      *
@@ -313,8 +324,8 @@ export class IndexedDBCryptoStore {
      * @param {function(string)} func Called with the private key
      * @param {string} type A key type
      */
-    getCrossSigningPrivateKey(txn, func, type) {
-        this._backend.getCrossSigningPrivateKey(txn, func, type);
+    getSecretStorePrivateKey(txn, func, type) {
+        this._backend.getSecretStorePrivateKey(txn, func, type);
     }
 
     /**
@@ -334,8 +345,8 @@ export class IndexedDBCryptoStore {
      * @param {string} type The type of cross-signing private key to store
      * @param {string} key keys object as getCrossSigningKeys()
      */
-    storeCrossSigningPrivateKey(txn, type, key) {
-        this._backend.storeCrossSigningPrivateKey(txn, type, key);
+    storeSecretStorePrivateKey(txn, type, key) {
+        this._backend.storeSecretStorePrivateKey(txn, type, key);
     }
 
     // Olm sessions

src/crypto/store/localStorage-crypto-store.js

@@ -367,7 +367,7 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore {
         func(keys);
     }
 
-    getCrossSigningPrivateKey(txn, func, type) {
+    getSecretStorePrivateKey(txn, func, type) {
         const key = getJsonItem(this.store, E2E_PREFIX + `ssss_cache.${type}`);
         func(key ? Uint8Array.from(key) : key);
     }
@@ -378,7 +378,7 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore {
         );
     }
 
-    storeCrossSigningPrivateKey(txn, type, key) {
+    storeSecretStorePrivateKey(txn, type, key) {
         setJsonItem(
             this.store, E2E_PREFIX + `ssss_cache.${type}`, Array.from(key),
         );

src/crypto/store/memory-crypto-store.js

@@ -34,6 +34,7 @@ export class MemoryCryptoStore {
         this._account = null;
         this._crossSigningKeys = null;
         this._privateKeys = {};
+        this._backupKeys = {};
 
         // Map of {devicekey -> {sessionId -> session pickle}}
         this._sessions = {};
@@ -165,6 +166,19 @@ export class MemoryCryptoStore {
         return Promise.resolve(null);
     }
 
+    /**
+     *
+     * @param {Number} wantedState
+     * @return {Promise<Array<*>>} All OutgoingRoomKeyRequests in state
+     */
+    getAllOutgoingRoomKeyRequestsByState(wantedState) {
+        return Promise.resolve(
+            this._outgoingRoomKeyRequests.filter(
+                (r) => r.state == wantedState,
+            ),
+        );
+    }
+
     getOutgoingRoomKeyRequestsByTarget(userId, deviceId, wantedStates) {
         const results = [];
 
@@ -256,7 +270,7 @@ export class MemoryCryptoStore {
         func(this._crossSigningKeys);
     }
 
-    getCrossSigningPrivateKey(txn, func, type) {
+    getSecretStorePrivateKey(txn, func, type) {
         const result = this._privateKeys[type];
         return func(result || null);
     }
@@ -265,7 +279,7 @@ export class MemoryCryptoStore {
         this._crossSigningKeys = keys;
     }
 
-    storeCrossSigningPrivateKey(txn, type, key) {
+    storeSecretStorePrivateKey(txn, type, key) {
         this._privateKeys[type] = key;
     }
 

src/crypto/verification/Base.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2018 New Vector Ltd
+Copyright 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -121,6 +122,11 @@ export class VerificationBase extends EventEmitter {
         if (this._done) {
             return Promise.reject(new Error("Verification is already done"));
         }
+        const existingEvent = this.request.getEventFromOtherParty(type);
+        if (existingEvent) {
+            return Promise.resolve(existingEvent);
+        }
+
         this._expectedEvent = type;
         return new Promise((resolve, reject) => {
             this._resolveEvent = resolve;
@@ -198,13 +204,14 @@ export class VerificationBase extends EventEmitter {
             if (this._baseApis.getUserId() !== this.userId) {
                 return;
             }
+            // FIXME: This is a lot of logic that isn't anything to do with verification
+            // and probably ought to be somewhere else.
             console.log("VerificationBase.done: Self-verification done; requesting keys");
             /* This happens asynchronously, and we're not concerned about
              * waiting for it.  We return here in order to test. */
             return new Promise((resolve, reject) => {
                 const client = this._baseApis;
                 const original = client._crypto._crossSigningInfo;
-                const storage = client._crypto._secretStorage;
 
                 /* We already have all of the infrastructure we need to validate and
                  * cache cross-signing keys, so instead of replicating that, here we
@@ -215,8 +222,9 @@ export class VerificationBase extends EventEmitter {
                     { getCrossSigningKey: async (type) => {
                         console.debug("VerificationBase.done: requesting secret",
                                       type, this.deviceId);
-                        const { promise } =
-                            storage.request(`m.cross_signing.${type}`, [this.deviceId]);
+                        const { promise } = client.requestSecret(
+                            `m.cross_signing.${type}`, [this.deviceId],
+                        );
                         const result = await promise;
                         const decoded = decodeBase64(result);
                         return Uint8Array.from(decoded);
@@ -237,11 +245,39 @@ export class VerificationBase extends EventEmitter {
                     );
                 });
 
+                // also request and cache the key backup key
+                const backupKeyPromise = new Promise(async resolve => {
+                    const cachedKey = await client._crypto.getSessionBackupPrivateKey();
+                    if (!cachedKey) {
+                        logger.info("No cached backup key found. Requesting...");
+                        const secretReq = client.requestSecret(
+                            'm.megolm_backup.v1', [this.deviceId],
+                        );
+                        const base64Key = await secretReq.promise;
+                        logger.info("Got key backup key, decoding...");
+                        const decodedKey = decodeBase64(base64Key);
+                        logger.info("Decoded backup key, storing...");
+                        client._crypto.storeSessionBackupPrivateKey(
+                            Uint8Array.from(decodedKey),
+                        );
+                        logger.info("Backup key stored. Starting backup restore...");
+                        const backupInfo = await client.getKeyBackupVersion();
+                        // no need to await for this - just let it go in the bg
+                        client.restoreKeyBackupWithCache(
+                            undefined, undefined, backupInfo,
+                        ).then(() => {
+                            logger.info("Backup restored.");
+                        });
+                    }
+                    resolve();
+                });
+
                 /* We call getCrossSigningKey() for its side-effects */
                 return Promise.race([
                     Promise.all([
                         crossSigning.getCrossSigningKey("self_signing"),
                         crossSigning.getCrossSigningKey("user_signing"),
+                        backupKeyPromise,
                     ]),
                     timeout,
                 ]).then(resolve, reject);
@@ -256,6 +292,7 @@ export class VerificationBase extends EventEmitter {
         this._endTimer(); // always kill the activity timer
         if (!this._done) {
             this.cancelled = true;
+            this.request.onVerifierCancelled();
             if (this.userId && this.deviceId) {
                 // send a cancellation to the other user (if it wasn't
                 // cancelled by the other user)
@@ -338,7 +375,7 @@ export class VerificationBase extends EventEmitter {
 
         for (const [keyId, keyInfo] of Object.entries(keys)) {
             const deviceId = keyId.split(':', 2)[1];
-            const device = await this._baseApis.getStoredDevice(userId, deviceId);
+            const device = this._baseApis.getStoredDevice(userId, deviceId);
             if (device) {
                 await verifier(keyId, device, keyInfo);
                 verifiedDevices.push(deviceId);

src/crypto/verification/QRCode.js

@@ -23,8 +23,9 @@ limitations under the License.
 import {VerificationBase as Base} from "./Base";
 import {
     newKeyMismatchError,
-    newUserMismatchError,
+    newUserCancelledError,
 } from './Error';
+import {encodeUnpaddedBase64, decodeBase64} from "../olmlib";
 
 export const SHOW_QR_CODE_METHOD = "m.qr_code.show.v1";
 export const SCAN_QR_CODE_METHOD = "m.qr_code.scan.v1";
@@ -49,47 +50,46 @@ export class ReciprocateQRCode extends Base {
                 "with this method yet.");
         }
 
-        const targetUserId = this.startEvent.getSender();
-        if (!this.userId) {
-            console.log("Asking to confirm user ID");
-            this.userId = await new Promise((resolve, reject) => {
-                this.emit("confirm_user_id", {
-                    userId: targetUserId,
-                    confirm: resolve, // takes a userId
-                    cancel: () => reject(newUserMismatchError()),
-                });
-            });
-        } else if (targetUserId !== this.userId) {
-            throw newUserMismatchError({
-                expected: this.userId,
-                actual: targetUserId,
-            });
-        }
-
-        if (this.startEvent.getContent()['secret'] !== this.request.encodedSharedSecret) {
+        const {qrCodeData} = this.request;
+        // 1. check the secret
+        if (this.startEvent.getContent()['secret'] !== qrCodeData.encodedSharedSecret) {
             throw newKeyMismatchError();
         }
 
-        // If we've gotten this far, verify the user's master cross signing key
-        const xsignInfo = this._baseApis.getStoredCrossSigningForUser(this.userId);
-        if (!xsignInfo) throw new Error("Missing cross signing info");
-
-        const masterKey = xsignInfo.getId("master");
-        const masterKeyId = `ed25519:${masterKey}`;
-        const keys = {[masterKeyId]: masterKey};
-
-        const devices = (await this._baseApis.getStoredDevicesForUser(this.userId)) || [];
-        const targetDevice = devices.find(d => {
-            return d.deviceId === this.request.targetDevice.deviceId;
+        // 2. ask if other user shows shield as well
+        await new Promise((resolve, reject) => {
+            this.reciprocateQREvent = {
+                confirm: resolve,
+                cancel: () => reject(newUserCancelledError()),
+            };
+            this.emit("show_reciprocate_qr", this.reciprocateQREvent);
         });
-        if (!targetDevice) throw new Error("Device not found, somehow");
-        keys[`ed25519:${targetDevice.deviceId}`] = targetDevice.getFingerprint();
 
-        if (this.request.requestingUserId === this.request.receivingUserId) {
-            delete keys[masterKeyId];
+        // 3. determine key to sign / mark as trusted
+        const keys = {};
+
+        switch (qrCodeData.mode) {
+            case MODE_VERIFY_OTHER_USER: {
+                // add master key to keys to be signed, only if we're not doing self-verification
+                const masterKey = qrCodeData.otherUserMasterKey;
+                keys[`ed25519:${masterKey}`] = masterKey;
+                break;
+            }
+            case MODE_VERIFY_SELF_TRUSTED: {
+                const deviceId = this.request.targetDevice.deviceId;
+                keys[`ed25519:${deviceId}`] = qrCodeData.otherDeviceKey;
+                break;
+            }
+            case MODE_VERIFY_SELF_UNTRUSTED: {
+                const masterKey = qrCodeData.myMasterKey;
+                keys[`ed25519:${masterKey}`] = masterKey;
+                break;
+            }
         }
 
+        // 4. sign the key (or mark own MSK as verified in case of MODE_VERIFY_SELF_TRUSTED)
         await this._verifyKeys(this.userId, keys, (keyId, device, keyInfo) => {
+            // make sure the device has the expected keys
             const targetKey = keys[keyId];
             if (!targetKey) throw newKeyMismatchError();
 
@@ -106,8 +106,200 @@ export class ReciprocateQRCode extends Base {
                     throw newKeyMismatchError();
                 }
             }
-
-            // Otherwise it is probably fine
         });
     }
 }
+
+const CODE_VERSION = 0x02; // the version of binary QR codes we support
+const BINARY_PREFIX = "MATRIX"; // ASCII, used to prefix the binary format
+const MODE_VERIFY_OTHER_USER = 0x00; // Verifying someone who isn't us
+const MODE_VERIFY_SELF_TRUSTED = 0x01; // We trust the master key
+const MODE_VERIFY_SELF_UNTRUSTED = 0x02; // We do not trust the master key
+
+export class QRCodeData {
+    constructor(
+        mode, sharedSecret, otherUserMasterKey,
+        otherDeviceKey, myMasterKey, buffer,
+    ) {
+        this._sharedSecret = sharedSecret;
+        this._mode = mode;
+        this._otherUserMasterKey = otherUserMasterKey;
+        this._otherDeviceKey = otherDeviceKey;
+        this._myMasterKey = myMasterKey;
+        this._buffer = buffer;
+    }
+
+    static async create(request, client) {
+        const sharedSecret = QRCodeData._generateSharedSecret();
+        const mode = QRCodeData._determineMode(request, client);
+        let otherUserMasterKey = null;
+        let otherDeviceKey = null;
+        let myMasterKey = null;
+        if (mode === MODE_VERIFY_OTHER_USER) {
+            const otherUserCrossSigningInfo =
+                client.getStoredCrossSigningForUser(request.otherUserId);
+            otherUserMasterKey = otherUserCrossSigningInfo.getId("master");
+        } else if (mode === MODE_VERIFY_SELF_TRUSTED) {
+            otherDeviceKey = await QRCodeData._getOtherDeviceKey(request, client);
+        } else if (mode === MODE_VERIFY_SELF_UNTRUSTED) {
+            const myUserId = client.getUserId();
+            const myCrossSigningInfo = client.getStoredCrossSigningForUser(myUserId);
+            myMasterKey = myCrossSigningInfo.getId("master");
+        }
+        const qrData = QRCodeData._generateQrData(
+            request, client, mode,
+            sharedSecret,
+            otherUserMasterKey,
+            otherDeviceKey,
+            myMasterKey,
+        );
+        const buffer = QRCodeData._generateBuffer(qrData);
+        return new QRCodeData(mode, sharedSecret,
+            otherUserMasterKey, otherDeviceKey, myMasterKey, buffer);
+    }
+
+    get buffer() {
+        return this._buffer;
+    }
+
+    get mode() {
+        return this._mode;
+    }
+
+    /**
+     * only set when mode is MODE_VERIFY_SELF_TRUSTED
+     * @return {string} device key of other party at time of generating QR code
+     */
+    get otherDeviceKey() {
+        return this._otherDeviceKey;
+    }
+
+    /**
+     * only set when mode is MODE_VERIFY_OTHER_USER
+     * @return {string} master key of other party at time of generating QR code
+     */
+    get otherUserMasterKey() {
+        return this._otherUserMasterKey;
+    }
+
+    /**
+     * only set when mode is MODE_VERIFY_SELF_UNTRUSTED
+     * @return {string} own master key at time of generating QR code
+     */
+    get myMasterKey() {
+        return this._myMasterKey;
+    }
+
+    /**
+     * The unpadded base64 encoded shared secret.
+     */
+    get encodedSharedSecret() {
+        return this._sharedSecret;
+    }
+
+    static _generateSharedSecret() {
+        const secretBytes = new Uint8Array(11);
+        global.crypto.getRandomValues(secretBytes);
+        return encodeUnpaddedBase64(secretBytes);
+    }
+
+    static async _getOtherDeviceKey(request, client) {
+        const myUserId = client.getUserId();
+        const otherDevice = request.targetDevice;
+        const otherDeviceId = otherDevice ? otherDevice.deviceId : null;
+        const device = client.getStoredDevice(myUserId, otherDeviceId);
+        if (!device) {
+            throw new Error("could not find device " + otherDeviceId);
+        }
+        const key = device.getFingerprint();
+        return key;
+    }
+
+    static _determineMode(request, client) {
+        const myUserId = client.getUserId();
+        const otherUserId = request.otherUserId;
+
+        let mode = MODE_VERIFY_OTHER_USER;
+        if (myUserId === otherUserId) {
+            // Mode changes depending on whether or not we trust the master cross signing key
+            const myTrust = client.checkUserTrust(myUserId);
+            if (myTrust.isCrossSigningVerified()) {
+                mode = MODE_VERIFY_SELF_TRUSTED;
+            } else {
+                mode = MODE_VERIFY_SELF_UNTRUSTED;
+            }
+        }
+        return mode;
+    }
+
+    static _generateQrData(request, client, mode,
+        encodedSharedSecret, otherUserMasterKey,
+        otherDeviceKey, myMasterKey,
+    ) {
+        const myUserId = client.getUserId();
+        const transactionId = request.channel.transactionId;
+        const qrData = {
+            prefix: BINARY_PREFIX,
+            version: CODE_VERSION,
+            mode,
+            transactionId,
+            firstKeyB64: '', // worked out shortly
+            secondKeyB64: '', // worked out shortly
+            secretB64: encodedSharedSecret,
+        };
+
+        const myCrossSigningInfo = client.getStoredCrossSigningForUser(myUserId);
+
+        if (mode === MODE_VERIFY_OTHER_USER) {
+            // First key is our master cross signing key
+            qrData.firstKeyB64 = myCrossSigningInfo.getId("master");
+            // Second key is the other user's master cross signing key
+            qrData.secondKeyB64 = otherUserMasterKey;
+        } else if (mode === MODE_VERIFY_SELF_TRUSTED) {
+            // First key is our master cross signing key
+            qrData.firstKeyB64 = myCrossSigningInfo.getId("master");
+            qrData.secondKeyB64 = otherDeviceKey;
+        } else if (mode === MODE_VERIFY_SELF_UNTRUSTED) {
+            // First key is our device's key
+            qrData.firstKeyB64 = client.getDeviceEd25519Key();
+            // Second key is what we think our master cross signing key is
+            qrData.secondKeyB64 = myMasterKey;
+        }
+        return qrData;
+    }
+
+    static _generateBuffer(qrData) {
+        let buf = Buffer.alloc(0); // we'll concat our way through life
+
+        const appendByte = (b) => {
+            const tmpBuf = Buffer.from([b]);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+        const appendInt = (i) => {
+            const tmpBuf = Buffer.alloc(2);
+            tmpBuf.writeInt16BE(i, 0);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+        const appendStr = (s, enc, withLengthPrefix = true) => {
+            const tmpBuf = Buffer.from(s, enc);
+            if (withLengthPrefix) appendInt(tmpBuf.byteLength);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+        const appendEncBase64 = (b64) => {
+            const b = decodeBase64(b64);
+            const tmpBuf = Buffer.from(b);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+
+        // Actually build the buffer for the QR code
+        appendStr(qrData.prefix, "ascii", false);
+        appendByte(qrData.version);
+        appendByte(qrData.mode);
+        appendStr(qrData.transactionId, "utf-8");
+        appendEncBase64(qrData.firstKeyB64);
+        appendEncBase64(qrData.secondKeyB64);
+        appendEncBase64(qrData.secretB64);
+
+        return buf;
+    }
+}

src/crypto/verification/SAS.js

@@ -295,7 +295,7 @@ export class SAS extends Base {
         const hashCommitment = content.commitment;
         const olmSAS = new global.Olm.SAS();
         try {
-            this._send("m.key.verification.key", {
+            await this._send("m.key.verification.key", {
                 key: olmSAS.get_pubkey(),
             });
 
@@ -318,9 +318,13 @@ export class SAS extends Base {
             const verifySAS = new Promise((resolve, reject) => {
                 this.sasEvent = {
                     sas: generateSas(sasBytes, sasMethods),
-                    confirm: () => {
-                        this._sendMAC(olmSAS, macMethod);
-                        resolve();
+                    confirm: async () => {
+                        try {
+                            await this._sendMAC(olmSAS, macMethod);
+                            resolve();
+                        } catch (err) {
+                            reject(err);
+                        }
                     },
                     cancel: () => reject(newUserCancelledError()),
                     mismatch: () => reject(newMismatchedSASError()),
@@ -377,7 +381,7 @@ export class SAS extends Base {
         const olmSAS = new global.Olm.SAS();
         try {
             const commitmentStr = olmSAS.get_pubkey() + anotherjson.stringify(content);
-            this._send("m.key.verification.accept", {
+            await this._send("m.key.verification.accept", {
                 key_agreement_protocol: keyAgreement,
                 hash: hashMethod,
                 message_authentication_code: macMethod,
@@ -391,7 +395,7 @@ export class SAS extends Base {
             // FIXME: make sure event is properly formed
             content = e.getContent();
             olmSAS.set_their_key(content.key);
-            this._send("m.key.verification.key", {
+            await this._send("m.key.verification.key", {
                 key: olmSAS.get_pubkey(),
             });
 
@@ -403,9 +407,13 @@ export class SAS extends Base {
             const verifySAS = new Promise((resolve, reject) => {
                 this.sasEvent = {
                     sas: generateSas(sasBytes, sasMethods),
-                    confirm: () => {
-                        this._sendMAC(olmSAS, macMethod);
-                        resolve();
+                    confirm: async () => {
+                        try {
+                            await this._sendMAC(olmSAS, macMethod);
+                            resolve();
+                        } catch(err) {
+                            reject(err);
+                        }
                     },
                     cancel: () => reject(newUserCancelledError()),
                     mismatch: () => reject(newMismatchedSASError()),
@@ -461,7 +469,7 @@ export class SAS extends Base {
             keyList.sort().join(","),
             baseInfo + "KEY_IDS",
         );
-        this._send("m.key.verification.mac", { mac, keys });
+        return this._send("m.key.verification.mac", { mac, keys });
     }
 
     async _checkMAC(olmSAS, content, method) {

src/crypto/verification/request/InRoomChannel.js

@@ -44,11 +44,6 @@ export class InRoomChannel {
         this._requestEventId = null;
     }
 
-    /** Whether this channel needs m.key.verification.done messages to be sent after a successful verification */
-    get needsDoneMessage() {
-        return true;
-    }
-
     get receiveStartFromOtherDevices() {
         return true;
     }
@@ -183,6 +178,11 @@ export class InRoomChannel {
      * @returns {Promise} a promise that resolves when any requests as an anwser to the passed-in event are sent.
      */
     async handleEvent(event, request, isLiveEvent) {
+        // prevent processing the same event multiple times, as under
+        // some circumstances Room.timeline can get emitted twice for the same event
+        if (request.hasEventId(event.getId())) {
+            return;
+        }
         const type = InRoomChannel.getEventType(event);
         // do validations that need state (roomId, userId),
         // ignore if invalid

src/crypto/verification/request/ToDeviceChannel.js

@@ -61,10 +61,6 @@ export class ToDeviceChannel {
         return this._deviceId;
     }
 
-    get needsDoneMessage() {
-        return false;
-    }
-
     static getEventType(event) {
         return event.getType();
     }

src/crypto/verification/request/VerificationRequest.js

@@ -23,7 +23,7 @@ import {
     newUnexpectedMessageError,
     newUnknownMethodError,
 } from "../Error";
-import * as olmlib from "../../olmlib";
+import {QRCodeData, SCAN_QR_CODE_METHOD} from "../QRCode";
 
 // the recommended amount of time before a verification request
 // should be (automatically) cancelled without user interaction
@@ -70,10 +70,16 @@ export class VerificationRequest extends EventEmitter {
         this._eventsByThem = new Map();
         this._observeOnly = false;
         this._timeoutTimer = null;
-        this._sharedSecret = null; // used for QR codes
         this._accepting = false;
         this._declining = false;
         this._verifierHasFinished = false;
+        this._cancelled = false;
+        this._chosenMethod = null;
+        // we keep a copy of the QR Code data (including other user master key) around
+        // for QR reciprocate verification, to protect against
+        // cross-signing identity reset between the .ready and .start event
+        // and signing the wrong key after .start
+        this._qrCodeData = null;
     }
 
     /**
@@ -154,6 +160,11 @@ export class VerificationRequest extends EventEmitter {
         return this._commonMethods;
     }
 
+    /** the method picked in the .start event */
+    get chosenMethod() {
+        return this._chosenMethod;
+    }
+
     /** The current remaining amount of ms before the request should be automatically cancelled */
     get timeout() {
         const requestEvent = this._getEventByEither(REQUEST_TYPE);
@@ -201,19 +212,34 @@ export class VerificationRequest extends EventEmitter {
             this._phase !== PHASE_CANCELLED;
     }
 
+    /** Only set after a .ready if the other party can scan a QR code */
+    get qrCodeData() {
+        return this._qrCodeData;
+    }
+
     /** Checks whether the other party supports a given verification method.
      *  This is useful when setting up the QR code UI, as it is somewhat asymmetrical:
      *  if the other party supports SCAN_QR, we should show a QR code in the UI, and vice versa.
      *  For methods that need to be supported by both ends, use the `methods` property.
      *  @param {string} method the method to check
+     *  @param {boolean} force to check even if the phase is not ready or started yet, internal usage
      *  @return {bool} whether or not the other party said the supported the method */
-    otherPartySupportsMethod(method) {
-        if (!this.ready && !this.started) {
+    otherPartySupportsMethod(method, force = false) {
+        if (!force && !this.ready && !this.started) {
             return false;
         }
         const theirMethodEvent = this._eventsByThem.get(REQUEST_TYPE) ||
             this._eventsByThem.get(READY_TYPE);
         if (!theirMethodEvent) {
+            // if we started straight away with .start event,
+            // we are assuming that the other side will support the
+            // chosen method, so return true for that.
+            if (this.started && this.initiatedByMe) {
+                const myStartEvent = this._eventsByUs.get(START_TYPE);
+                const content = myStartEvent && myStartEvent.getContent();
+                const myStartMethod = content && content.method;
+                return method == myStartMethod;
+            }
             return false;
         }
         const content = theirMethodEvent.getContent();
@@ -277,6 +303,10 @@ export class VerificationRequest extends EventEmitter {
         return this.channel.userId;
     }
 
+    get isSelfVerification() {
+        return this._client.getUserId() === this.otherUserId;
+    }
+
     /**
      * The id of the user that cancelled the request,
      * only defined when phase is PHASE_CANCELLED
@@ -306,14 +336,6 @@ export class VerificationRequest extends EventEmitter {
         return this._observeOnly;
     }
 
-    /**
-     * The unpadded base64 encoded shared secret. Primarily used for QR code
-     * verification.
-     */
-    get encodedSharedSecret() {
-        if (!this._sharedSecret) this._generateSharedSecret();
-        return this._sharedSecret;
-    }
 
     /**
      * Gets which device the verification should be started with
@@ -360,6 +382,7 @@ export class VerificationRequest extends EventEmitter {
                 if (!this._verifier) {
                     throw newUnknownMethodError();
                 }
+                this._chosenMethod = method;
             }
         }
         return this._verifier;
@@ -373,7 +396,6 @@ export class VerificationRequest extends EventEmitter {
         if (!this.observeOnly && this._phase === PHASE_UNSENT) {
             const methods = [...this._verificationMethods.keys()];
             await this.channel.send(REQUEST_TYPE, {methods});
-            this._generateSharedSecret();
         }
     }
 
@@ -406,16 +428,9 @@ export class VerificationRequest extends EventEmitter {
             this._accepting = true;
             this.emit("change");
             await this.channel.send(READY_TYPE, {methods});
-            this._generateSharedSecret();
         }
     }
 
-    _generateSharedSecret() {
-        const secretBytes = new Uint8Array(8);
-        global.crypto.getRandomValues(secretBytes);
-        this._sharedSecret = olmlib.encodeUnpaddedBase64(secretBytes);
-    }
-
     /**
      * Can be used to listen for state changes until the callback returns true.
      * @param {Function} fn callback to evaluate whether the request is in the desired state.
@@ -511,7 +526,7 @@ export class VerificationRequest extends EventEmitter {
         }
 
         const cancelEvent = this._getEventByEither(CANCEL_TYPE);
-        if (cancelEvent && phase() !== PHASE_DONE) {
+        if ((this._cancelled || cancelEvent) && phase() !== PHASE_DONE) {
             transitions.push({phase: PHASE_CANCELLED, event: cancelEvent});
             return transitions;
         }
@@ -550,6 +565,14 @@ export class VerificationRequest extends EventEmitter {
             const {method} = event.getContent();
             if (!this._verifier && !this.observeOnly) {
                 this._verifier = this._createVerifier(method, event);
+                if (!this._verifier) {
+                    this.cancel({
+                        code: "m.unknown_method",
+                        reason: `Unknown method: ${method}`,
+                    });
+                } else {
+                    this._chosenMethod = method;
+                }
             }
         }
     }
@@ -566,6 +589,56 @@ export class VerificationRequest extends EventEmitter {
         return newTransitions;
     }
 
+    _isWinningStartRace(newEvent) {
+        if (newEvent.getType() !== START_TYPE) {
+            return false;
+        }
+        const oldEvent = this._verifier.startEvent;
+
+        let oldRaceIdentifier;
+        if (this.isSelfVerification) {
+            // if the verifier does not have a startEvent,
+            // it is because it's still sending and we are on the initator side
+            // we know we are sending a .start event because we already
+            // have a verifier (checked in calling method)
+            if (oldEvent) {
+                const oldContent = oldEvent.getContent();
+                oldRaceIdentifier = oldContent && oldContent.from_device;
+            } else {
+                oldRaceIdentifier = this._client.getDeviceId();
+            }
+        } else {
+            if (oldEvent) {
+                oldRaceIdentifier = oldEvent.getSender();
+            } else {
+                oldRaceIdentifier = this._client.getUserId();
+            }
+        }
+
+        let newRaceIdentifier;
+        if (this.isSelfVerification) {
+            const newContent = newEvent.getContent();
+            newRaceIdentifier = newContent && newContent.from_device;
+        } else {
+            newRaceIdentifier = newEvent.getSender();
+        }
+        return newRaceIdentifier < oldRaceIdentifier;
+    }
+
+    hasEventId(eventId) {
+        for (const event of this._eventsByUs.values()) {
+            if (event.getId() === eventId) {
+                return true;
+            }
+        }
+        for (const event of this._eventsByThem.values()) {
+            if (event.getId() === eventId) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
      * Changes the state of the request and verifier in response to a key verification event.
      * @param {string} type the "symbolic" event type, as returned by the `getEventType` function on the channel.
@@ -591,6 +664,18 @@ export class VerificationRequest extends EventEmitter {
             }
         }
 
+        // This assumes verification won't need to send an event with
+        // the same type for the same party twice.
+        // This is true for QR and SAS verification, and was
+        // added here to prevent verification getting cancelled
+        // when the server duplicates an event (https://github.com/matrix-org/synapse/issues/3365)
+        const isDuplicateEvent = isSentByUs ?
+            this._eventsByUs.has(type) :
+            this._eventsByThem.has(type);
+        if (isDuplicateEvent) {
+            return;
+        }
+
         const oldPhase = this.phase;
         this._addEvent(type, event, isSentByUs);
 
@@ -600,16 +685,11 @@ export class VerificationRequest extends EventEmitter {
             // only pass events from the other side to the verifier,
             // no remote echos of our own events
             if (this._verifier && !this.observeOnly) {
-                const oldEvent = this._verifier.startEvent;
-                // if the verifier does not have a startEvent, it is because it's still sending and we are on the initator side
-                const oldSender = oldEvent ?
-                    oldEvent.getSender() :
-                    this._client.getUserId();
-                const newEventWinsRace = event.getSender() < oldSender;
+                const newEventWinsRace = this._isWinningStartRace(event);
                 if (this._verifier.canSwitchStartEvent(event) && newEventWinsRace) {
                     this._verifier.switchStartEvent(event);
                 } else if (!isRemoteEcho) {
-                     if (type === CANCEL_TYPE || (this._verifier.events
+                    if (type === CANCEL_TYPE || (this._verifier.events
                         && this._verifier.events.includes(type))) {
                         this._verifier.handleEvent(event);
                     }
@@ -617,6 +697,20 @@ export class VerificationRequest extends EventEmitter {
             }
 
             if (newTransitions.length) {
+                // create QRCodeData if the other side can scan
+                // important this happens before emitting a phase change,
+                // so listeners can rely on it being there already
+                // We only do this for live events because it is important that
+                // we sign the keys that were in the QR code, and not the keys
+                // we happen to have at some later point in time.
+                if (isLiveEvent && newTransitions.some(t => t.phase === PHASE_READY)) {
+                    const shouldGenerateQrCode =
+                        this.otherPartySupportsMethod(SCAN_QR_CODE_METHOD, true);
+                    if (shouldGenerateQrCode) {
+                        this._qrCodeData = await QRCodeData.create(this, this._client);
+                    }
+                }
+
                 const lastTransition = newTransitions[newTransitions.length - 1];
                 const {phase} = lastTransition;
 
@@ -765,15 +859,26 @@ export class VerificationRequest extends EventEmitter {
         return true;
     }
 
-    onVerifierFinished() {
-        if (this.channel.needsDoneMessage) {
-            // verification in DM requires a done message
-            this.channel.send("m.key.verification.done", {});
-        }
-        this._verifierHasFinished = true;
+    onVerifierCancelled() {
+        this._cancelled = true;
+        // move to cancelled phase
         const newTransitions = this._applyPhaseTransitions();
         if (newTransitions.length) {
             this._setPhase(newTransitions[newTransitions.length - 1].phase);
         }
     }
+
+    onVerifierFinished() {
+        this.channel.send("m.key.verification.done", {});
+        this._verifierHasFinished = true;
+        // move to .done phase
+        const newTransitions = this._applyPhaseTransitions();
+        if (newTransitions.length) {
+            this._setPhase(newTransitions[newTransitions.length - 1].phase);
+        }
+    }
+
+    getEventFromOtherParty(type) {
+        return this._eventsByThem.get(type);
+    }
 }

src/errors.js

@@ -44,3 +44,10 @@ InvalidCryptoStoreError.prototype = Object.create(Error.prototype, {
   },
 });
 Reflect.setPrototypeOf(InvalidCryptoStoreError, Error);
+
+export class KeySignatureUploadError extends Error {
+  constructor(message, value) {
+    super(message);
+    this.value = value;
+  }
+}

src/filter.js

@@ -56,13 +56,6 @@ Filter.LAZY_LOADING_MESSAGES_FILTER = {
     lazy_load_members: true,
 };
 
-Filter.LAZY_LOADING_SYNC_FILTER = {
-    room: {
-        state: Filter.LAZY_LOADING_MESSAGES_FILTER,
-    },
-};
-
-
 /**
  * Get the ID of this filter on your homeserver (if known)
  * @return {?Number} The filter ID
@@ -96,6 +89,7 @@ Filter.prototype.setDefinition = function(definition) {
     //     "state": {
     //       "types": ["m.room.*"],
     //       "not_rooms": ["!726s6s6q:example.com"],
+    //       "lazy_load_members": true,
     //     },
     //     "timeline": {
     //       "limit": 10,
@@ -177,6 +171,10 @@ Filter.prototype.setTimelineLimit = function(limit) {
     setProp(this.definition, "room.timeline.limit", limit);
 };
 
+Filter.prototype.setLazyLoadMembers = function(enabled) {
+    setProp(this.definition, "room.state.lazy_load_members", !!enabled);
+};
+
 /**
  * Control whether left rooms should be included in responses.
  * @param {boolean} includeLeave True to make rooms the user has left appear

src/http-api.js

@@ -276,6 +276,9 @@ MatrixHttpApi.prototype = {
                         callbacks.clearTimeout(xhr.timeout_timer);
                         var resp;
                         try {
+                            if (xhr.status === 0) {
+                                throw new AbortError();
+                            }
                             if (!xhr.responseText) {
                                 throw new Error('No response body.');
                             }
@@ -789,6 +792,17 @@ const requestCallback = function(
     userDefinedCallback = userDefinedCallback || function() {};
 
     return function(err, response, body) {
+        if (err) {
+            // the unit tests use matrix-mock-request, which throw the string "aborted" when aborting a request.
+            // See https://github.com/matrix-org/matrix-mock-request/blob/3276d0263a561b5b8326b47bae720578a2c7473a/src/index.js#L48
+            const aborted = err.name === "AbortError" || err === "aborted";
+            if (!aborted && !(err instanceof MatrixError)) {
+                // browser-request just throws normal Error objects,
+                // not `TypeError`s like fetch does. So just assume any
+                // error is due to the connection.
+                err = new ConnectionError("request failed", err);
+            }
+        }
         if (!err) {
             try {
                 if (response.statusCode >= 400) {
@@ -892,12 +906,76 @@ function getResponseContentType(response) {
  * @prop {Object} data The raw Matrix error JSON used to construct this object.
  * @prop {integer} httpStatus The numeric HTTP status code given
  */
-export function MatrixError(errorJson) {
-    errorJson = errorJson || {};
-    this.errcode = errorJson.errcode;
-    this.name = errorJson.errcode || "Unknown error code";
-    this.message = errorJson.error || "Unknown message";
-    this.data = errorJson;
+export class MatrixError extends Error {
+    constructor(errorJson) {
+        errorJson = errorJson || {};
+        super(`MatrixError: ${errorJson.errcode}`);
+        this.errcode = errorJson.errcode;
+        this.name = errorJson.errcode || "Unknown error code";
+        this.message = errorJson.error || "Unknown message";
+        this.data = errorJson;
+    }
+}
+
+/**
+ * Construct a ConnectionError. This is a JavaScript Error indicating
+ * that a request failed because of some error with the connection, either
+ * CORS was not correctly configured on the server, the server didn't response,
+ * the request timed out, or the internet connection on the client side went down.
+ * @constructor
+ */
+export class ConnectionError extends Error {
+    constructor(message, cause = undefined) {
+        super(message + (cause ? `: ${cause.message}` : ""));
+        this._cause = cause;
+    }
+
+    get name() {
+        return "ConnectionError";
+    }
+
+    get cause() {
+        return this._cause;
+    }
+}
+
+export class AbortError extends Error {
+    constructor() {
+        super("Operation aborted");
+    }
+
+    get name() {
+        return "AbortError";
+    }
+}
+
+/**
+ * Retries a network operation run in a callback.
+ * @param  {number}   maxAttempts maximum attempts to try
+ * @param  {Function} callback    callback that returns a promise of the network operation. If rejected with ConnectionError, it will be retried by calling the callback again.
+ * @return {any} the result of the network operation
+ * @throws {ConnectionError} If after maxAttempts the callback still throws ConnectionError
+ */
+export async function retryNetworkOperation(maxAttempts, callback) {
+    let attempts = 0;
+    let lastConnectionError = null;
+    while (attempts < maxAttempts) {
+        try {
+            if (attempts > 0) {
+                const timeout = 1000 * Math.pow(2, attempts);
+                console.log(`network operation failed ${attempts} times,` +
+                    ` retrying in ${timeout}ms...`);
+                await new Promise(r => setTimeout(r, timeout));
+            }
+            return await callback();
+        } catch (err) {
+            if (err instanceof ConnectionError) {
+                attempts += 1;
+                lastConnectionError = err;
+            } else {
+                throw err;
+            }
+        }
+    }
+    throw lastConnectionError;
 }
-MatrixError.prototype = Object.create(Error.prototype);
-MatrixError.prototype.constructor = MatrixError;

src/index.ts

@@ -21,5 +21,12 @@ import request from "request";
 matrixcs.request(request);
 utils.runPolyfills();
 
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    console.log('nodejs was compiled without crypto support');
+}
+
 export * from "./matrix";
 export default matrixcs;

src/interactive-auth.js

@@ -163,6 +163,8 @@ InteractiveAuth.prototype = {
      */
     poll: async function() {
         if (!this._data.session) return;
+        // likewise don't poll if there is no auth session in progress
+        if (!this._resolveFunc) return;
         // if we currently have a request in flight, there's no point making
         // another just to check what the status is
         if (this._submitPromise) return;
@@ -318,6 +320,8 @@ InteractiveAuth.prototype = {
         try {
             const result = await this._requestCallback(auth, background);
             this._resolveFunc(result);
+            this._resolveFunc = null;
+            this._rejectFunc = null;
         } catch (error) {
             // sometimes UI auth errors don't come with flows
             const errorFlows = error.data ? error.data.flows : null;
@@ -347,7 +351,13 @@ InteractiveAuth.prototype = {
                 error.data.session = this._data.session;
             }
             this._data = error.data;
-            this._startNextAuthStage();
+            try {
+                this._startNextAuthStage();
+            } catch (e) {
+                this._rejectFunc(e);
+                this._resolveFunc = null;
+                this._rejectFunc = null;
+            }
 
             if (
                 !this._emailSid &&
@@ -379,8 +389,10 @@ InteractiveAuth.prototype = {
                     // (or not being registered, depending on what we're trying
                     // to do) or it could be a network failure. Either way, pass
                     // the failure up as the user can't complete auth if we can't
-                    // send the email, foe whatever reason.
+                    // send the email, for whatever reason.
                     this._rejectFunc(e);
+                    this._resolveFunc = null;
+                    this._rejectFunc = null;
                 } finally {
                     this._requestingEmailToken = false;
                 }

src/models/room.js

@@ -675,7 +675,7 @@ Room.prototype.hasUnverifiedDevices = async function() {
     }
     const e2eMembers = await this.getEncryptionTargetMembers();
     for (const member of e2eMembers) {
-        const devices = await this._client.getStoredDevicesForUser(member.userId);
+        const devices = this._client.getStoredDevicesForUser(member.userId);
         if (devices.some((device) => device.isUnverified())) {
             return true;
         }
@@ -1278,6 +1278,11 @@ Room.prototype._handleRemoteEcho = function(remoteEvent, localEvent) {
     const newEventId = remoteEvent.getId();
     const oldStatus = localEvent.status;
 
+    logger.debug(
+        `Got remote echo for event ${oldEventId} -> ${newEventId} ` +
+        `old status ${oldStatus}`,
+    );
+
     // no longer pending
     delete this._txnToEvent[remoteEvent.getUnsigned().transaction_id];
 
@@ -1347,7 +1352,10 @@ ALLOWED_TRANSITIONS[EventStatus.CANCELLED] =
  * @fires module:client~MatrixClient#event:"Room.localEchoUpdated"
  */
 Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
-    logger.log(`setting pendingEvent status to ${newStatus} in ${event.getRoomId()}`);
+    logger.log(
+        `setting pendingEvent status to ${newStatus} in ${event.getRoomId()} ` +
+        `event ID ${event.getId()} -> ${newEventId}`,
+    );
 
     // if the message was sent, we expect an event id
     if (newStatus == EventStatus.SENT && !newEventId) {

src/store/memory.js

@@ -25,6 +25,15 @@ limitations under the License.
 import {User} from "../models/user";
 import * as utils from "../utils";
 
+function isValidFilterId(filterId) {
+    const isValidStr = typeof filterId === "string" &&
+        !!filterId &&
+        filterId !== "undefined" && // exclude these as we've serialized undefined in localStorage before
+        filterId !== "null";
+
+    return isValidStr || typeof filterId === "number";
+}
+
 /**
  * Construct a new in-memory data store for the Matrix Client.
  * @constructor
@@ -273,8 +282,17 @@ MemoryStore.prototype = {
         if (!this.localStorage) {
             return null;
         }
+        const key = "mxjssdk_memory_filter_" + filterName;
+        // XXX Storage.getItem doesn't throw ...
+        // or are we using something different
+        // than window.localStorage in some cases
+        // that does throw?
+        // that would be very naughty
         try {
-            return this.localStorage.getItem("mxjssdk_memory_filter_" + filterName);
+            const value = this.localStorage.getItem(key);
+            if (isValidFilterId(value)) {
+                return value;
+            }
         } catch (e) {}
         return null;
     },
@@ -288,8 +306,13 @@ MemoryStore.prototype = {
         if (!this.localStorage) {
             return;
         }
+        const key = "mxjssdk_memory_filter_" + filterName;
         try {
-            this.localStorage.setItem("mxjssdk_memory_filter_" + filterName, filterId);
+            if (isValidFilterId(filterId)) {
+                this.localStorage.setItem(key, filterId);
+            } else {
+                this.localStorage.removeItem(key);
+            }
         } catch (e) {}
     },
 

src/sync.js

@@ -511,6 +511,12 @@ SyncApi.prototype.sync = function() {
         checkLazyLoadStatus(); // advance to the next stage
     }
 
+    function buildDefaultFilter() {
+        const filter = new Filter(client.credentials.userId);
+        filter.setTimelineLimit(self.opts.initialSyncLimit);
+        return filter;
+    }
+
     const checkLazyLoadStatus = async () => {
         debuglog("Checking lazy load status...");
         if (this.opts.lazyLoadMembers && client.isGuest()) {
@@ -520,19 +526,11 @@ SyncApi.prototype.sync = function() {
             debuglog("Checking server lazy load support...");
             const supported = await client.doesServerSupportLazyLoading();
             if (supported) {
-                try {
-                    debuglog("Creating and storing lazy load sync filter...");
-                    this.opts.filter = await client.createFilter(
-                        Filter.LAZY_LOADING_SYNC_FILTER,
-                    );
-                    debuglog("Created and stored lazy load sync filter");
-                } catch (err) {
-                    logger.error(
-                        "Creating and storing lazy load sync filter failed",
-                        err,
-                    );
-                    throw err;
+                debuglog("Enabling lazy load on sync filter...");
+                if (!this.opts.filter) {
+                    this.opts.filter = buildDefaultFilter();
                 }
+                this.opts.filter.setLazyLoadMembers(true);
             } else {
                 debuglog("LL: lazy loading requested but not supported " +
                     "by server, so disabling");
@@ -575,8 +573,7 @@ SyncApi.prototype.sync = function() {
         if (self.opts.filter) {
             filter = self.opts.filter;
         } else {
-            filter = new Filter(client.credentials.userId);
-            filter.setTimelineLimit(self.opts.initialSyncLimit);
+            filter = buildDefaultFilter();
         }
 
         let filterId;

src/utils.ts

@@ -21,6 +21,7 @@ limitations under the License.
  */
 
 import unhomoglyph from 'unhomoglyph';
+import {ConnectionError} from "./http-api";
 
 /**
  * Encode a dictionary of query parameters.
@@ -734,3 +735,17 @@ export async function promiseMapSeries<T>(
 export function promiseTry<T>(fn: () => T): Promise<T> {
     return new Promise((resolve) => resolve(fn()));
 }
+
+// We need to be able to access the Node.js crypto library from within the
+// Matrix SDK without needing to `require("crypto")`, which will fail in
+// browsers.  So `index.ts` will call `setCrypto` to store it, and when we need
+// it, we can call `getCrypto`.
+let crypto: Object;
+
+export function setCrypto(c: Object) {
+    crypto = c;
+}
+
+export function getCrypto(): Object {
+    return crypto;
+}

src/webrtc/call.js

@@ -165,29 +165,26 @@ MatrixCall.prototype.placeVideoCall = function(remoteVideoElement, localVideoEle
  * @throws If you have not specified a listener for 'error' events.
  */
 MatrixCall.prototype.placeScreenSharingCall =
-    function(remoteVideoElement, localVideoElement) {
+    async function(remoteVideoElement, localVideoElement) {
     debuglog("placeScreenSharingCall");
     checkForErrorListener(this);
-    const screenConstraints = _getScreenSharingConstraints(this);
-    if (!screenConstraints) {
-        return;
-    }
     this.localVideoElement = localVideoElement;
     this.remoteVideoElement = remoteVideoElement;
     const self = this;
-    this.webRtc.getUserMedia(screenConstraints, function(stream) {
-        self.screenSharingStream = stream;
+    try {
+        self.screenSharingStream = await this.webRtc.getDisplayMedia({'audio': false});
         debuglog("Got screen stream, requesting audio stream...");
         const audioConstraints = _getUserMediaVideoContraints('voice');
         _placeCallWithConstraints(self, audioConstraints);
-    }, function(err) {
+    } catch(err) {
         self.emit("error",
             callError(
                 MatrixCall.ERR_NO_USER_MEDIA,
                 "Failed to get screen-sharing stream: " + err,
             ),
         );
-    });
+    }
+
     this.type = 'video';
     _tryPlayRemoteStream(this);
 };
@@ -1231,31 +1228,6 @@ const _createPeerConnection = function(self) {
     return pc;
 };
 
-const _getScreenSharingConstraints = function(call) {
-    const screen = global.screen;
-    if (!screen) {
-        call.emit("error", callError(
-            MatrixCall.ERR_NO_USER_MEDIA,
-            "Couldn't determine screen sharing constaints.",
-        ));
-        return;
-    }
-
-    return {
-        video: {
-            mediaSource: 'screen',
-            mandatory: {
-                chromeMediaSource: "screen",
-                chromeMediaSourceId: "" + Date.now(),
-                maxWidth: screen.width,
-                maxHeight: screen.height,
-                minFrameRate: 1,
-                maxFrameRate: 10,
-            },
-        },
-    };
-};
-
 const _getUserMediaVideoContraints = function(callType) {
     const isWebkit = !!global.window.navigator.webkitGetUserMedia;
 
@@ -1371,6 +1343,14 @@ export function createNewMatrixCall(client, roomId, options) {
         };
     }
 
+    const getDisplayMedia = (
+        w.navigator.mediaDevices && w.navigator.mediaDevices.getDisplayMedia ||
+        w.navigator.getDisplayMedia
+    );
+    if (getDisplayMedia) {
+        webRtc.getDisplayMedia = getDisplayMedia.bind(w.navigator.mediaDevices);
+    }
+
     // Firefox throws on so little as accessing the RTCPeerConnection when operating in
     // a secure mode. There's some information at https://bugzilla.mozilla.org/show_bug.cgi?id=1542616
     // though the concern is that the browser throwing a SecurityError will brick the