v8.5.0

Add support for olm fallback keys

.eslintrc.js

@@ -28,6 +28,8 @@ module.exports = {
         "indent": "off",
         "no-constant-condition": "off",
         "no-async-promise-executor": "off",
+        // We use a `logger` intermediary module
+        "no-console": "error",
     },
     overrides: [{
         "files": ["src/**/*.ts"],

CHANGELOG.md

@@ -1,3 +1,97 @@
+Changes in [8.5.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.5.0) (2020-10-12)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.5.0-rc.1...v8.5.0)
+
+* No changes since rc.1
+
+Changes in [8.5.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.5.0-rc.1) (2020-10-07)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.4.1...v8.5.0-rc.1)
+
+ * Add support for olm fallback keys
+   [\#1467](https://github.com/matrix-org/matrix-js-sdk/pull/1467)
+ * Fix editing local echoes not updating them in real time
+   [\#1492](https://github.com/matrix-org/matrix-js-sdk/pull/1492)
+ * Fix re-emit of Event.replaced to be on client and not room
+   [\#1491](https://github.com/matrix-org/matrix-js-sdk/pull/1491)
+ * Add space to log line
+   [\#1496](https://github.com/matrix-org/matrix-js-sdk/pull/1496)
+ * Revert "Convert call.js to Typescript & update WebRTC APIs"
+   [\#1493](https://github.com/matrix-org/matrix-js-sdk/pull/1493)
+ * Convert call.js to Typescript & update WebRTC APIs
+   [\#1487](https://github.com/matrix-org/matrix-js-sdk/pull/1487)
+ * Dehydrate and rehydrate devices
+   [\#1436](https://github.com/matrix-org/matrix-js-sdk/pull/1436)
+ * Keep local device after processing device list sync
+   [\#1490](https://github.com/matrix-org/matrix-js-sdk/pull/1490)
+ * Enforce logger module via lint rules
+   [\#1489](https://github.com/matrix-org/matrix-js-sdk/pull/1489)
+ * Extend method redactEvent with reason
+   [\#1462](https://github.com/matrix-org/matrix-js-sdk/pull/1462)
+ * Catch exception from call event handler
+   [\#1484](https://github.com/matrix-org/matrix-js-sdk/pull/1484)
+ * Ignore invalid candidates
+   [\#1483](https://github.com/matrix-org/matrix-js-sdk/pull/1483)
+ * Always push docs if they are generated
+   [\#1478](https://github.com/matrix-org/matrix-js-sdk/pull/1478)
+ * Only sign key backup with cross-signing keys when available
+   [\#1481](https://github.com/matrix-org/matrix-js-sdk/pull/1481)
+ * Upgrade dependencies
+   [\#1479](https://github.com/matrix-org/matrix-js-sdk/pull/1479)
+
+Changes in [8.4.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.4.1) (2020-09-28)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.4.0...v8.4.1)
+
+ * Catch exception from call event handler
+   [\#1486](https://github.com/matrix-org/matrix-js-sdk/pull/1486)
+ * Ignore invalid candidates
+   [\#1485](https://github.com/matrix-org/matrix-js-sdk/pull/1485)
+
+Changes in [8.4.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.4.0) (2020-09-28)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.4.0-rc.1...v8.4.0)
+
+ * Only sign key backup with cross-signing keys when available
+   [\#1482](https://github.com/matrix-org/matrix-js-sdk/pull/1482)
+
+Changes in [8.4.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.4.0-rc.1) (2020-09-23)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.3.0...v8.4.0-rc.1)
+
+ * If there are extraParams set, ensure that queryParams is defined
+   [\#1477](https://github.com/matrix-org/matrix-js-sdk/pull/1477)
+ * Add diagnostics to security bootstrap paths
+   [\#1475](https://github.com/matrix-org/matrix-js-sdk/pull/1475)
+ * Switch to a combination of better-docs and docdash
+   [\#1459](https://github.com/matrix-org/matrix-js-sdk/pull/1459)
+ * Undo attempts to cache private keys aggressively
+   [\#1474](https://github.com/matrix-org/matrix-js-sdk/pull/1474)
+ * Repair secret storage reset, cache keys when missing
+   [\#1472](https://github.com/matrix-org/matrix-js-sdk/pull/1472)
+ * Prevent parallel getVersions calls
+   [\#1471](https://github.com/matrix-org/matrix-js-sdk/pull/1471)
+ * Send end-of-candidates
+   [\#1473](https://github.com/matrix-org/matrix-js-sdk/pull/1473)
+ * Add a function for checking the /versions flag for forced e2ee
+   [\#1470](https://github.com/matrix-org/matrix-js-sdk/pull/1470)
+ * Add option to allow users of pantialaimon to use the SDK
+   [\#1469](https://github.com/matrix-org/matrix-js-sdk/pull/1469)
+ * Fixed Yarn broken link
+   [\#1468](https://github.com/matrix-org/matrix-js-sdk/pull/1468)
+ * some TypeScript and doc fixes
+   [\#1466](https://github.com/matrix-org/matrix-js-sdk/pull/1466)
+ * Remove Travis CI reference
+   [\#1464](https://github.com/matrix-org/matrix-js-sdk/pull/1464)
+ * Inject identity server token for 3pid invites on createRoom
+   [\#1463](https://github.com/matrix-org/matrix-js-sdk/pull/1463)
+
+Changes in [8.3.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.3.0) (2020-09-14)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.3.0-rc.1...v8.3.0)
+
+* No changes since rc.1
+
 Changes in [8.3.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.3.0-rc.1) (2020-09-09)
 ==========================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.2.0...v8.3.0-rc.1)

CONTRIBUTING.rst

@@ -28,10 +28,9 @@ use GitHub's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use Travis for continuous integration, and all pull requests get
-automatically tested by Travis: if your change breaks the build, then the PR
-will show that there are failed checks, so please check back after a few
-minutes.
+We use continuous integration, and all pull requests get automatically tested:
+if your change breaks the build, then the PR will show that there are failed
+checks, so please check back after a few minutes.
 
 Code style
 ~~~~~~~~~~

README.md

@@ -30,7 +30,7 @@ This SDK targets Node 10 for compatibility, which translates to ES6. If you're u
 a bundler like webpack you'll likely have to transpile dependencies, including this
 SDK, to match your target browsers.
 
-Using `yarn` instead of `npm` is recommended. Please see the Yarn [install guide](https://yarnpkg.com/docs/install/) 
+Using `yarn` instead of `npm` is recommended. Please see the Yarn [install guide](https://classic.yarnpkg.com/en/docs/install) 
 if you do not have it already.
 
 ``yarn add matrix-js-sdk``

jsdoc.json

@@ -18,6 +18,13 @@
     "readme": "README.md",
     "recurse": true,
     "verbose": true,
-    "template": "node_modules/better-docs"
+    "template": "node_modules/docdash"
+  },
+  "docdash": {
+    "static": true,
+    "private": false,
+    "search": true,
+    "collapse": true,
+    "typedefs": true
   }
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-js-sdk",
-  "version": "8.3.0-rc.1",
+  "version": "8.5.0",
   "description": "Matrix Client-Server SDK for Javascript",
   "scripts": {
     "prepare": "yarn build",
@@ -46,48 +46,49 @@
     "release.sh"
   ],
   "dependencies": {
-    "@babel/runtime": "^7.8.3",
+    "@babel/runtime": "^7.11.2",
     "another-json": "^0.2.0",
     "browser-request": "^0.3.3",
     "bs58": "^4.0.1",
-    "content-type": "^1.0.2",
-    "loglevel": "^1.6.4",
-    "qs": "^6.5.2",
-    "request": "^2.88.0",
-    "unhomoglyph": "^1.0.2"
+    "content-type": "^1.0.4",
+    "loglevel": "^1.7.0",
+    "qs": "^6.9.4",
+    "request": "^2.88.2",
+    "unhomoglyph": "^1.0.6"
   },
   "devDependencies": {
-    "@babel/cli": "^7.7.5",
-    "@babel/core": "^7.7.5",
-    "@babel/plugin-proposal-class-properties": "^7.7.4",
-    "@babel/plugin-proposal-numeric-separator": "^7.7.4",
-    "@babel/plugin-proposal-object-rest-spread": "^7.7.4",
-    "@babel/plugin-syntax-dynamic-import": "^7.7.4",
-    "@babel/plugin-transform-runtime": "^7.8.3",
-    "@babel/preset-env": "^7.7.6",
-    "@babel/preset-typescript": "^7.7.4",
-    "@babel/register": "^7.7.4",
+    "@babel/cli": "^7.11.6",
+    "@babel/core": "^7.11.6",
+    "@babel/plugin-proposal-class-properties": "^7.10.4",
+    "@babel/plugin-proposal-numeric-separator": "^7.10.4",
+    "@babel/plugin-proposal-object-rest-spread": "^7.11.0",
+    "@babel/plugin-syntax-dynamic-import": "^7.8.3",
+    "@babel/plugin-transform-runtime": "^7.11.5",
+    "@babel/preset-env": "^7.11.5",
+    "@babel/preset-typescript": "^7.10.4",
+    "@babel/register": "^7.11.5",
     "@types/node": "12",
-    "@types/request": "^2.48.4",
-    "babel-eslint": "^10.0.3",
+    "@types/request": "^2.48.5",
+    "babel-eslint": "^10.1.0",
     "babel-jest": "^24.9.0",
     "babelify": "^10.0.0",
-    "better-docs": "^1.4.7",
-    "browserify": "^16.5.0",
-    "eslint": "7.3.1",
+    "better-docs": "^2.3.2",
+    "browserify": "^16.5.2",
+    "docdash": "^1.2.0",
+    "eslint": "7.9.0",
     "eslint-config-matrix-org": "^0.1.2",
-    "eslint-plugin-babel": "^5.3.0",
+    "eslint-plugin-babel": "^5.3.1",
     "exorcist": "^1.0.1",
-    "fake-indexeddb": "^3.0.0",
+    "fake-indexeddb": "^3.1.2",
     "jest": "^24.9.0",
-    "jest-localstorage-mock": "^2.4.0",
-    "jsdoc": "^3.5.5",
+    "jest-localstorage-mock": "^2.4.3",
+    "jsdoc": "^3.6.6",
     "matrix-mock-request": "^1.2.3",
-    "olm": "https://packages.matrix.org/npm/olm/olm-3.1.4.tgz",
-    "rimraf": "^3.0.0",
-    "terser": "^4.4.3",
-    "tsify": "^4.0.1",
-    "typescript": "^3.7.3"
+    "olm": "https://packages.matrix.org/npm/olm/olm-3.2.1.tgz",
+    "rimraf": "^3.0.2",
+    "terser": "^4.8.0",
+    "tsify": "^4.0.2",
+    "typescript": "^3.9.7"
   },
   "jest": {
     "testEnvironment": "node"

release.sh

@@ -327,6 +327,7 @@ if [ -z "$skip_jsdoc" ]; then
         $release index.html
     git add "$release"
     git commit --no-verify -m "Add jsdoc for $release" index.html "$release"
+    git push origin gh-pages
 fi
 
 # if it is a pre-release, leave it on the release branch for now.
@@ -341,11 +342,8 @@ git checkout master
 git pull
 git merge "$rel_branch"
 
-# push master and docs (if generated) to github
+# push master to github
 git push origin master
-if [ -z "$skip_jsdoc" ]; then
-    git push origin gh-pages
-fi
 
 # finally, merge master back onto develop (if it exists)
 if [ $(git branch -lr | grep origin/develop -c) -ge 1 ]; then

spec/browserify/sync-browserify.spec.js

@@ -99,5 +99,5 @@ describe("Browserify Test", function() {
                 client.once("sync.unexpectedError", reject);
             }),
         ]);
-    }, 10000);
+    }, 20000); // additional timeout as this test can take quite a while
 });

spec/olm-loader.js

@@ -31,5 +31,5 @@ try {
     const crypto = require('crypto');
     utils.setCrypto(crypto);
 } catch (err) {
-    console.log('nodejs was compiled without crypto support: some tests will fail');
+    logger.log('nodejs was compiled without crypto support: some tests will fail');
 }

spec/unit/crypto/CrossSigningInfo.spec.js

@@ -26,6 +26,7 @@ import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
 import 'fake-indexeddb/auto';
 import 'jest-localstorage-mock';
 import {OlmDevice} from "../../../src/crypto/OlmDevice";
+import {logger} from '../../../src/logger';
 
 const userId = "@alice:example.com";
 
@@ -51,7 +52,7 @@ const masterKeyPub = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk";
 
 describe("CrossSigningInfo.getCrossSigningKey", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 
@@ -83,9 +84,16 @@ describe("CrossSigningInfo.getCrossSigningKey", function() {
         const info = new CrossSigningInfo(userId, {
             getCrossSigningKey: () => testKey,
         });
-        const [pubKey, ab] = await info.getCrossSigningKey("master", masterKeyPub);
+        const [pubKey, pkSigning] = await info.getCrossSigningKey("master", masterKeyPub);
         expect(pubKey).toEqual(masterKeyPub);
-        expect(ab).toEqual({a: 106712, b: 106712});
+        // check that the pkSigning object corresponds to the pubKey
+        const signature = pkSigning.sign("message");
+        const util = new global.Olm.Utility();
+        try {
+            util.ed25519_verify(pubKey, "message", signature);
+        } finally {
+            util.free();
+        }
     });
 
     it.each(types)("should request a key from the cache callback (if set)" +

spec/unit/crypto/DeviceList.spec.js

@@ -72,6 +72,8 @@ describe('DeviceList', function() {
     function createTestDeviceList() {
         const baseApis = {
             downloadKeysForUsers: downloadSpy,
+            getUserId: () => '@test1:sw1v.org',
+            deviceId: 'HGKAWHRVJQ',
         };
         const mockOlm = {
             verifySignature: function(key, message, signature) {},

spec/unit/crypto/cross-signing.spec.js

@@ -22,6 +22,7 @@ import {TestClient} from '../../TestClient';
 import {HttpResponse, setHttpResponses} from '../../test-utils';
 import { resetCrossSigningKeys } from "./crypto-utils";
 import { MatrixError } from '../../../src/http-api';
+import {logger} from '../../../src/logger';
 
 async function makeTestClient(userInfo, options, keys) {
     if (!keys) keys = {};
@@ -49,7 +50,7 @@ async function makeTestClient(userInfo, options, keys) {
 
 describe("Cross Signing", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 

spec/unit/crypto/secrets.spec.js

@@ -22,6 +22,7 @@ import {TestClient} from '../../TestClient';
 import {makeTestClients} from './verification/util';
 import {encryptAES} from "../../../src/crypto/aes";
 import {resetCrossSigningKeys, createSecretStorageKey} from "./crypto-utils";
+import {logger} from '../../../src/logger';
 
 import * as utils from "../../../src/utils";
 
@@ -29,7 +30,7 @@ try {
     const crypto = require('crypto');
     utils.setCrypto(crypto);
 } catch (err) {
-    console.log('nodejs was compiled without crypto support');
+    logger.log('nodejs was compiled without crypto support');
 }
 
 async function makeTestClient(userInfo, options) {
@@ -60,7 +61,7 @@ function sign(obj, key, userId) {
 
 describe("Secrets", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 

spec/unit/crypto/verification/util.js

@@ -18,12 +18,13 @@ limitations under the License.
 import {TestClient} from '../../../TestClient';
 import {MatrixEvent} from "../../../../src/models/event";
 import nodeCrypto from "crypto";
+import {logger} from '../../../../src/logger';
 
 export async function makeTestClients(userInfos, options) {
     const clients = [];
     const clientMap = {};
     const sendToDevice = function(type, map) {
-        // console.log(this.getUserId(), "sends", type, map);
+        // logger.log(this.getUserId(), "sends", type, map);
         for (const [userId, devMap] of Object.entries(map)) {
             if (userId in clientMap) {
                 for (const [deviceId, msg] of Object.entries(devMap)) {
@@ -67,7 +68,7 @@ export async function makeTestClients(userInfos, options) {
         setImmediate(() => {
             for (const tc of clients) {
                 if (tc.client === this) { // eslint-disable-line babel/no-invalid-this
-                    console.log("sending remote echo!!");
+                    logger.log("sending remote echo!!");
                     tc.client.emit("Room.timeline", remoteEcho);
                 } else {
                     tc.client.emit("Room.timeline", event);

src/@types/global.d.ts

@@ -14,6 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// this is needed to tell TS about global.Olm
+import * as Olm from "olm"; // eslint-disable-line @typescript-eslint/no-unused-vars
+
 export {};
 
 declare global {

src/base-apis.js

@@ -462,8 +462,26 @@ MatrixBaseApis.prototype.getFallbackAuthUrl = function(loginType, authSessionId)
  * room_alias: {string(opt)}}</code>
  * @return {module:http-api.MatrixError} Rejects: with an error response.
  */
-MatrixBaseApis.prototype.createRoom = function(options, callback) {
-    // valid options include: room_alias_name, visibility, invite
+MatrixBaseApis.prototype.createRoom = async function(options, callback) {
+    // some valid options include: room_alias_name, visibility, invite
+
+    // inject the id_access_token if inviting 3rd party addresses
+    const invitesNeedingToken = (options.invite_3pid || [])
+        .filter(i => !i.id_access_token);
+    if (
+        invitesNeedingToken.length > 0 &&
+        this.identityServer &&
+        this.identityServer.getAccessToken &&
+        await this.doesServerAcceptIdentityAccessToken()
+    ) {
+        const identityAccessToken = await this.identityServer.getAccessToken();
+        if (identityAccessToken) {
+            for (const invite of invitesNeedingToken) {
+                invite.id_access_token = identityAccessToken;
+            }
+        }
+    }
+
     return this._http.authedRequest(
         callback, "POST", "/createRoom", undefined, options,
     );

src/client.js

@@ -55,6 +55,7 @@ import {PushProcessor} from "./pushprocessor";
 import {encodeBase64, decodeBase64} from "./crypto/olmlib";
 import { User } from "./models/user";
 import {AutoDiscovery} from "./autodiscovery";
+import {DEHYDRATION_ALGORITHM} from "./crypto/dehydration";
 
 const SCROLLBACK_DELAY_MS = 3000;
 export const CRYPTO_ENABLED = isCryptoAvailable();
@@ -182,6 +183,11 @@ function keyFromRecoverySession(session, decryptionKey) {
  * Optional. Whether to allow a fallback ICE server should be used for negotiating a
  * WebRTC connection if the homeserver doesn't provide any servers. Defaults to false.
  *
+ * @param {boolean} [opts.usingExternalCrypto]
+ * Optional. Whether to allow sending messages to encrypted rooms when encryption
+ * is not available internally within this SDK. This is useful if you are using an external
+ * E2E proxy, for example. Defaults to false.
+ *
  * @param {object} opts.cryptoCallbacks Optional. Callbacks for crypto and cross-signing.
  *     The cross-signing API is currently UNSTABLE and may change without notice.
  *
@@ -248,12 +254,12 @@ function keyFromRecoverySession(session, decryptionKey) {
  * device.
  * Args:
  *   {string} name The name of the secret being requested.
- *   {string} user_id (string) The user ID of the client requesting
- *   {string} device_id The device ID of the client requesting the secret.
- *   {string} request_id The ID of the request. Used to match a
+ *   {string} userId The user ID of the client requesting
+ *   {string} deviceId The device ID of the client requesting the secret.
+ *   {string} requestId The ID of the request. Used to match a
  *     corresponding `crypto.secrets.request_cancelled`. The request ID will be
  *     unique per sender, device pair.
- *   {DeviceTrustLevel} device_trust: The trust status of the device requesting
+ *   {DeviceTrustLevel} deviceTrust: The trust status of the device requesting
  *     the secret as returned by {@link module:client~MatrixClient#checkDeviceTrust}.
  */
 export function MatrixClient(opts) {
@@ -266,6 +272,8 @@ export function MatrixClient(opts) {
 
     this.reEmitter = new ReEmitter(this);
 
+    this.usingExternalCrypto = opts.usingExternalCrypto;
+
     this.store = opts.store || new StubStore();
 
     this.deviceId = opts.deviceId || null;
@@ -359,9 +367,9 @@ export function MatrixClient(opts) {
     // The pushprocessor caches useful things, so keep one and re-use it
     this._pushProcessor = new PushProcessor(this);
 
-    // Cache of the server's /versions response
+    // Promise to a response of the server's /versions response
     // TODO: This should expire: https://github.com/matrix-org/matrix-js-sdk/issues/1020
-    this._serverVersionsCache = null;
+    this._serverVersionsPromise = null;
 
     this._cachedCapabilities = null; // { capabilities: {}, lastUpdated: timestamp }
 
@@ -452,6 +460,121 @@ export function MatrixClient(opts) {
 utils.inherits(MatrixClient, EventEmitter);
 utils.extend(MatrixClient.prototype, MatrixBaseApis.prototype);
 
+/**
+ * Try to rehydrate a device if available.  The client must have been
+ * initialized with a `cryptoCallback.getDehydrationKey` option, and this
+ * function must be called before initCrypto and startClient are called.
+ *
+ * @return {Promise} Resolves to undefined if a device could not be dehydrated, or
+ *     to the new device ID if the dehydration was successful.
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixClient.prototype.rehydrateDevice = async function() {
+    if (this._crypto) {
+        throw new Error("Cannot rehydrate device after crypto is initialized");
+    }
+
+    if (!this._cryptoCallbacks.getDehydrationKey) {
+        return;
+    }
+
+    let getDeviceResult;
+    try {
+        getDeviceResult = await this._http.authedRequest(
+            undefined,
+            "GET",
+            "/dehydrated_device",
+            undefined, undefined,
+            {
+                prefix: "/_matrix/client/unstable/org.matrix.msc2697.v2",
+            },
+        );
+    } catch (e) {
+        logger.info("could not get dehydrated device", e);
+        return;
+    }
+
+    if (!getDeviceResult.device_data || !getDeviceResult.device_id) {
+        logger.info("no dehydrated device found");
+        return;
+    }
+
+    const account = new global.Olm.Account();
+    try {
+        const deviceData = getDeviceResult.device_data;
+        if (deviceData.algorithm !== DEHYDRATION_ALGORITHM) {
+            logger.warn("Wrong algorithm for dehydrated device");
+            return;
+        }
+        logger.log("unpickling dehydrated device");
+        const key = await this._cryptoCallbacks.getDehydrationKey(
+            deviceData,
+            (k) => {
+                // copy the key so that it doesn't get clobbered
+                account.unpickle(new Uint8Array(k), deviceData.account);
+            },
+        );
+        account.unpickle(key, deviceData.account);
+        logger.log("unpickled device");
+
+        const rehydrateResult = await this._http.authedRequest(
+            undefined,
+            "POST",
+            "/dehydrated_device/claim",
+            undefined,
+            {
+                device_id: getDeviceResult.device_id,
+            },
+            {
+                prefix: "/_matrix/client/unstable/org.matrix.msc2697.v2",
+            },
+        );
+
+        if (rehydrateResult.success === true) {
+            this.deviceId = getDeviceResult.device_id;
+            logger.info("using dehydrated device");
+            const pickleKey = this.pickleKey || "DEFAULT_KEY";
+            this._exportedOlmDeviceToImport = {
+                pickledAccount: account.pickle(pickleKey),
+                sessions: [],
+                pickleKey: pickleKey,
+            };
+            account.free();
+            return this.deviceId;
+        } else {
+            account.free();
+            logger.info("not using dehydrated device");
+            return;
+        }
+    } catch (e) {
+        account.free();
+        logger.warn("could not unpickle", e);
+    }
+};
+
+/**
+ * Set the dehydration key.  This will also periodically dehydrate devices to
+ * the server.
+ *
+ * @param {Uint8Array} key the dehydration key
+ * @param {object} [keyInfo] Information about the key.  Primarily for
+ *     information about how to generate the key from a passphrase.
+ * @param {string} [deviceDisplayName] The device display name for the
+ *     dehydrated device.
+ * @return {Promise} A promise that resolves when the dehydrated device is stored.
+ */
+MatrixClient.prototype.setDehydrationKey = async function(
+    key, keyInfo = {}, deviceDisplayName = undefined,
+) {
+    if (!(this._crypto)) {
+        logger.warn('not dehydrating device if crypto is not enabled');
+        return;
+    }
+    return await this._crypto._dehydrationManager.setDehydrationKey(
+        key, keyInfo, deviceDisplayName,
+    );
+};
+
 MatrixClient.prototype.exportDevice = async function() {
     if (!(this._crypto)) {
         logger.warn('not exporting device if crypto is not enabled');
@@ -2047,7 +2170,7 @@ MatrixClient.prototype._restoreKeyBackup = function(
     // This is async.
     this._crypto.storeSessionBackupPrivateKey(privKey)
     .catch((e) => {
-        console.warn("Error caching session backup key:", e);
+        logger.warn("Error caching session backup key:", e);
     }).then(cacheCompleteCallback);
 
     if (progressCallback) {
@@ -2749,6 +2872,13 @@ function _encryptEventIfNeeded(client, event, room) {
         return null;
     }
 
+    if (!client._crypto && client.usingExternalCrypto) {
+        // The client has opted to allow sending messages to encrypted
+        // rooms even if the room is encrypted, and we haven't setup
+        // crypto. This is useful for users of matrix-org/pantalaimon
+        return null;
+    }
+
     if (event.getType() === "m.reaction") {
         // For reactions, there is a very little gained by encrypting the entire
         // event, as relation data is already kept in the clear. Event
@@ -2844,14 +2974,19 @@ function _sendEventHttpRequest(client, event) {
  * @param {string} eventId
  * @param {string} [txnId]  transaction id. One will be made up if not
  *    supplied.
- * @param {module:client.callback} callback Optional.
+ * @param {object|module:client.callback} callbackOrOpts
+ *    Options to pass on, may contain `reason`.
+ *    Can be callback for backwards compatibility.
  * @return {Promise} Resolves: TODO
  * @return {module:http-api.MatrixError} Rejects: with an error response.
  */
-MatrixClient.prototype.redactEvent = function(roomId, eventId, txnId, callback) {
+MatrixClient.prototype.redactEvent = function(roomId, eventId, txnId, callbackOrOpts) {
+    const opts = typeof(callbackOrOpts) === 'object' ? callbackOrOpts : {};
+    const reason = opts.reason;
+    const callback = typeof(callbackOrOpts) === 'function' ? callbackOrOpts : undefined;
     return this._sendCompleteEvent(roomId, {
         type: "m.room.redaction",
-        content: {},
+        content: { reason: reason },
         redacts: eventId,
     }, txnId, callback);
 };
@@ -4981,19 +5116,22 @@ MatrixClient.prototype.stopClient = function() {
  * unstable APIs it supports
  * @return {Promise<object>} The server /versions response
  */
-MatrixClient.prototype.getVersions = async function() {
-    if (this._serverVersionsCache === null) {
-        this._serverVersionsCache = await this._http.request(
-            undefined, // callback
-            "GET", "/_matrix/client/versions",
-            undefined, // queryParams
-            undefined, // data
-            {
-                prefix: '',
-            },
-        );
+MatrixClient.prototype.getVersions = function() {
+    if (this._serverVersionsPromise) {
+        return this._serverVersionsPromise;
     }
-    return this._serverVersionsCache;
+
+    this._serverVersionsPromise = this._http.request(
+        undefined, // callback
+        "GET", "/_matrix/client/versions",
+        undefined, // queryParams
+        undefined, // data
+        {
+            prefix: '',
+        },
+    );
+
+    return this._serverVersionsPromise;
 };
 
 /**
@@ -5092,6 +5230,20 @@ MatrixClient.prototype.doesServerSupportUnstableFeature = async function(feature
     return unstableFeatures && !!unstableFeatures[feature];
 };
 
+/**
+ * Query the server to see if it is forcing encryption to be enabled for
+ * a given room preset, based on the /versions response.
+ * @param {string} presetName The name of the preset to check.
+ * @returns {Promise<boolean>} true if the server is forcing encryption
+ * for the preset.
+ */
+MatrixClient.prototype.doesServerForceEncryptionForPreset = async function(presetName) {
+    const response = await this.getVersions();
+    if (!response) return false;
+    const unstableFeatures = response["unstable_features"];
+    return unstableFeatures && !!unstableFeatures[`io.element.e2ee_forced.${presetName}`];
+};
+
 /**
  * Get if lazy loading members is being used.
  * @return {boolean} Whether or not members are lazy loaded by this client
@@ -5200,7 +5352,11 @@ function setupCallEventHandler(client) {
                     // This call has previously been answered or hung up: ignore it
                     return;
                 }
-                callEventHandler(e);
+                try {
+                    callEventHandler(e);
+                } catch (e) {
+                    logger.error("Caught exception handling call event", e);
+                }
             });
             callEventBuffer = [];
         }
@@ -5226,7 +5382,11 @@ function setupCallEventHandler(client) {
                 } else {
                     // This one wasn't buffered so just run the event handler for it
                     // straight away
-                    callEventHandler(event);
+                    try {
+                        callEventHandler(event);
+                    } catch (e) {
+                        logger.error("Caught exception handling call event", e);
+                    }
                 }
             });
         }
@@ -5431,9 +5591,8 @@ function _PojoToMatrixEventMapper(client, options) {
             }
             event.attemptDecryption(client._crypto);
         }
-        const room = client.getRoom(event.getRoomId());
-        if (room && !preventReEmit) {
-            room.reEmitter.reEmit(event, ["Event.replaced"]);
+        if (!preventReEmit) {
+            client.reEmitter.reEmit(event, ["Event.replaced"]);
         }
         return event;
     }

src/crypto/CrossSigning.js

@@ -211,13 +211,16 @@ export class CrossSigningInfo extends EventEmitter {
     /**
      * Check whether the private keys exist in the local key cache.
      *
+     * @param {string} [type] The type of key to get. One of "master",
+     * "self_signing", or "user_signing". Optional, will check all by default.
      * @returns {boolean} True if all keys are stored in the local cache.
      */
-    async isStoredInKeyCache() {
+    async isStoredInKeyCache(type) {
         const cacheCallbacks = this._cacheCallbacks;
         if (!cacheCallbacks) return false;
-        for (const type of ["master", "self_signing", "user_signing"]) {
-            if (!await cacheCallbacks.getCrossSigningKeyCache(type)) {
+        const types = type ? [type] : ["master", "self_signing", "user_signing"];
+        for (const t of types) {
+            if (!await cacheCallbacks.getCrossSigningKeyCache(t)) {
                 return false;
             }
         }
@@ -235,6 +238,9 @@ export class CrossSigningInfo extends EventEmitter {
         if (!cacheCallbacks) return keys;
         for (const type of ["master", "self_signing", "user_signing"]) {
             const privKey = await cacheCallbacks.getCrossSigningKeyCache(type);
+            if (!privKey) {
+                continue;
+            }
             keys.set(type, privKey);
         }
         return keys;
@@ -728,7 +734,7 @@ export async function requestKeysDuringVerification(baseApis, userId, deviceId)
     if (baseApis.getUserId() !== userId) {
         return;
     }
-    console.log("Cross-signing: Self-verification done; requesting keys");
+    logger.log("Cross-signing: Self-verification done; requesting keys");
     // This happens asynchronously, and we're not concerned about waiting for
     // it.  We return here in order to test.
     return new Promise((resolve, reject) => {
@@ -742,7 +748,7 @@ export async function requestKeysDuringVerification(baseApis, userId, deviceId)
         const crossSigning = new CrossSigningInfo(
             original.userId,
             { getCrossSigningKey: async (type) => {
-                console.debug("Cross-signing: requesting secret",
+                logger.debug("Cross-signing: requesting secret",
                                 type, deviceId);
                 const { promise } = client.requestSecret(
                     `m.cross_signing.${type}`, [deviceId],
@@ -805,6 +811,6 @@ export async function requestKeysDuringVerification(baseApis, userId, deviceId)
             timeout,
         ]).then(resolve, reject);
     }).catch((e) => {
-        console.warn("Cross-signing: failure while requesting keys:", e);
+        logger.warn("Cross-signing: failure while requesting keys:", e);
     });
 }

src/crypto/DeviceList.js

@@ -848,6 +848,7 @@ class DeviceListUpdateSerialiser {
 
             await _updateStoredDeviceKeysForUser(
                 this._olmDevice, userId, userStore, dkResponse || {},
+                this._baseApis.getUserId(), this._baseApis.deviceId,
             );
 
             // put the updates into the object that will be returned as our results
@@ -885,8 +886,9 @@ class DeviceListUpdateSerialiser {
 }
 
 
-async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
-        userResult) {
+async function _updateStoredDeviceKeysForUser(
+    _olmDevice, userId, userStore, userResult, localUserId, localDeviceId,
+) {
     let updated = false;
 
     // remove any devices in the store which aren't in the response
@@ -896,6 +898,13 @@ async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
         }
 
         if (!(deviceId in userResult)) {
+            if (userId === localUserId && deviceId === localDeviceId) {
+                logger.warn(
+                    `Local device ${deviceId} missing from sync, skipping removal`,
+                );
+                continue;
+            }
+
             logger.log("Device " + userId + ":" + deviceId +
                 " has been removed");
             delete userStore[deviceId];

src/crypto/OlmDevice.js

@@ -144,7 +144,7 @@ OlmDevice.prototype.init = async function(opts = {}) {
     try {
         if (fromExportedDevice) {
             if (pickleKey) {
-                console.warn(
+                logger.warn(
                     'ignoring opts.pickleKey'
                     + ' because opts.fromExportedDevice is present.',
                 );
@@ -477,6 +477,36 @@ OlmDevice.prototype.generateOneTimeKeys = function(numKeys) {
     );
 };
 
+/**
+ * Generate a new fallback keys
+ *
+ * @return {Promise} Resolved once the account is saved back having generated the key
+ */
+OlmDevice.prototype.generateFallbackKey = async function() {
+    await this._cryptoStore.doTxn(
+        'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                account.generate_fallback_key();
+                this._storeAccount(txn, account);
+            });
+        },
+    );
+};
+
+OlmDevice.prototype.getFallbackKey = async function() {
+    let result;
+    await this._cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                result = JSON.parse(account.fallback_key());
+            });
+        },
+    );
+    return result;
+};
+
 /**
  * Generate a new outbound session
  *

src/crypto/SecretStorage.js

@@ -247,7 +247,7 @@ export class SecretStorage extends EventEmitter {
         ) {
             const hasKey = await this.hasKey(keys[0]);
             if (hasKey) {
-                console.log("Fixing up passthrough secret: " + name);
+                logger.log("Fixing up passthrough secret: " + name);
                 await this.storePassthrough(name, keys[0]);
                 const newData = await this._baseApis.getAccountDataFromServer(name);
                 return newData;

src/crypto/dehydration.ts

@@ -0,0 +1,265 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {decodeBase64, encodeBase64} from './olmlib';
+import {IndexedDBCryptoStore} from '../crypto/store/indexeddb-crypto-store';
+import {decryptAES, encryptAES} from './aes';
+import anotherjson from "another-json";
+import {logger} from '../logger';
+
+// FIXME: these types should eventually go in a different file
+type Signatures = Record<string, Record<string, string>>;
+
+interface DeviceKeys {
+    algorithms: Array<string>;
+    device_id: string; // eslint-disable-line camelcase
+    user_id: string; // eslint-disable-line camelcase
+    keys: Record<string, string>;
+    signatures?: Signatures;
+}
+
+interface OneTimeKey {
+    key: string;
+    fallback?: boolean;
+    signatures?: Signatures;
+}
+
+export const DEHYDRATION_ALGORITHM = "org.matrix.msc2697.v1.olm.libolm_pickle";
+
+const oneweek = 7 * 24 * 60 * 60 * 1000;
+
+export class DehydrationManager {
+    private inProgress = false;
+    private timeoutId: any;
+    private key: Uint8Array;
+    private keyInfo: {[props: string]: any};
+    private deviceDisplayName: string;
+    constructor(private crypto) {
+        this.getDehydrationKeyFromCache();
+    }
+    async getDehydrationKeyFromCache(): Promise<void> {
+        return this.crypto._cryptoStore.doTxn(
+            'readonly',
+            [IndexedDBCryptoStore.STORE_ACCOUNT],
+            (txn) => {
+                this.crypto._cryptoStore.getSecretStorePrivateKey(
+                    txn,
+                    async (result) => {
+                        if (result) {
+                            const {key, keyInfo, deviceDisplayName, time} = result;
+                            const pickleKey = Buffer.from(this.crypto._olmDevice._pickleKey);
+                            const decrypted = await decryptAES(key, pickleKey, DEHYDRATION_ALGORITHM);
+                            this.key = decodeBase64(decrypted);
+                            this.keyInfo = keyInfo;
+                            this.deviceDisplayName = deviceDisplayName;
+                            const now = Date.now();
+                            const delay = Math.max(1, time + oneweek - now);
+                            this.timeoutId = global.setTimeout(
+                                this.dehydrateDevice.bind(this), delay,
+                            );
+                        }
+                    },
+                    "dehydration",
+                );
+            },
+        );
+    }
+    async setDehydrationKey(
+        key: Uint8Array, keyInfo: {[props: string]: any} = {},
+        deviceDisplayName: string = undefined,
+    ): Promise<void> {
+        if (!key) {
+            // unsetting the key -- cancel any pending dehydration task
+            if (this.timeoutId) {
+                global.clearTimeout(this.timeoutId);
+                this.timeoutId = undefined;
+            }
+            // clear storage
+            this.crypto._cryptoStore.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    this.crypto._cryptoStore.storeSecretStorePrivateKey(
+                        txn, "dehydration", null,
+                    );
+                },
+            );
+            this.key = undefined;
+            this.keyInfo = undefined;
+            return;
+        }
+
+        // Check to see if it's the same key as before.  If it's different,
+        // dehydrate a new device.  If it's the same, we can keep the same
+        // device.  (Assume that keyInfo and deviceDisplayNamme will be the
+        // same if the key is the same.)
+        let matches: boolean = this.key && key.length == this.key.length;
+        for (let i = 0; matches && i < key.length; i++) {
+            if (key[i] != this.key[i]) {
+                matches = false;
+            }
+        }
+        if (!matches) {
+            this.key = key;
+            this.keyInfo = keyInfo;
+            this.deviceDisplayName = deviceDisplayName;
+            // start dehydration in the background
+            this.dehydrateDevice();
+        }
+    }
+    private async dehydrateDevice(): Promise<void> {
+        if (this.inProgress) {
+            logger.log("Dehydration already in progress -- not starting new dehydration");
+            return;
+        }
+        this.inProgress = true;
+        if (this.timeoutId) {
+            global.clearTimeout(this.timeoutId);
+            this.timeoutId = undefined;
+        }
+        try {
+            const pickleKey = Buffer.from(this.crypto._olmDevice._pickleKey);
+
+            // update the crypto store with the timestamp
+            const key = await encryptAES(encodeBase64(this.key), pickleKey, DEHYDRATION_ALGORITHM);
+            this.crypto._cryptoStore.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    this.crypto._cryptoStore.storeSecretStorePrivateKey(
+                        txn, "dehydration",
+                        {
+                            keyInfo: this.keyInfo,
+                            key,
+                            deviceDisplayName: this.deviceDisplayName,
+                            time: Date.now(),
+                        },
+                    );
+                },
+            );
+            logger.log("Attempting to dehydrate device");
+
+            logger.log("Creating account");
+            // create the account and all the necessary keys
+            const account = new global.Olm.Account();
+            account.create();
+            const e2eKeys = JSON.parse(account.identity_keys());
+
+            const maxKeys = account.max_number_of_one_time_keys();
+            // FIXME: generate in small batches?
+            account.generate_one_time_keys(maxKeys / 2);
+            account.generate_fallback_key();
+            const otks: Record<string, string> = JSON.parse(account.one_time_keys());
+            const fallbacks: Record<string, string> = JSON.parse(account.fallback_key());
+            account.mark_keys_as_published();
+
+            // dehydrate the account and store it on the server
+            const pickledAccount = account.pickle(new Uint8Array(this.key));
+
+            const deviceData: {[props: string]: any} = {
+                algorithm: DEHYDRATION_ALGORITHM,
+                account: pickledAccount,
+            };
+            if (this.keyInfo.passphrase) {
+                deviceData.passphrase = this.keyInfo.passphrase;
+            }
+
+            logger.log("Uploading account to server");
+            const dehydrateResult = await this.crypto._baseApis._http.authedRequest(
+                undefined,
+                "PUT",
+                "/dehydrated_device",
+                undefined,
+                {
+                    device_data: deviceData,
+                    initial_device_display_name: this.deviceDisplayName,
+                },
+                {
+                    prefix: "/_matrix/client/unstable/org.matrix.msc2697.v2",
+                },
+            );
+
+            // send the keys to the server
+            const deviceId = dehydrateResult.device_id;
+            logger.log("Preparing device keys", deviceId);
+            const deviceKeys: DeviceKeys = {
+                algorithms: this.crypto._supportedAlgorithms,
+                device_id: deviceId,
+                user_id: this.crypto._userId,
+                keys: {
+                    [`ed25519:${deviceId}`]: e2eKeys.ed25519,
+                    [`curve25519:${deviceId}`]: e2eKeys.curve25519,
+                },
+            };
+            const deviceSignature = account.sign(anotherjson.stringify(deviceKeys));
+            deviceKeys.signatures = {
+                [this.crypto._userId]: {
+                    [`ed25519:${deviceId}`]: deviceSignature,
+                },
+            };
+            if (this.crypto._crossSigningInfo.getId("self_signing")) {
+                await this.crypto._crossSigningInfo.signObject(deviceKeys, "self_signing");
+            }
+
+            logger.log("Preparing one-time keys");
+            const oneTimeKeys = {};
+            for (const [keyId, key] of Object.entries(otks.curve25519)) {
+                const k: OneTimeKey = {key};
+                const signature = account.sign(anotherjson.stringify(k));
+                k.signatures = {
+                    [this.crypto._userId]: {
+                        [`ed25519:${deviceId}`]: signature,
+                    },
+                };
+                oneTimeKeys[`signed_curve25519:${keyId}`] = k;
+            }
+
+            logger.log("Preparing fallback keys");
+            const fallbackKeys = {};
+            for (const [keyId, key] of Object.entries(fallbacks.curve25519)) {
+                const k: OneTimeKey = {key, fallback: true};
+                const signature = account.sign(anotherjson.stringify(k));
+                k.signatures = {
+                    [this.crypto._userId]: {
+                        [`ed25519:${deviceId}`]: signature,
+                    },
+                };
+                fallbackKeys[`signed_curve25519:${keyId}`] = k;
+            }
+
+            logger.log("Uploading keys to server");
+            await this.crypto._baseApis._http.authedRequest(
+                undefined,
+                "POST",
+                "/keys/upload/" + encodeURI(deviceId),
+                undefined,
+                {
+                    "device_keys": deviceKeys,
+                    "one_time_keys": oneTimeKeys,
+                    "org.matrix.msc2732.fallback_keys": fallbackKeys,
+                },
+            );
+            logger.log("Done dehydrating");
+
+            // dehydrate again in a week
+            this.timeoutId = global.setTimeout(
+                this.dehydrateDevice.bind(this), oneweek,
+            );
+        } finally {
+            this.inProgress = false;
+        }
+    }
+}

src/crypto/index.js

@@ -56,6 +56,7 @@ import {ToDeviceChannel, ToDeviceRequests} from "./verification/request/ToDevice
 import {IllegalMethod} from "./verification/IllegalMethod";
 import {KeySignatureUploadError} from "../errors";
 import {decryptAES, encryptAES} from './aes';
+import {DehydrationManager} from './dehydration';
 
 const DeviceVerification = DeviceInfo.DeviceVerification;
 
@@ -146,7 +147,7 @@ export function Crypto(baseApis, sessionStore, userId, deviceId,
                     method,
                 );
             } else {
-                console.warn(`Excluding unknown verification method ${method}`);
+                logger.warn(`Excluding unknown verification method ${method}`);
             }
         }
     } else {
@@ -243,6 +244,8 @@ export function Crypto(baseApis, sessionStore, userId, deviceId,
         baseApis, cryptoCallbacks,
     );
 
+    this._dehydrationManager = new DehydrationManager(this);
+
     // Assuming no app-supplied callback, default to getting from SSSS.
     if (!cryptoCallbacks.getCrossSigningKey && cryptoCallbacks.getSecretStorageKey) {
         cryptoCallbacks.getCrossSigningKey = async (type) => {
@@ -534,6 +537,15 @@ Crypto.prototype.bootstrapCrossSigning = async function({
         privateKeysInStorage
     );
 
+    // Log all relevant state for easier parsing of debug logs.
+    logger.log({
+        setupNewCrossSigning,
+        publicKeysOnDevice,
+        privateKeysInCache,
+        privateKeysInStorage,
+        privateKeysExistSomewhere,
+    });
+
     if (!privateKeysExistSomewhere || setupNewCrossSigning) {
         logger.log(
             "Cross-signing private keys not found locally or in secret storage, " +
@@ -691,6 +703,15 @@ Crypto.prototype.bootstrapSecretStorage = async function({
         oldKeyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES
     );
 
+    // Log all relevant state for easier parsing of debug logs.
+    logger.log({
+        keyBackupInfo,
+        setupNewKeyBackup,
+        setupNewSecretStorage,
+        storageExists,
+        oldKeyInfo,
+    });
+
     if (!storageExists && !keyBackupInfo) {
         // either we don't have anything, or we've been asked to restart
         // from scratch
@@ -740,10 +761,20 @@ Crypto.prototype.bootstrapSecretStorage = async function({
         // The backup is trusted because the user provided the private key.
         // Sign the backup with the cross-signing key so the key backup can
         // be trusted via cross-signing.
-        logger.log("Adding cross signing signature to key backup");
-        await this._crossSigningInfo.signObject(
-            keyBackupInfo.auth_data, "master",
-        );
+        if (
+            this._crossSigningInfo.getId() &&
+            this._crossSigningInfo.isStoredInKeyCache("master")
+        ) {
+            logger.log("Adding cross-signing signature to key backup");
+            await this._crossSigningInfo.signObject(
+                keyBackupInfo.auth_data, "master",
+            );
+        } else {
+            logger.warn(
+                "Cross-signing keys not available, skipping signature on key backup",
+            );
+        }
+
         builder.addSessionBackup(keyBackupInfo);
     } else {
         // 4S is already set up
@@ -792,8 +823,20 @@ Crypto.prototype.bootstrapSecretStorage = async function({
             algorithm: info.algorithm,
             auth_data: info.auth_data,
         };
-        // sign with cross-sign master key
-        await this._crossSigningInfo.signObject(data.auth_data, "master");
+
+        if (
+            this._crossSigningInfo.getId() &&
+            this._crossSigningInfo.isStoredInKeyCache("master")
+        ) {
+            // sign with cross-sign master key
+            logger.log("Adding cross-signing signature to key backup");
+            await this._crossSigningInfo.signObject(data.auth_data, "master");
+        } else {
+            logger.warn(
+                "Cross-signing keys not available, skipping signature on key backup",
+            );
+        }
+
         // sign with the device fingerprint
         await this._signObject(data.auth_data);
 
@@ -1281,23 +1324,20 @@ Crypto.prototype.checkOwnCrossSigningTrust = async function() {
     const seenPubkey = newCrossSigning.getId();
     const masterChanged = this._crossSigningInfo.getId() !== seenPubkey;
     if (masterChanged) {
-        // try to get the private key if the master key changed
         logger.info("Got new master public key", seenPubkey);
-
+        logger.info("Attempting to retrieve cross-signing master private key");
         let signing = null;
         try {
             const ret = await this._crossSigningInfo.getCrossSigningKey(
                 'master', seenPubkey,
             );
             signing = ret[1];
-            if (!signing) {
-                throw new Error("Cross-signing master private key not available");
-            }
+            logger.info("Got cross-signing master private key");
+        } catch (e) {
+            logger.error("Cross-signing master private key not available", e);
         } finally {
             if (signing) signing.free();
         }
-
-        logger.info("Got matching private key from callback for new public master key");
     }
 
     const oldSelfSigningId = this._crossSigningInfo.getId("self_signing");
@@ -1306,21 +1346,23 @@ Crypto.prototype.checkOwnCrossSigningTrust = async function() {
     // Update the version of our keys in our cross-signing object and the local store
     this._storeTrustedSelfKeys(newCrossSigning.keys);
 
+    const selfSigningChanged = oldSelfSigningId !== newCrossSigning.getId("self_signing");
+    const userSigningChanged = oldUserSigningId !== newCrossSigning.getId("user_signing");
+
     const keySignatures = {};
 
-    if (oldSelfSigningId !== newCrossSigning.getId("self_signing")) {
+    if (selfSigningChanged) {
         logger.info("Got new self-signing key", newCrossSigning.getId("self_signing"));
-
-        // Try to cache the self-signing private key as a side-effect
+        logger.info("Attempting to retrieve cross-signing self-signing private key");
         let signing = null;
         try {
             const ret = await this._crossSigningInfo.getCrossSigningKey(
                 "self_signing", newCrossSigning.getId("self_signing"),
             );
             signing = ret[1];
-            logger.info(
-                "Got matching private key from callback for new public self-signing key",
-            );
+            logger.info("Got cross-signing self-signing private key");
+        } catch (e) {
+            logger.error("Cross-signing self-signing private key not available", e);
         } finally {
             if (signing) signing.free();
         }
@@ -1331,19 +1373,18 @@ Crypto.prototype.checkOwnCrossSigningTrust = async function() {
         );
         keySignatures[this._deviceId] = signedDevice;
     }
-    if (oldUserSigningId !== newCrossSigning.getId("user_signing")) {
+    if (userSigningChanged) {
         logger.info("Got new user-signing key", newCrossSigning.getId("user_signing"));
-
-        // Try to cache the user-signing private key as a side-effect
+        logger.info("Attempting to retrieve cross-signing user-signing private key");
         let signing = null;
         try {
             const ret = await this._crossSigningInfo.getCrossSigningKey(
                 "user_signing", newCrossSigning.getId("user_signing"),
             );
             signing = ret[1];
-            logger.info(
-                "Got matching private key from callback for new public user-signing key",
-            );
+            logger.info("Got cross-signing user-signing private key");
+        } catch (e) {
+            logger.error("Cross-signing user-signing private key not available", e);
         } finally {
             if (signing) signing.free();
         }
@@ -1818,6 +1859,14 @@ Crypto.prototype.updateOneTimeKeyCount = function(currentCount) {
     }
 };
 
+Crypto.prototype.setNeedsNewFallback = function(needsNewFallback) {
+    this._needsNewFallback = !!needsNewFallback;
+};
+
+Crypto.prototype.getNeedsNewFallback = function() {
+    return this._needsNewFallback;
+};
+
 // check if it's time to upload one-time keys, and do so if so.
 function _maybeUploadOneTimeKeys(crypto) {
     // frequency with which to check & upload one-time keys
@@ -1865,27 +1914,31 @@ function _maybeUploadOneTimeKeys(crypto) {
     // out stale private keys that won't receive a message.
     const keyLimit = Math.floor(maxOneTimeKeys / 2);
 
-    function uploadLoop(keyCount) {
-        if (keyLimit <= keyCount) {
-            // If we don't need to generate any more keys then we are done.
-            return Promise.resolve();
-        }
+    async function uploadLoop(keyCount) {
+        while (keyLimit > keyCount || crypto.getNeedsNewFallback()) {
+            // Ask olm to generate new one time keys, then upload them to synapse.
+            if (keyLimit > keyCount) {
+                logger.info("generating oneTimeKeys");
+                const keysThisLoop = Math.min(keyLimit - keyCount, maxKeysPerCycle);
+                await crypto._olmDevice.generateOneTimeKeys(keysThisLoop);
+            }
 
-        const keysThisLoop = Math.min(keyLimit - keyCount, maxKeysPerCycle);
+            if (crypto.getNeedsNewFallback()) {
+                logger.info("generating fallback key");
+                await crypto._olmDevice.generateFallbackKey();
+            }
 
-        // Ask olm to generate new one time keys, then upload them to synapse.
-        return crypto._olmDevice.generateOneTimeKeys(keysThisLoop).then(() => {
-            return _uploadOneTimeKeys(crypto);
-        }).then((res) => {
+            logger.info("calling _uploadOneTimeKeys");
+            const res = await _uploadOneTimeKeys(crypto);
             if (res.one_time_key_counts && res.one_time_key_counts.signed_curve25519) {
                 // if the response contains a more up to date value use this
                 // for the next loop
-                return uploadLoop(res.one_time_key_counts.signed_curve25519);
+                keyCount = res.one_time_key_counts.signed_curve25519;
             } else {
-                throw new Error("response for uploading keys does not contain "
-                              + "one_time_key_counts.signed_curve25519");
+                throw new Error("response for uploading keys does not contain " +
+                                "one_time_key_counts.signed_curve25519");
             }
-        });
+        }
     }
 
     crypto._oneTimeKeyCheckInProgress = true;
@@ -1917,11 +1970,22 @@ function _maybeUploadOneTimeKeys(crypto) {
 
 // returns a promise which resolves to the response
 async function _uploadOneTimeKeys(crypto) {
+    const promises = [];
+
+    const fallbackJson = {};
+    if (crypto.getNeedsNewFallback()) {
+        const fallbackKeys = await crypto._olmDevice.getFallbackKey();
+        for (const [keyId, key] of Object.entries(fallbackKeys.curve25519)) {
+            const k = { key, fallback: true };
+            fallbackJson["signed_curve25519:" + keyId] = k;
+            promises.push(crypto._signObject(k));
+        }
+        crypto.setNeedsNewFallback(false);
+    }
+
     const oneTimeKeys = await crypto._olmDevice.getOneTimeKeys();
     const oneTimeJson = {};
 
-    const promises = [];
-
     for (const keyId in oneTimeKeys.curve25519) {
         if (oneTimeKeys.curve25519.hasOwnProperty(keyId)) {
             const k = {
@@ -1935,7 +1999,8 @@ async function _uploadOneTimeKeys(crypto) {
     await Promise.all(promises);
 
     const res = await crypto._baseApis.uploadKeysRequest({
-        one_time_keys: oneTimeJson,
+        "one_time_keys": oneTimeJson,
+        "org.matrix.msc2732.fallback_keys": fallbackJson,
     });
 
     await crypto._olmDevice.markKeysAsPublished();
@@ -2362,7 +2427,7 @@ Crypto.prototype.getEventSenderDeviceInfo = function(event) {
     if (claimedKey !== device.getFingerprint()) {
         logger.warn(
             "Event " + event.getId() + " claims ed25519 key " + claimedKey +
-                "but sender device has key " + device.getFingerprint());
+                " but sender device has key " + device.getFingerprint());
         return null;
     }
 

src/crypto/verification/QRCode.js

@@ -26,6 +26,7 @@ import {
     newUserCancelledError,
 } from './Error';
 import {encodeUnpaddedBase64, decodeBase64} from "../olmlib";
+import {logger} from '../../logger';
 
 export const SHOW_QR_CODE_METHOD = "m.qr_code.show.v1";
 export const SCAN_QR_CODE_METHOD = "m.qr_code.scan.v1";
@@ -94,7 +95,7 @@ export class ReciprocateQRCode extends Base {
             if (!targetKey) throw newKeyMismatchError();
 
             if (keyInfo !== targetKey) {
-                console.error("key ID from key info does not match");
+                logger.error("key ID from key info does not match");
                 throw newKeyMismatchError();
             }
             for (const deviceKeyId in device.keys) {
@@ -102,7 +103,7 @@ export class ReciprocateQRCode extends Base {
                 const deviceTargetKey = keys[deviceKeyId];
                 if (!deviceTargetKey) throw newKeyMismatchError();
                 if (device.keys[deviceKeyId] !== deviceTargetKey) {
-                    console.error("master key does not match");
+                    logger.error("master key does not match");
                     throw newKeyMismatchError();
                 }
             }

src/http-api.js

@@ -651,12 +651,10 @@ MatrixHttpApi.prototype = {
 
         const self = this;
         if (this.opts.extraParams) {
-            for (const key in this.opts.extraParams) {
-                if (!this.opts.extraParams.hasOwnProperty(key)) {
-                    continue;
-                }
-                queryParams[key] = this.opts.extraParams[key];
-            }
+            queryParams = {
+              ...queryParams,
+              ...this.opts.extraParams,
+            };
         }
 
         const headers = utils.extend({}, opts.headers || {});
@@ -963,7 +961,7 @@ export async function retryNetworkOperation(maxAttempts, callback) {
         try {
             if (attempts > 0) {
                 const timeout = 1000 * Math.pow(2, attempts);
-                console.log(`network operation failed ${attempts} times,` +
+                logger.log(`network operation failed ${attempts} times,` +
                     ` retrying in ${timeout}ms...`);
                 await new Promise(r => setTimeout(r, timeout));
             }

src/logger.js

@@ -40,11 +40,13 @@ log.methodFactory = function(methodName, logLevel, loggerName) {
             methodName === "warn" ||
             methodName === "trace" ||
             methodName === "info";
+        /* eslint-disable no-console */
         if (supportedByConsole) {
             return console[methodName](...args);
         } else {
             return console.log(...args);
         }
+        /* eslint-enable no-console */
     };
 };
 

src/matrix.ts

@@ -121,11 +121,41 @@ interface ICreateClientOpts {
     scheduler?: MatrixScheduler;
     request?: Request;
     userId?: string;
+    deviceId?: string;
     accessToken?: string;
     identityServer?: any;
     localTimeoutMs?: number;
     useAuthorizationHeader?: boolean;
     queryParams?: Record<string, unknown>;
+    deviceToImport?: {
+        olmDevice: {
+            pickledAccount: string;
+            sessions: Array<Record<string, any>>;
+            pickleKey: string;
+        };
+        userId: string;
+        deviceId: string;
+    };
+    sessionStore?: any;
+    unstableClientRelationAggregation?: boolean;
+    verificationMethods?: Array<any>;
+    forceTURN?: boolean;
+    fallbackICEServerAllowed?: boolean;
+    cryptoCallbacks?: {
+        getCrossSigningKey?: (keyType: string, pubKey: Uint8Array) => Promise<Uint8Array>;
+        saveCrossSigningKeys?: (keys: Record<string, Uint8Array>) => unknown;
+        shouldUpgradeDeviceVerifications?: (
+            users: Record<string, any>
+        ) => Promise<Array<string>>;
+        getSecretStorageKey?: (
+            keys: {keys: Record<string, {pubkey: Uint8Array}>}, name: string
+        ) => Promise<[string, Uint8Array] | null>;
+        cacheSecretStorageKey?: (keyId: string, key: Uint8Array) => unknown;
+        onSecretRequested?: (
+            name: string, userId: string, deviceId: string,
+            requestId: string, deviceTrust: any
+        ) => Promise<string>;
+    };
 }
 
 /**

src/models/event-timeline-set.js

@@ -806,7 +806,7 @@ EventTimelineSet.prototype.aggregateRelations = function(event) {
             this.room,
         );
         isNewRelations = true;
-        relatesToEvent = this.findEventById(relatesToEventId);
+        relatesToEvent = this.findEventById(relatesToEventId) || this.room.getPendingEvent(relatesToEventId);
         if (relatesToEvent) {
             relationsWithEventType.setTargetEvent(relatesToEvent);
         }

src/models/event.js

@@ -904,6 +904,8 @@ utils.extend(MatrixEvent.prototype, {
     /**
      * Set an event that replaces the content of this event, through an m.replace relation.
      *
+     * @fires module:models/event.MatrixEvent#"Event.replaced"
+     *
      * @param {MatrixEvent?} newEvent the event with the replacing content, if any.
      */
     makeReplaced(newEvent) {

src/models/relations.js

@@ -16,6 +16,7 @@ limitations under the License.
 
 import {EventEmitter} from 'events';
 import {EventStatus} from '../models/event';
+import {logger} from '../logger';
 
 /**
  * A container for relation events that supports easy access to common ways of
@@ -60,7 +61,7 @@ export class Relations extends EventEmitter {
 
         const relation = event.getRelation();
         if (!relation) {
-            console.error("Event must have relation info");
+            logger.error("Event must have relation info");
             return;
         }
 
@@ -68,7 +69,7 @@ export class Relations extends EventEmitter {
         const eventType = event.getType();
 
         if (this.relationType !== relationType || this.eventType !== eventType) {
-            console.error("Event relation info doesn't match this container");
+            logger.error("Event relation info doesn't match this container");
             return;
         }
 
@@ -104,7 +105,7 @@ export class Relations extends EventEmitter {
 
         const relation = event.getRelation();
         if (!relation) {
-            console.error("Event must have relation info");
+            logger.error("Event must have relation info");
             return;
         }
 
@@ -112,7 +113,7 @@ export class Relations extends EventEmitter {
         const eventType = event.getType();
 
         if (this.relationType !== relationType || this.eventType !== eventType) {
-            console.error("Event relation info doesn't match this container");
+            logger.error("Event relation info doesn't match this container");
             return;
         }
 

src/models/room.js

@@ -373,6 +373,20 @@ Room.prototype.hasPendingEvent = function(eventId) {
     return this._pendingEventList.some(event => event.getId() === eventId);
 };
 
+/**
+ * Get a specific event from the pending event list, if configured, null otherwise.
+ *
+ * @param {string} eventId The event ID to check for.
+ * @return {MatrixEvent}
+ */
+Room.prototype.getPendingEvent = function(eventId) {
+    if (this._opts.pendingEventOrdering !== "detached") {
+        return null;
+    }
+
+    return this._pendingEventList.find(event => event.getId() === eventId);
+};
+
 /**
  * Get the live unfiltered timeline for this room.
  *

src/pushprocessor.js

@@ -16,6 +16,7 @@ limitations under the License.
 */
 
 import {escapeRegExp, globToRegexp, isNullOrUndefined} from "./utils";
+import {logger} from './logger';
 
 /**
  * @module pushprocessor
@@ -444,7 +445,7 @@ PushProcessor.rewriteDefaultRules = function(incomingRules) {
         } else {
             // Add the rule
             const ruleId = override.rule_id;
-            console.warn(`Adding default global override for ${ruleId}`);
+            logger.warn(`Adding default global override for ${ruleId}`);
             globalOverrides.push(override);
         }
     }

src/sync.js

@@ -1361,6 +1361,16 @@ SyncApi.prototype._processSyncResponse = async function(
         const currentCount = data.device_one_time_keys_count.signed_curve25519 || 0;
         this.opts.crypto.updateOneTimeKeyCount(currentCount);
     }
+    if (this.opts.crypto && data["org.matrix.msc2732.device_unused_fallback_key_types"]) {
+        // The presence of device_unused_fallback_key_types indicates that the
+        // server supports fallback keys. If there's no unused
+        // signed_curve25519 fallback key we need a new one.
+        const unusedFallbackKeys = data["org.matrix.msc2732.device_unused_fallback_key_types"];
+        this.opts.crypto.setNeedsNewFallback(
+            unusedFallbackKeys instanceof Array &&
+            !unusedFallbackKeys.includes("signed_curve25519"),
+        );
+    }
 };
 
 /**

src/webrtc/call.js

@@ -92,6 +92,8 @@ export function MatrixCall(opts) {
     this.screenSharingStream = null;
 
     this._answerContent = null;
+
+    this._sentEndOfCandidates = false;
 }
 /** The length of time a call can be ringing for. */
 MatrixCall.CALL_TIMEOUT_MS = 60000;
@@ -703,12 +705,29 @@ MatrixCall.prototype._gotLocalIceCandidate = function(event) {
 
         // As with the offer, note we need to make a copy of this object, not
         // pass the original: that broke in Chrome ~m43.
+        if (event.candidate.candidate !== '' || !this._sentEndOfCandidates) {
+            const c = {
+                candidate: event.candidate.candidate,
+                sdpMid: event.candidate.sdpMid,
+                sdpMLineIndex: event.candidate.sdpMLineIndex,
+            };
+            sendCandidate(this, c);
+
+            if (event.candidate.candidate === '') this._sentEndOfCandidates = true;
+        }
+    }
+};
+
+MatrixCall.prototype._onIceGatheringStateChange = function(event) {
+    debuglog("ice gathering state changed to " + this.peerConn.iceGatheringState);
+    if (this.peerConn.iceGatheringState === 'complete' && !this._sentEndOfCandidates) {
+        // If we didn't get an empty-string candidate to signal the end of candidates,
+        // create one ourselves now gathering has finished.
         const c = {
-            candidate: event.candidate.candidate,
-            sdpMid: event.candidate.sdpMid,
-            sdpMLineIndex: event.candidate.sdpMLineIndex,
+            candidate: '',
         };
         sendCandidate(this, c);
+        this._sentEndOfCandidates = true;
     }
 };
 
@@ -722,6 +741,13 @@ MatrixCall.prototype._gotRemoteIceCandidate = function(cand) {
         //debuglog("Ignoring remote ICE candidate because call has ended");
         return;
     }
+    if (
+            (cand.sdpMid === null || cand.sdpMid === undefined) &&
+            (cand.sdpMLineIndex === null || cand.sdpMLineIndex === undefined)
+    ) {
+        debuglog("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex");
+        return;
+    }
     debuglog("Got remote ICE " + cand.sdpMid + " candidate: " + cand.candidate);
     this.peerConn.addIceCandidate(
         new this.webRtc.RtcIceCandidate(cand),
@@ -1231,6 +1257,7 @@ const _createPeerConnection = function(self) {
     pc.oniceconnectionstatechange = hookCallback(self, self._onIceConnectionStateChanged);
     pc.onsignalingstatechange = hookCallback(self, self._onSignallingStateChanged);
     pc.onicecandidate = hookCallback(self, self._gotLocalIceCandidate);
+    pc.onicegatheringstatechange = hookCallback(self, self._onIceGatheringStateChange);
     pc.onaddstream = hookCallback(self, self._onAddStream);
     return pc;
 };