v9.7.0-rc.1

Handle undefined peerconn

.babelrc

@@ -1,13 +1,19 @@
 {
-    "presets": ["es2015"],
-    "plugins": [
-        // this transforms async functions into generator functions, which
-        // are then made to use the regenerator module by babel's
-        // transform-regnerator plugin (which is enabled by es2015).
-        "transform-async-to-bluebird",
-
-        // This makes sure that the regenerator runtime is available to
-        // the transpiled code.
-        "transform-runtime",
+    "sourceMaps": true,
+    "presets": [
+        ["@babel/preset-env", {
+            "targets": {
+                "node": 10
+            },
+            "modules": "commonjs"
+        }],
+        "@babel/preset-typescript"
     ],
+    "plugins": [
+        "@babel/plugin-proposal-numeric-separator",
+        "@babel/plugin-proposal-class-properties",
+        "@babel/plugin-proposal-object-rest-spread",
+        "@babel/plugin-syntax-dynamic-import",
+        "@babel/plugin-transform-runtime"
+    ]
 }

.eslintrc.js

@@ -1,71 +1,45 @@
 module.exports = {
-    parser: "babel-eslint",
-    parserOptions: {
-        ecmaVersion: 6,
-        sourceType: "module",
-        ecmaFeatures: {
-        }
-    },
+    extends: ["matrix-org"],
+    plugins: [
+        "babel",
+    ],
     env: {
         browser: true,
         node: true,
-
-        // babel's transform-runtime converts references to ES6 globals such as
-        // Promise and Map to core-js polyfills, so we can use ES6 globals.
-        es6: true,
     },
-    extends: ["eslint:recommended", "google"],
+
     rules: {
-        // rules we've always adhered to or now do
-        "max-len": ["error", {
-            code: 90,
-            ignoreComments: true,
-        }],
-        curly: ["error", "multi-line"],
-        "prefer-const": ["error"],
-        "comma-dangle": ["error", {
-            arrays: "always-multiline",
-            objects: "always-multiline",
-            imports: "always-multiline",
-            exports: "always-multiline",
-            functions: "always-multiline",
-        }],
-
-        // loosen jsdoc requirements a little
-        "require-jsdoc": ["error", {
-            require: {
-                FunctionDeclaration: false,
-            }
-        }],
-        "valid-jsdoc": ["error", {
-            requireParamDescription: false,
-            requireReturn: false,
-            requireReturnDescription: false,
-        }],
-
-        // rules we do not want from eslint-recommended
-        "no-console": ["off"],
-        "no-constant-condition": ["off"],
-        "no-empty": ["error", { "allowEmptyCatch": true }],
-
-        // rules we do not want from the google styleguide
-        "object-curly-spacing": ["off"],
-        "spaced-comment": ["off"],
-
-        // in principle we prefer single quotes, but life is too short
-        quotes: ["off"],
-
-        // rules we'd ideally like to adhere to, but the current
-        // code does not (in most cases because it's still ES5)
-        // we set these to warnings, and assert that the number
-        // of warnings doesn't exceed a given threshold
         "no-var": ["warn"],
-        "brace-style": ["warn", "1tbs", {"allowSingleLine": true}],
         "prefer-rest-params": ["warn"],
         "prefer-spread": ["warn"],
         "one-var": ["warn"],
         "padded-blocks": ["warn"],
         "no-extend-native": ["warn"],
         "camelcase": ["warn"],
-    }
-}
+        "no-multi-spaces": ["error", { "ignoreEOLComments": true }],
+        "space-before-function-paren": ["error", {
+            "anonymous": "never",
+            "named": "never",
+            "asyncArrow": "always",
+        }],
+        "arrow-parens": "off",
+        "prefer-promise-reject-errors": "off",
+        "quotes": "off",
+        "indent": "off",
+        "no-constant-condition": "off",
+        "no-async-promise-executor": "off",
+        // We use a `logger` intermediary module
+        "no-console": "error",
+    },
+    overrides: [{
+        "files": ["src/**/*.ts"],
+        "extends": ["matrix-org/ts"],
+        "rules": {
+            // We're okay being explicit at the moment
+            "@typescript-eslint/no-empty-interface": "off",
+            // While we're converting to ts we make heavy use of this
+            "@typescript-eslint/no-explicit-any": "off",
+            "quotes": "off",
+        },
+    }],
+};

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+patreon: matrixdotorg
+liberapay: matrixdotorg

.gitignore

@@ -2,16 +2,17 @@
 /.jsdoc
 
 node_modules
+/.npmrc
+/*.log
+package-lock.json
 .lock-wscript
 build/Release
 coverage
 lib-cov
 out
-reports
 /dist
 /lib
-/specbuild
 
-# version file and tarball created by 'npm pack'
+# version file and tarball created by `npm pack` / `yarn pack`
 /git-revision.txt
 /matrix-js-sdk-*.tgz

.travis.yml

@@ -1,5 +0,0 @@
-language: node_js
-node_js:
-    - "10.11.0"
-script:
-    - ./travis.sh

CONTRIBUTING.rst

@@ -24,20 +24,27 @@ works. Develop is the unstable branch where all the development actually
 happens: the workflow is that contributors should fork the develop branch to
 make a 'feature' branch for a particular contribution, and then make a pull
 request to merge this back into the matrix.org 'official' develop branch. We
-use github's pull request workflow to review the contribution, and either ask
+use GitHub's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use Travis for continuous integration, and all pull requests get
-automatically tested by Travis: if your change breaks the build, then the PR
-will show that there are failed checks, so please check back after a few
-minutes.
+We use continuous integration, and all pull requests get automatically tested:
+if your change breaks the build, then the PR will show that there are failed
+checks, so please check back after a few minutes.
 
 Code style
 ~~~~~~~~~~
 
-The code-style for matrix-js-sdk is not formally documented, but contributors
-are encouraged to read the code style document for matrix-react-sdk
+The js-sdk aims to target TypeScript/ES6. All new files should be written in
+TypeScript and existing files should use ES6 principles where possible.
+
+Members should not be exported as a default export in general - it causes problems
+with the architecture of the SDK (index file becomes less clear) and could
+introduce naming problems (as default exports get aliased upon import). In
+general, avoid using `export default`.
+
+The remaining code-style for matrix-js-sdk is not formally documented, but
+contributors are encouraged to read the code style document for matrix-react-sdk
 (`<https://github.com/matrix-org/matrix-react-sdk/blob/master/code_style.md>`_)
 and follow the principles set out there.
 
@@ -60,8 +67,8 @@ Sign off
 ~~~~~~~~
 
 In order to have a concrete record that your contribution is intentional
-and you agree to license it under the same terms as the project's license, we've adopted the
-same lightweight approach that the Linux Kernel
+and you agree to license it under the same terms as the project's license, we've
+adopted the same lightweight approach that the Linux Kernel
 (https://www.kernel.org/doc/Documentation/SubmittingPatches), Docker
 (https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other
 projects use: the DCO (Developer Certificate of Origin:
@@ -109,7 +116,16 @@ include the line in your commit or pull request comment::
 
     Signed-off-by: Your Name <your@email.example.org>
 
-...using your real name; unfortunately pseudonyms and anonymous contributions
-can't be accepted. Git makes this trivial - just use the -s flag when you do
-``git commit``, having first set ``user.name`` and ``user.email`` git configs
-(which you should have done anyway :)
+We accept contributions under a legally identifiable name, such as your name on
+government documentation or common-law names (names claimed by legitimate usage
+or repute). Unfortunately, we cannot accept anonymous contributions at this
+time.
+
+Git allows you to add this signoff automatically when using the ``-s`` flag to
+``git commit``, which uses the name and email set in your ``user.name`` and
+``user.email`` git configs.
+
+If you forgot to sign off your commits before making your pull request and are
+on Git 2.17+ you can mass signoff using rebase::
+
+    git rebase --signoff origin/develop

README.md

@@ -1,8 +1,7 @@
 Matrix Javascript SDK
 =====================
-[![Build Status](http://matrix.org/jenkins/buildStatus/icon?job=JavascriptSDK)](http://matrix.org/jenkins/job/JavascriptSDK/)
 
-This is the [Matrix](https://matrix.org) Client-Server v1/v2 alpha SDK for
+This is the [Matrix](https://matrix.org) Client-Server r0 SDK for
 JavaScript. This SDK can be run in a browser or in Node.js.
 
 Quickstart
@@ -10,22 +9,35 @@ Quickstart
 
 In a browser
 ------------
-Download either the full or minified version from
+Download the browser version from
 https://github.com/matrix-org/matrix-js-sdk/releases/latest and add that as a
 ``<script>`` to your page. There will be a global variable ``matrixcs``
 attached to ``window`` through which you can access the SDK. See below for how to
 include libolm to enable end-to-end-encryption.
 
+The browser bundle supports recent versions of browsers. Typically this is ES2015
+or `> 0.5%, last 2 versions, Firefox ESR, not dead` if using 
+[browserlists](https://github.com/browserslist/browserslist).
+
 Please check [the working browser example](examples/browser) for more information.
 
 In Node.js
 ----------
 
-``npm install matrix-js-sdk``
+Ensure you have the latest LTS version of Node.js installed.
+
+This SDK targets Node 10 for compatibility, which translates to ES6. If you're using
+a bundler like webpack you'll likely have to transpile dependencies, including this
+SDK, to match your target browsers.
+
+Using `yarn` instead of `npm` is recommended. Please see the Yarn [install guide](https://classic.yarnpkg.com/en/docs/install) 
+if you do not have it already.
+
+``yarn add matrix-js-sdk``
 
 ```javascript
-  var sdk = require("matrix-js-sdk");
-  var client = sdk.createClient("https://matrix.org");
+  import * as sdk from "matrix-js-sdk";
+  const client = sdk.createClient("https://matrix.org");
   client.publicRooms(function(err, data) {
     console.log("Public Rooms: %s", JSON.stringify(data));
   });
@@ -56,7 +68,7 @@ client.once('sync', function(state, prevState, res) {
 To send a message:
 
 ```javascript
-var content = {
+const content = {
     "body": "message text",
     "msgtype": "m.text"
 };
@@ -76,7 +88,7 @@ client.on("Room.timeline", function(event, room, toStartOfTimeline) {
 });
 ```
 
-By default, the `matrix-js-sdk` client uses the `MatrixInMemoryStore` to store events as they are received. For example to iterate through the currently stored timeline for a room:
+By default, the `matrix-js-sdk` client uses the `MemoryStore` to store events as they are received. For example to iterate through the currently stored timeline for a room:
 
 ```javascript
 Object.keys(client.store.rooms).forEach((roomId) => {
@@ -158,7 +170,7 @@ which will be fulfilled in the future.
 The typical usage is something like:
 
 ```javascript
-  matrixClient.someMethod(arg1, arg2).done(function(result) {
+  matrixClient.someMethod(arg1, arg2).then(function(result) {
     ...
   });
 ```
@@ -170,10 +182,8 @@ you can pass the result of the promise into it with something like:
   matrixClient.someMethod(arg1, arg2).nodeify(callback);
 ```
 
-The main thing to note is that it is an error to discard the result of a
-promise-returning function, as that will cause exceptions to go unobserved. If
-you have nothing better to do with the result, just call ``.done()`` on it. See
-http://documentup.com/kriskowal/q/#the-end for more information.
+The main thing to note is that it is problematic to discard the result of a
+promise-returning function, as that will cause exceptions to go unobserved.
 
 Methods which return a promise show this in their documentation.
 
@@ -188,10 +198,10 @@ This section provides some useful code snippets which demonstrate the
 core functionality of the SDK. These examples assume the SDK is setup like this:
 
 ```javascript
-   var sdk = require("matrix-js-sdk");
-   var myUserId = "@example:localhost";
-   var myAccessToken = "QGV4YW1wbGU6bG9jYWxob3N0.qPEvLuYfNBjxikiCjP";
-   var matrixClient = sdk.createClient({
+   import * as sdk from "matrix-js-sdk";
+   const myUserId = "@example:localhost";
+   const myAccessToken = "QGV4YW1wbGU6bG9jYWxob3N0.qPEvLuYfNBjxikiCjP";
+   const matrixClient = sdk.createClient({
        baseUrl: "http://localhost:8008",
        accessToken: myAccessToken,
        userId: myUserId
@@ -203,7 +213,7 @@ core functionality of the SDK. These examples assume the SDK is setup like this:
 ```javascript
    matrixClient.on("RoomMember.membership", function(event, member) {
        if (member.membership === "invite" && member.userId === myUserId) {
-           matrixClient.joinRoom(member.roomId).done(function() {
+           matrixClient.joinRoom(member.roomId).then(function() {
                console.log("Auto-joined %s", member.roomId);
            });
        }
@@ -244,11 +254,11 @@ Output:
 
 ```javascript
    matrixClient.on("RoomState.members", function(event, state, member) {
-       var room = matrixClient.getRoom(state.roomId);
+       const room = matrixClient.getRoom(state.roomId);
        if (!room) {
            return;
        }
-       var memberList = state.getMembers();
+       const memberList = state.getMembers();
        console.log(room.name);
        console.log(Array(room.name.length + 1).join("="));  // underline
        for (var i = 0; i < memberList.length; i++) {
@@ -283,7 +293,7 @@ This SDK uses JSDoc3 style comments. You can manually build and
 host the API reference from the source files like this:
 
 ```
-  $ npm run gendoc
+  $ yarn gendoc
   $ cd .jsdoc
   $ python -m SimpleHTTPServer 8005
 ```
@@ -294,10 +304,10 @@ End-to-end encryption support
 =============================
 
 The SDK supports end-to-end encryption via the Olm and Megolm protocols, using
-[libolm](http://matrix.org/git/olm). It is left up to the application to make
-libolm available, via the ``Olm`` global.
+[libolm](https://gitlab.matrix.org/matrix-org/olm). It is left up to the
+application to make libolm available, via the ``Olm`` global.
 
-It is also necessry to call ``matrixClient.initCrypto()`` after creating a new
+It is also necessary to call ``matrixClient.initCrypto()`` after creating a new
 ``MatrixClient`` (but **before** calling ``matrixClient.startClient()``) to
 initialise the crypto layer.
 
@@ -314,20 +324,20 @@ specification.
 
 To provide the Olm library in a browser application:
 
- * download the transpiled libolm (from https://matrix.org/packages/npm/olm/).
+ * download the transpiled libolm (from https://packages.matrix.org/npm/olm/).
  * load ``olm.js`` as a ``<script>`` *before* ``browser-matrix.js``.
- 
+
 To provide the Olm library in a node.js application:
 
- * ``npm install https://matrix.org/packages/npm/olm/olm-3.0.0.tgz``
+ * ``yarn add https://packages.matrix.org/npm/olm/olm-3.1.4.tgz``
    (replace the URL with the latest version you want to use from
-    https://matrix.org/packages/npm/olm/)
+    https://packages.matrix.org/npm/olm/)
  * ``global.Olm = require('olm');`` *before* loading ``matrix-js-sdk``.
 
-If you want to package Olm as dependency for your node.js application, you
-can use ``npm install https://matrix.org/packages/npm/olm/olm-3.0.0.tgz
---save-optional`` (if your application also works without e2e crypto enabled)
-or ``--save`` (if it doesn't) to do so.
+If you want to package Olm as dependency for your node.js application, you can
+use ``yarn add https://packages.matrix.org/npm/olm/olm-3.1.4.tgz``. If your
+application also works without e2e crypto enabled, add ``--optional`` to mark it
+as an optional dependency.
 
 
 Contributing
@@ -337,7 +347,7 @@ want to use this SDK, skip this section.*
 
 First, you need to pull in the right build tools:
 ```
- $ npm install
+ $ yarn install
 ```
 
 Building
@@ -345,20 +355,15 @@ Building
 
 To build a browser version from scratch when developing::
 ```
- $ npm run build
-```
-
-To constantly do builds when files are modified (using ``watchify``)::
-```
- $ npm run watch
+ $ yarn build
 ```
 
 To run tests (Jasmine)::
 ```
- $ npm test
+ $ yarn test
 ```
 
 To run linting:
 ```
- $ npm run lint
+ $ yarn lint
 ```

browser-index.js

@@ -1,35 +0,0 @@
-var matrixcs = require("./lib/matrix");
-const request = require('browser-request');
-const queryString = require('qs');
-
-matrixcs.request(function(opts, fn) {
-    // We manually fix the query string for browser-request because
-    // it doesn't correctly handle cases like ?via=one&via=two. Instead
-    // we mimic `request`'s query string interface to make it all work
-    // as expected.
-    // browser-request will happily take the constructed string as the
-    // query string without trying to modify it further.
-    opts.qs = queryString.stringify(opts.qs || {}, opts.qsStringifyOptions);
-    return request(opts, fn);
-});
-
-// just *accessing* indexedDB throws an exception in firefox with
-// indexeddb disabled.
-var indexedDB;
-try {
-    indexedDB = global.indexedDB;
-} catch(e) {}
-
-// if our browser (appears to) support indexeddb, use an indexeddb crypto store.
-if (indexedDB) {
-    matrixcs.setCryptoStoreFactory(
-        function() {
-            return new matrixcs.IndexedDBCryptoStore(
-                indexedDB, "matrix-js-sdk:crypto"
-            );
-        }
-    );
-}
-
-module.exports = matrixcs; // keep export for browserify package deps
-global.matrixcs = matrixcs;

docs/storage-notes.md

@@ -0,0 +1,70 @@
+# Browser Storage Notes
+
+## Overview
+
+Browsers examined: Firefox 67, Chrome 75
+
+The examination below applies to the default, non-persistent storage policy.
+
+## Quota Measurement
+
+Browsers appear to enforce and measure the quota in terms of space on disk, not
+data stored, so you may be able to store more data than the simple sum of all
+input data depending on how compressible your data is.
+
+## Quota Limit
+
+Specs and documentation suggest we should consistently receive
+`QuotaExceededError` when we're near space limits, but the reality is a bit
+blurrier.
+
+When we are low on disk space overall or near the group limit / origin quota:
+
+* Chrome
+    * Log database may fail to start with AbortError
+    * IndexedDB fails to start for crypto: AbortError in connect from
+      indexeddb-store-worker
+    * When near the quota, QuotaExceededError is used more consistently
+* Firefox
+    * The first error will be QuotaExceededError
+    * Future write attempts will fail with various errors when space is low,
+      including nonsense like "InvalidStateError: A mutation operation was
+      attempted on a database that did not allow mutations."
+    * Once you start getting errors, the DB is effectively wedged in read-only
+      mode
+    * Can revive access if you reopen the DB
+
+## Cache Eviction
+
+While the Storage Standard says all storage for an origin group should be
+limited by a single quota, in practice, browsers appear to handle `localStorage`
+separately from the others, so it has a separate quota limit and isn't evicted
+when low on space.
+
+* Chrome, Firefox
+    * IndexedDB for origin deleted
+    * Local Storage remains in place
+
+## Persistent Storage
+
+Storage Standard offers a `navigator.storage.persist` API that can be used to
+request persistent storage that won't be deleted by the browser because of low
+space.
+
+* Chrome
+    * Chrome 75 seems to grant this without any prompt based on [interaction
+      criteria](https://developers.google.com/web/updates/2016/06/persistent-storage)
+* Firefox
+    * Firefox 67 shows a prompt to grant
+    * Reverting persistent seems to require revoking permission _and_ clearing
+      site data
+
+## Storage Estimation
+
+Storage Standard offers a `navigator.storage.estimate` API to get some clue of
+how much space remains.
+
+* Chrome, Firefox
+    * Can run this at any time to request an estimate of space remaining
+* Firefox
+    * Returns `0` for `usage` if a site is persisted

examples/browser/browserTest.js

@@ -1,4 +1,3 @@
-"use strict";
 console.log("Loading browser sdk");
 
 var client = matrixcs.createClient("http://matrix.org");

examples/crypto-browser/lib/.gitignore

@@ -0,0 +1,2 @@
+olm.js
+olm.wasm

examples/crypto-browser/lib/matrix.js

@@ -0,0 +1 @@
+../../../dist/browser-matrix.js

examples/crypto-browser/olm-device-export-import.html

@@ -0,0 +1,59 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="ie=edge">
+    <title>Test Crypto in Browser</title>
+    <script src="lib/olm.js"></script>
+    <script src="lib/matrix.js"></script>
+</head>
+<body>
+    <h1>Testing export/import of Olm devices in the browser</h1>
+    <ul>
+        <li>
+            Make sure you built the current version of the Matrix JS SDK
+            (<code>yarn build</code>)
+        </li>
+        <li>
+            copy <code>olm.js</code> and <code>olm.wasm</code>
+            from a recent release of Olm (was tested with version 3.1.4)
+            in directory <code>lib/</code>
+        </li>
+        <li>start a local Matrix homeserver (on port 8008, or change the port in the code)</li>
+        <li>Serve this HTML file (e.g. <code>python3 -m http.server</code>) and go to it through your browser</li>
+        <li>
+            in the JS console, do:
+            <pre>
+aliceMatrixClient = await newMatrixClient("alice-"+randomHex());
+await aliceMatrixClient.exportDevice();
+await aliceMatrixClient.getAccessToken();
+            </pre>
+        </li>
+        <li>
+            copy the result of <code>exportDevice</code> and <code>getAccessToken</code> somewhere
+            (<strong>not</strong> in a JS variable as it will be destroyed when you refresh the page)
+        </li>
+        <li><strong>refresh the page (F5)</strong> to make sure the client is destroyed</li>
+        <li>
+            Do the following, replacing <code>ALICE_ID</code>
+            with the user ID of Alice (you can find it in the exported data)
+            <pre>
+bobMatrixClient = await newMatrixClient("bob-"+randomHex());
+roomId = await bobMatrixClient.createEncryptedRoom([ALICE_ID]);
+await bobMatrixClient.sendTextMessage('Hi Alice!', roomId);
+            </pre>
+        </li>
+        <li>Again, <strong>refresh the page (F5)</strong>. You may want to clear your console as well.</li>
+        <li>
+            Now do the following, using the exported data and the access token you saved previously:
+            <pre>
+aliceMatrixClient = await importMatrixClient(EXPORTED_DATA, ACCESS_TOKEN);
+            </pre>
+        </li>
+        <li>You should see the message sent by Bob printed in the console.</li>
+    </ul>
+
+    <script src="olm-device-export-import.js"></script>
+</body>
+</html>

examples/crypto-browser/olm-device-export-import.js

@@ -0,0 +1,122 @@
+if (!Olm) {
+    console.error(
+        "global.Olm does not seem to be present."
+        + " Did you forget to add olm in the lib/ directory?"
+    );
+}
+
+const BASE_URL = 'http://localhost:8008';
+const ROOM_CRYPTO_CONFIG = { algorithm: 'm.megolm.v1.aes-sha2' };
+const PASSWORD = 'password';
+
+// useful to create new usernames
+window.randomHex = () => Math.floor(Math.random() * (10**6)).toString(16);
+
+window.newMatrixClient = async function (username) {
+    const registrationClient = matrixcs.createClient(BASE_URL);
+
+    const userRegisterResult = await registrationClient.register(
+      username,
+      PASSWORD,
+      null,
+      { type: 'm.login.dummy' }
+    );
+  
+    const matrixClient = matrixcs.createClient({
+      baseUrl: BASE_URL,
+      userId: userRegisterResult.user_id,
+      accessToken: userRegisterResult.access_token,
+      deviceId: userRegisterResult.device_id,
+      sessionStore: new matrixcs.WebStorageSessionStore(window.localStorage),
+      cryptoStore: new matrixcs.MemoryCryptoStore(),
+    });
+
+    extendMatrixClient(matrixClient);
+
+    await matrixClient.initCrypto();
+    await matrixClient.startClient();
+    return matrixClient;
+}
+
+window.importMatrixClient = async function (exportedDevice, accessToken) {
+    const matrixClient = matrixcs.createClient({
+      baseUrl: BASE_URL,
+      deviceToImport: exportedDevice,
+      accessToken,
+      sessionStore: new matrixcs.WebStorageSessionStore(window.localStorage),
+      cryptoStore: new matrixcs.MemoryCryptoStore(),
+    });
+
+    extendMatrixClient(matrixClient);
+
+    await matrixClient.initCrypto();
+    await matrixClient.startClient();
+    return matrixClient;
+}
+
+function extendMatrixClient(matrixClient) {
+    // automatic join
+    matrixClient.on('RoomMember.membership', async (event, member) => {
+        if (member.membership === 'invite' && member.userId === matrixClient.getUserId()) {
+            await matrixClient.joinRoom(member.roomId);
+            // setting up of room encryption seems to be triggered automatically
+            // but if we don't wait for it the first messages we send are unencrypted
+            await matrixClient.setRoomEncryption(member.roomId, { algorithm: 'm.megolm.v1.aes-sha2' })
+        }
+    });
+
+    matrixClient.onDecryptedMessage = message => {
+        console.log('Got encrypted message: ', message);
+    }
+
+    matrixClient.on('Event.decrypted', (event) => {
+        if (event.getType() === 'm.room.message'){
+            matrixClient.onDecryptedMessage(event.getContent().body);
+        } else {
+            console.log('decrypted an event of type', event.getType());
+            console.log(event);
+        }
+    });
+  
+    matrixClient.createEncryptedRoom = async function(usersToInvite) {
+        const {
+          room_id: roomId,
+        } = await this.createRoom({
+          visibility: 'private',
+          invite: usersToInvite,
+        });
+
+        // matrixClient.setRoomEncryption() only updates local state
+        // but does not send anything to the server
+        // (see https://github.com/matrix-org/matrix-js-sdk/issues/905)
+        // so we do it ourselves with 'sendStateEvent'
+        await this.sendStateEvent(
+          roomId, 'm.room.encryption', ROOM_CRYPTO_CONFIG,
+        );
+        await this.setRoomEncryption(
+          roomId, ROOM_CRYPTO_CONFIG,
+        );
+
+        // Marking all devices as verified
+        let room = this.getRoom(roomId);
+        let members = (await room.getEncryptionTargetMembers()).map(x => x["userId"])
+        let memberkeys = await this.downloadKeys(members);
+        for (const userId in memberkeys) {
+          for (const deviceId in memberkeys[userId]) {
+            await this.setDeviceVerified(userId, deviceId);
+          }
+        }
+
+        return roomId;
+    }
+
+    matrixClient.sendTextMessage = async function(message, roomId) {
+        return matrixClient.sendMessage(
+            roomId,
+            {
+                body: message,
+                msgtype: 'm.text',
+            }
+        )
+    }
+}

examples/node/app.js

@@ -1,5 +1,3 @@
-"use strict";
-
 var myUserId = "@example:localhost";
 var myAccessToken = "QGV4YW1wbGU6bG9jYWxob3N0.qPEvLuYfNBjxikiCjP";
 var sdk = require("matrix-js-sdk");
@@ -56,7 +54,7 @@ rl.on('line', function(line) {
                 }
             }
             if (notSentEvent) {
-                matrixClient.resendEvent(notSentEvent, viewingRoom).done(function() {
+                matrixClient.resendEvent(notSentEvent, viewingRoom).then(function() {
                     printMessages();
                     rl.prompt();
                 }, function(err) {
@@ -70,7 +68,7 @@ rl.on('line', function(line) {
         }
         else if (line.indexOf("/more ") === 0) {
             var amount = parseInt(line.split(" ")[1]) || 20;
-            matrixClient.scrollback(viewingRoom, amount).done(function(room) {
+            matrixClient.scrollback(viewingRoom, amount).then(function(room) {
                 printMessages();
                 rl.prompt();
             }, function(err) {
@@ -79,7 +77,7 @@ rl.on('line', function(line) {
         }
         else if (line.indexOf("/invite ") === 0) {
             var userId = line.split(" ")[1].trim();
-            matrixClient.invite(viewingRoom.roomId, userId).done(function() {
+            matrixClient.invite(viewingRoom.roomId, userId).then(function() {
                 printMessages();
                 rl.prompt();
             }, function(err) {
@@ -92,7 +90,7 @@ rl.on('line', function(line) {
             matrixClient.uploadContent({
                 stream: stream,
                 name: filename
-            }).done(function(url) {
+            }).then(function(url) {
                 var content = {
                     msgtype: "m.file",
                     body: filename,
@@ -116,7 +114,7 @@ rl.on('line', function(line) {
             viewingRoom = roomList[roomIndex];
             if (viewingRoom.getMember(myUserId).membership === "invite") {
                 // join the room first
-                matrixClient.joinRoom(viewingRoom.roomId).done(function(room) {
+                matrixClient.joinRoom(viewingRoom.roomId).then(function(room) {
                     setRoomList();
                     viewingRoom = room;
                     printMessages();
@@ -128,7 +126,7 @@ rl.on('line', function(line) {
             else {
                 printMessages();
             }
-        } 
+        }
     }
     rl.prompt();
 });
@@ -281,8 +279,8 @@ function printMemberList(room) {
             member.membership + new Array(10 - member.membership.length).join(" ")
         );
         print(
-            "%s"+fmt(" :: ")+"%s"+fmt(" (")+"%s"+fmt(")"), 
-            membershipWithPadding, member.name, 
+            "%s"+fmt(" :: ")+"%s"+fmt(" (")+"%s"+fmt(")"),
+            membershipWithPadding, member.name,
             (member.userId === myUserId ? "Me" : member.userId),
             fmt
         );
@@ -290,26 +288,27 @@ function printMemberList(room) {
 }
 
 function printRoomInfo(room) {
-    var eventDict = room.currentState.events;
+    var eventMap = room.currentState.events;
     var eTypeHeader = "    Event Type(state_key)    ";
     var sendHeader = "        Sender        ";
     // pad content to 100
     var restCount = (
-        100 - "Content".length - " | ".length - " | ".length - 
+        100 - "Content".length - " | ".length - " | ".length -
         eTypeHeader.length - sendHeader.length
     );
     var padSide = new Array(Math.floor(restCount/2)).join(" ");
     var contentHeader = padSide + "Content" + padSide;
     print(eTypeHeader+sendHeader+contentHeader);
     print(new Array(100).join("-"));
-    Object.keys(eventDict).forEach(function(eventType) {
+    eventMap.keys().forEach(function(eventType) {
         if (eventType === "m.room.member") { return; } // use /members instead.
-        Object.keys(eventDict[eventType]).forEach(function(stateKey) {
+        var eventEventMap = eventMap.get(eventType);
+        eventEventMap.keys().forEach(function(stateKey) {
             var typeAndKey = eventType + (
                 stateKey.length > 0 ? "("+stateKey+")" : ""
             );
             var typeStr = fixWidth(typeAndKey, eTypeHeader.length);
-            var event = eventDict[eventType][stateKey];
+            var event = eventEventMap.get(stateKey);
             var sendStr = fixWidth(event.getSender(), sendHeader.length);
             var contentStr = fixWidth(
                 JSON.stringify(event.getContent()), contentHeader.length

examples/voip/browserTest.js

@@ -1,4 +1,3 @@
-"use strict";
 console.log("Loading browser sdk");
 var BASE_URL = "https://matrix.org";
 var TOKEN = "accesstokengoeshere";

git-hooks/pre-commit

@@ -21,4 +21,4 @@ export PATH="$rootdir/node_modules/.bin:$PATH"
 
 # now run our checks
 cd "$tmpdir"
-npm run lint
+yarn lint

index.js

@@ -1,6 +0,0 @@
-var matrixcs = require("./lib/matrix");
-matrixcs.request(require("request"));
-module.exports = matrixcs;
-
-var utils = require("./lib/utils");
-utils.runPolyfills();

jenkins.sh

@@ -1,34 +0,0 @@
-#!/bin/bash -l
-
-set -x
-
-export NVM_DIR="$HOME/.nvm"
-[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
-
-nvm use 10 || exit $?
-npm install || exit $?
-
-RC=0
-
-function fail {
-    echo $@ >&2
-    RC=1
-}
-
-# don't use last time's test reports
-rm -rf reports coverage || exit $?
-
-npm test || fail "npm test finished with return code $?"
-
-npm run -s lint -- -f checkstyle > eslint.xml ||
-    fail "eslint finished with return code $?"
-
-# delete the old tarball, if it exists
-rm -f matrix-js-sdk-*.tgz
-
-npm pack ||
-    fail "npm pack finished with return code $?"
-
-npm run gendoc || fail "JSDoc failed with code $?"
-
-exit $RC

jsdoc.json

@@ -0,0 +1,30 @@
+{
+  "tags": {
+    "allowUnknownTags": true
+  },
+  "plugins": [
+    "node_modules/better-docs/category",
+    "node_modules/better-docs/typescript"
+  ],
+  "source": {
+    "include": [
+      "src"
+    ],
+    "includePattern": ".(ts|js)$"
+  },
+  "opts": {
+    "encoding": "utf8",
+    "destination": ".jsdoc",
+    "readme": "README.md",
+    "recurse": true,
+    "verbose": true,
+    "template": "node_modules/docdash"
+  },
+  "docdash": {
+    "static": true,
+    "private": false,
+    "search": true,
+    "collapse": true,
+    "typedefs": true
+  }
+}

package.json

@@ -1,24 +1,24 @@
 {
   "name": "matrix-js-sdk",
-  "version": "0.14.2",
+  "version": "9.7.0-rc.1",
   "description": "Matrix Client-Server SDK for Javascript",
-  "main": "index.js",
   "scripts": {
-    "test:build": "babel -s -d specbuild spec",
-    "test:run": "istanbul cover --report text --report cobertura --config .istanbul.yml -i \"lib/**/*.js\" node_modules/mocha/bin/_mocha -- --recursive specbuild --colors --reporter mocha-jenkins-reporter --reporter-options junit_report_path=reports/test-results.xml",
-    "test:watch": "mocha --watch --compilers js:babel-core/register --recursive spec --colors",
-    "test": "npm run test:build && npm run test:run",
-    "check": "npm run test:build && _mocha --recursive specbuild --colors",
-    "gendoc": "babel --no-babelrc -d .jsdocbuild src && jsdoc -r .jsdocbuild -P package.json -R README.md -d .jsdoc",
-    "start": "npm run start:init && npm run start:watch",
-    "start:watch": "babel -s -w --skip-initial-build -d lib src",
-    "start:init": "babel -s -d lib src",
+    "prepublishOnly": "yarn build",
+    "start": "echo THIS IS FOR LEGACY PURPOSES ONLY. && babel src -w -s -d lib --verbose --extensions \".ts,.js\"",
+    "dist": "echo 'This is for the release script so it can make assets (browser bundle).' && yarn build",
     "clean": "rimraf lib dist",
-    "build": "babel -s -d lib src && rimraf dist && mkdir dist && browserify -d browser-index.js | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js && uglifyjs -c -m -o dist/browser-matrix.min.js --source-map dist/browser-matrix.min.js.map --in-source-map dist/browser-matrix.js.map dist/browser-matrix.js",
-    "dist": "npm run build",
-    "watch": "watchify -d browser-index.js -o 'exorcist dist/browser-matrix.js.map > dist/browser-matrix.js' -v",
-    "lint": "eslint --max-warnings 102 src spec",
-    "prepublish": "npm run clean && npm run build && git rev-parse HEAD > git-revision.txt"
+    "build": "yarn build:dev && yarn build:compile-browser && yarn build:minify-browser",
+    "build:dev": "yarn clean && git rev-parse HEAD > git-revision.txt && yarn build:compile && yarn build:types",
+    "build:types": "tsc --emitDeclarationOnly",
+    "build:compile": "babel -d lib --verbose --extensions \".ts,.js\" src",
+    "build:compile-browser": "mkdirp dist && browserify -d src/browser-index.js -p [ tsify -p ./tsconfig.json ] -t [ babelify --sourceMaps=inline --presets [ @babel/preset-env @babel/preset-typescript ] ] | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js",
+    "build:minify-browser": "terser dist/browser-matrix.js --compress --mangle --source-map --output dist/browser-matrix.min.js",
+    "gendoc": "jsdoc -c jsdoc.json -P package.json",
+    "lint": "yarn lint:types && yarn lint:js",
+    "lint:js": "eslint --max-warnings 73 src spec",
+    "lint:types": "tsc --noEmit",
+    "test": "jest spec/ --coverage --testEnvironment node",
+    "test:watch": "jest spec/ --coverage --testEnvironment node --watch"
   },
   "repository": {
     "type": "git",
@@ -27,68 +27,74 @@
   "keywords": [
     "matrix-org"
   ],
-  "browser": "browser-index.js",
+  "main": "./lib/index.js",
+  "browser": "./lib/browser-index.js",
+  "matrix_src_main": "./src/index.ts",
+  "matrix_src_browser": "./src/browser-index.js",
+  "matrix_lib_main": "./lib/index.js",
+  "matrix_lib_typings": "./lib/index.d.ts",
   "author": "matrix.org",
   "license": "Apache-2.0",
   "files": [
-    ".babelrc",
-    ".eslintrc.js",
-    "spec/.eslintrc.js",
+    "dist",
+    "lib",
+    "src",
+    "git-revision.txt",
     "CHANGELOG.md",
     "CONTRIBUTING.rst",
     "LICENSE",
     "README.md",
-    "RELEASING.md",
-    "examples",
-    "git-hooks",
-    "git-revision.txt",
-    "index.js",
-    "browser-index.js",
-    "jenkins.sh",
-    "lib",
     "package.json",
-    "release.sh",
-    "spec",
-    "src"
+    "release.sh"
   ],
   "dependencies": {
+    "@babel/runtime": "^7.12.5",
     "another-json": "^0.2.0",
-    "babel-runtime": "^6.26.0",
-    "bluebird": "^3.5.0",
     "browser-request": "^0.3.3",
     "bs58": "^4.0.1",
-    "content-type": "^1.0.2",
-    "loglevel": "1.6.1",
-    "qs": "^6.5.2",
-    "request": "^2.88.0"
+    "content-type": "^1.0.4",
+    "loglevel": "^1.7.1",
+    "qs": "^6.9.6",
+    "request": "^2.88.2",
+    "unhomoglyph": "^1.0.6"
   },
   "devDependencies": {
-    "babel-cli": "^6.18.0",
-    "babel-eslint": "^8.1.1",
-    "babel-plugin-transform-async-to-bluebird": "^1.1.1",
-    "babel-plugin-transform-runtime": "^6.23.0",
-    "babel-preset-es2015": "^6.18.0",
-    "browserify": "^16.2.3",
-    "browserify-shim": "^3.8.13",
-    "eslint": "^3.13.1",
-    "eslint-config-google": "^0.7.1",
-    "exorcist": "^0.4.0",
-    "expect": "^1.20.2",
-    "istanbul": "^0.4.5",
-    "jsdoc": "^3.5.5",
-    "lolex": "^1.5.2",
-    "matrix-mock-request": "^1.2.2",
-    "mocha": "^5.2.0",
-    "mocha-jenkins-reporter": "^0.4.0",
-    "rimraf": "^2.5.4",
-    "source-map-support": "^0.4.11",
-    "sourceify": "^0.1.0",
-    "uglify-js": "^2.8.26",
-    "watchify": "^3.11.0"
+    "@babel/cli": "^7.12.10",
+    "@babel/core": "^7.12.10",
+    "@babel/plugin-proposal-class-properties": "^7.12.1",
+    "@babel/plugin-proposal-numeric-separator": "^7.12.7",
+    "@babel/plugin-proposal-object-rest-spread": "^7.12.1",
+    "@babel/plugin-syntax-dynamic-import": "^7.8.3",
+    "@babel/plugin-transform-runtime": "^7.12.10",
+    "@babel/preset-env": "^7.12.11",
+    "@babel/preset-typescript": "^7.12.7",
+    "@babel/register": "^7.12.10",
+    "@types/jest": "^26.0.20",
+    "@types/node": "12",
+    "@types/request": "^2.48.5",
+    "babel-eslint": "^10.1.0",
+    "babel-jest": "^26.6.3",
+    "babelify": "^10.0.0",
+    "better-docs": "^2.3.2",
+    "browserify": "^17.0.0",
+    "docdash": "^1.2.0",
+    "eslint": "7.18.0",
+    "eslint-config-matrix-org": "^0.2.0",
+    "eslint-plugin-babel": "^5.3.1",
+    "exorcist": "^1.0.1",
+    "fake-indexeddb": "^3.1.2",
+    "jest": "^26.6.3",
+    "jest-localstorage-mock": "^2.4.6",
+    "jsdoc": "^3.6.6",
+    "matrix-mock-request": "^1.2.3",
+    "olm": "https://packages.matrix.org/npm/olm/olm-3.2.1.tgz",
+    "rimraf": "^3.0.2",
+    "terser": "^5.5.1",
+    "tsify": "^5.0.2",
+    "typescript": "^4.1.3"
   },
-  "browserify": {
-    "transform": [
-      "sourceify"
-    ]
-  }
+  "jest": {
+    "testEnvironment": "node"
+  },
+  "typings": "./lib/index.d.ts"
 }

release.sh

@@ -1,17 +1,33 @@
 #!/bin/bash
 #
-# Script to perform a release of matrix-js-sdk.
+# Script to perform a release of matrix-js-sdk and downstream projects.
 #
 # Requires:
 #   github-changelog-generator; install via:
 #     pip install git+https://github.com/matrix-org/github-changelog-generator.git
 #   jq; install from your distribution's package manager (https://stedolan.github.io/jq/)
-#   hub; install via brew (OSX) or source/pre-compiled binaries (debian) (https://github.com/github/hub) - Tested on v2.2.9
+#   hub; install via brew (macOS) or source/pre-compiled binaries (debian) (https://github.com/github/hub) - Tested on v2.2.9
+#   npm; typically installed by Node.js
+#   yarn; install via brew (macOS) or similar (https://yarnpkg.com/docs/install/)
+#
+# Note: this script is also used to release matrix-react-sdk and element-web.
 
 set -e
 
 jq --version > /dev/null || (echo "jq is required: please install it"; kill $$)
-hub --version > /dev/null || (echo "hub is required: please install it"; kill $$)
+if [[ `command -v hub` ]] && [[ `hub --version` =~ hub[[:space:]]version[[:space:]]([0-9]*).([0-9]*) ]]; then
+    HUB_VERSION_MAJOR=${BASH_REMATCH[1]}
+    HUB_VERSION_MINOR=${BASH_REMATCH[2]}
+    if [[ $HUB_VERSION_MAJOR -lt 2 ]] || [[ $HUB_VERSION_MAJOR -eq 2 && $HUB_VERSION_MINOR -lt 5 ]]; then
+        echo "hub version 2.5 is required, you have $HUB_VERSION_MAJOR.$HUB_VERSION_MINOR installed"
+        exit
+    fi
+else
+    echo "hub is required: please install it"
+    exit
+fi
+npm --version > /dev/null || (echo "npm is required: please install it"; kill $$)
+yarn --version > /dev/null || (echo "yarn is required: please install it"; kill $$)
 
 USAGE="$0 [-xz] [-c changelog_file] vX.Y.Z"
 
@@ -22,6 +38,7 @@ $USAGE
     -c changelog_file:  specify name of file containing changelog
     -x:                 skip updating the changelog
     -z:                 skip generating the jsdoc
+    -n:                 skip publish to NPM
 EOF
 }
 
@@ -44,8 +61,10 @@ fi
 
 skip_changelog=
 skip_jsdoc=
+skip_npm=
 changelog_file="CHANGELOG.md"
-while getopts hc:xz f; do
+expected_npm_user="matrixdotorg"
+while getopts hc:u:xzn f; do
     case $f in
         h)
             help
@@ -60,6 +79,12 @@ while getopts hc:xz f; do
         z)
             skip_jsdoc=1
             ;;
+        n)
+            skip_npm=1
+            ;;
+        u)
+            expected_npm_user="$OPTARG"
+            ;;
     esac
 done
 shift `expr $OPTIND - 1`
@@ -69,11 +94,29 @@ if [ $# -ne 1 ]; then
     exit 1
 fi
 
+# We use Git branch / commit dependencies for some packages, and Yarn seems
+# to have a hard time getting that right. See also
+# https://github.com/yarnpkg/yarn/issues/4734. As a workaround, we clean the
+# global cache here to ensure we get the right thing.
+yarn cache clean
+# Ensure all dependencies are updated
+yarn install --ignore-scripts
+
 if [ -z "$skip_changelog" ]; then
     # update_changelog doesn't have a --version flag
     update_changelog -h > /dev/null || (echo "github-changelog-generator is required: please install it"; exit)
 fi
 
+# Login and publish continues to use `npm`, as it seems to have more clearly
+# defined options and semantics than `yarn` for writing to the registry.
+if [ -z "$skip_npm" ]; then
+    actual_npm_user=`npm whoami`;
+    if [ $expected_npm_user != $actual_npm_user ]; then
+        echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
+        exit 1
+    fi
+fi
+
 # ignore leading v on release
 release="${1#v}"
 tag="v${release}"
@@ -127,14 +170,35 @@ cat "${changelog_file}" | `dirname $0`/scripts/changelog_head.py > "${latest_cha
 set -x
 
 # Bump package.json and build the dist
-echo "npm version"
-# npm version will automatically commit its modification
+echo "yarn version"
+# yarn version will automatically commit its modification
 # and make a release tag. We don't want it to create the tag
 # because it can only sign with the default key, but we can
 # only turn off both of these behaviours, so we have to
 # manually commit the result.
-npm version --no-git-tag-version "$release"
-git commit package.json -m "$tag"
+yarn version --no-git-tag-version --new-version "$release"
+
+# For the published and dist versions of the package, we copy the
+# `matrix_lib_main` and `matrix_lib_typings` fields to `main` and `typings` (if
+# they exist). This small bit of gymnastics allows us to use the TypeScript
+# source directly for development without needing to build before linting or
+# testing.
+for i in main typings
+do
+    lib_value=$(jq -r ".matrix_lib_$i" package.json)
+    if [ "$lib_value" != "null" ]; then
+        jq ".$i = .matrix_lib_$i" package.json > package.json.new && mv package.json.new package.json
+    fi
+done
+
+# commit yarn.lock if it exists, is versioned, and is modified
+if [[ -f yarn.lock && `git status --porcelain yarn.lock | grep '^ M'` ]];
+then
+    pkglock='yarn.lock'
+else
+    pkglock=''
+fi
+git commit package.json $pkglock -m "$tag"
 
 
 # figure out if we should be signing this release
@@ -150,7 +214,7 @@ fi
 # assets.
 # We make a completely separate checkout to be sure
 # we're using released versions of the dependencies
-# (rather than whatever we're pulling in from npm link)
+# (rather than whatever we're pulling in from yarn link)
 assets=''
 dodist=0
 jq -e .scripts.dist package.json 2> /dev/null || dodist=$?
@@ -161,10 +225,10 @@ if [ $dodist -eq 0 ]; then
     pushd "$builddir"
     git clone "$projdir" .
     git checkout "$rel_branch"
-    npm install
+    yarn install
     # We haven't tagged yet, so tell the dist script what version
     # it's building
-    DIST_VERSION="$tag" npm run dist
+    DIST_VERSION="$tag" yarn dist
 
     popd
 
@@ -253,12 +317,22 @@ fi
 rm "${release_text}"
 rm "${latest_changes}"
 
-# publish to npmjs
-npm publish
+# Login and publish continues to use `npm`, as it seems to have more clearly
+# defined options and semantics than `yarn` for writing to the registry.
+# Tag both releases and prereleases as `next` so the last stable release remains
+# the default.
+if [ -z "$skip_npm" ]; then
+    npm publish --tag next
+    if [ $prerelease -eq 0 ]; then
+        # For a release, also add the default `latest` tag.
+        package=$(cat package.json | jq -er .name)
+        npm dist-tag add "$package@$release" latest
+    fi
+fi
 
 if [ -z "$skip_jsdoc" ]; then
     echo "generating jsdocs"
-    npm run gendoc
+    yarn gendoc
 
     echo "copying jsdocs to gh-pages branch"
     git checkout gh-pages
@@ -269,6 +343,7 @@ if [ -z "$skip_jsdoc" ]; then
         $release index.html
     git add "$release"
     git commit --no-verify -m "Add jsdoc for $release" index.html "$release"
+    git push origin gh-pages
 fi
 
 # if it is a pre-release, leave it on the release branch for now.
@@ -281,16 +356,40 @@ fi
 echo "updating master branch"
 git checkout master
 git pull
-git merge "$rel_branch"
+git merge "$rel_branch" --no-edit
 
-# push master  and docs (if generated) to github
+# push master to github
 git push origin master
-if [ -z "$skip_jsdoc" ]; then
-    git push origin gh-pages
-fi
 
-# finally, merge master back onto develop
-git checkout develop
-git pull
-git merge master
-git push origin develop
+# finally, merge master back onto develop (if it exists)
+if [ $(git branch -lr | grep origin/develop -c) -ge 1 ]; then
+    git checkout develop
+    git pull
+    git merge master --no-edit
+
+    # When merging to develop, we need revert the `main` and `typings` fields if
+    # we adjusted them previously.
+    for i in main typings
+    do
+        # If a `lib` prefixed value is present, it means we adjusted the field
+        # earlier at publish time, so we should revert it now.
+        if [ "$(jq -r ".matrix_lib_$i" package.json)" != "null" ]; then
+            # If there's a `src` prefixed value, use that, otherwise delete.
+            # This is used to delete the `typings` field and reset `main` back
+            # to the TypeScript source.
+            src_value=$(jq -r ".matrix_src_$i" package.json)
+            if [ "$src_value" != "null" ]; then
+                jq ".$i = .matrix_src_$i" package.json > package.json.new && mv package.json.new package.json
+            else
+                jq "del(.$i)" package.json > package.json.new && mv package.json.new package.json
+            fi
+        fi
+    done
+
+    if [ -n "$(git ls-files --modified package.json)" ]; then
+        echo "Committing develop package.json"
+        git commit package.json -m "Resetting package fields for development"
+    fi
+
+    git push origin develop
+fi

spec/.eslintrc.js

@@ -1,5 +0,0 @@
-module.exports = {
-    env: {
-        mocha: true,
-    },
-}

spec/MockStorageApi.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,7 +19,7 @@ limitations under the License.
  * A mock implementation of the webstorage api
  * @constructor
  */
-function MockStorageApi() {
+export function MockStorageApi() {
     this.data = {};
     this.keys = [];
     this.length = 0;
@@ -52,5 +53,3 @@ MockStorageApi.prototype = {
     },
 };
 
-/** */
-module.exports = MockStorageApi;

spec/TestClient.js

@@ -1,7 +1,7 @@
 /*
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
-Copyright 2018 New Vector Ltd
+Copyright 2018-2019 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,17 +16,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-"use strict";
-
 // load olm before the sdk if possible
 import './olm-loader';
 
-import sdk from '..';
-import testUtils from './test-utils';
 import MockHttpBackend from 'matrix-mock-request';
-import expect from 'expect';
-import Promise from 'bluebird';
-import LocalStorageCryptoStore from '../lib/crypto/store/localStorage-crypto-store';
+import {LocalStorageCryptoStore} from '../src/crypto/store/localStorage-crypto-store';
+import {logger} from '../src/logger';
+import {WebStorageSessionStore} from "../src/store/session/webstorage";
+import {syncPromise} from "./test-utils";
+import {createClient} from "../src/matrix";
+import {MockStorageApi} from "./MockStorageApi";
 
 /**
  * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
@@ -38,34 +37,41 @@ import LocalStorageCryptoStore from '../lib/crypto/store/localStorage-crypto-sto
  *
  * @param {WebStorage=} sessionStoreBackend a web storage object to use for the
  *     session store. If undefined, we will create a MockStorageApi.
+ * @param {object} options additional options to pass to the client
  */
-export default function TestClient(
-    userId, deviceId, accessToken, sessionStoreBackend,
+export function TestClient(
+    userId, deviceId, accessToken, sessionStoreBackend, options,
 ) {
     this.userId = userId;
     this.deviceId = deviceId;
 
     if (sessionStoreBackend === undefined) {
-        sessionStoreBackend = new testUtils.MockStorageApi();
+        sessionStoreBackend = new MockStorageApi();
     }
-    const sessionStore = new sdk.WebStorageSessionStore(sessionStoreBackend);
-
-    // expose this so the tests can get to it
-    this.cryptoStore = new LocalStorageCryptoStore(sessionStoreBackend);
+    const sessionStore = new WebStorageSessionStore(sessionStoreBackend);
 
     this.httpBackend = new MockHttpBackend();
-    this.client = sdk.createClient({
+
+    options = Object.assign({
         baseUrl: "http://" + userId + ".test.server",
         userId: userId,
         accessToken: accessToken,
         deviceId: deviceId,
         sessionStore: sessionStore,
-        cryptoStore: this.cryptoStore,
         request: this.httpBackend.requestFn,
-    });
+    }, options);
+    if (!options.cryptoStore) {
+        // expose this so the tests can get to it
+        this.cryptoStore = new LocalStorageCryptoStore(sessionStoreBackend);
+        options.cryptoStore = this.cryptoStore;
+    }
+    this.client = createClient(options);
 
     this.deviceKeys = null;
     this.oneTimeKeys = {};
+    this._callEventHandler = {
+        calls: new Map(),
+    };
 }
 
 TestClient.prototype.toString = function() {
@@ -78,7 +84,7 @@ TestClient.prototype.toString = function() {
  * @return {Promise}
  */
 TestClient.prototype.start = function() {
-    console.log(this + ': starting');
+    logger.log(this + ': starting');
     this.httpBackend.when("GET", "/pushrules").respond(200, {});
     this.httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
     this.expectDeviceKeyUpload();
@@ -94,9 +100,9 @@ TestClient.prototype.start = function() {
 
     return Promise.all([
         this.httpBackend.flushAllExpected(),
-        testUtils.syncPromise(this.client),
+        syncPromise(this.client),
     ]).then(() => {
-        console.log(this + ': started');
+        logger.log(this + ': started');
     });
 };
 
@@ -118,7 +124,7 @@ TestClient.prototype.expectDeviceKeyUpload = function() {
         expect(content.one_time_keys).toBe(undefined);
         expect(content.device_keys).toBeTruthy();
 
-        console.log(self + ': received device keys');
+        logger.log(self + ': received device keys');
         // we expect this to happen before any one-time keys are uploaded.
         expect(Object.keys(self.oneTimeKeys).length).toEqual(0);
 
@@ -154,8 +160,8 @@ TestClient.prototype.awaitOneTimeKeyUpload = function() {
           .respond(200, (path, content) => {
               expect(content.device_keys).toBe(undefined);
               expect(content.one_time_keys).toBeTruthy();
-              expect(content.one_time_keys).toNotEqual({});
-              console.log('%s: received %i one-time keys', this,
+              expect(content.one_time_keys).not.toEqual({});
+              logger.log('%s: received %i one-time keys', this,
                           Object.keys(content.one_time_keys).length);
               this.oneTimeKeys = content.one_time_keys;
               return {one_time_key_counts: {
@@ -181,7 +187,11 @@ TestClient.prototype.expectKeyQuery = function(response) {
     this.httpBackend.when('POST', '/keys/query').respond(
         200, (path, content) => {
             Object.keys(response.device_keys).forEach((userId) => {
-                expect(content.device_keys[userId]).toEqual({});
+                expect(content.device_keys[userId]).toEqual(
+                    [],
+                    "Expected key query for " + userId + ", got " +
+                    Object.keys(content.device_keys),
+                );
             });
             return response;
         });
@@ -215,11 +225,15 @@ TestClient.prototype.getSigningKey = function() {
  * @returns {Promise} promise which completes once the sync has been flushed
  */
 TestClient.prototype.flushSync = function() {
-    console.log(`${this}: flushSync`);
+    logger.log(`${this}: flushSync`);
     return Promise.all([
         this.httpBackend.flush('/sync', 1),
-        testUtils.syncPromise(this.client),
+        syncPromise(this.client),
     ]).then(() => {
-        console.log(`${this}: flushSync completed`);
+        logger.log(`${this}: flushSync completed`);
     });
 };
+
+TestClient.prototype.isFallbackICEServerAllowed = function() {
+    return true;
+};

spec/browserify/setupTests.js

@@ -0,0 +1,23 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// stub for browser-matrix browserify tests
+global.XMLHttpRequest = jest.fn();
+
+afterAll(() => {
+    // clean up XMLHttpRequest mock
+    global.XMLHttpRequest = undefined;
+});

spec/browserify/sync-browserify.spec.js

@@ -0,0 +1,103 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// load XmlHttpRequest mock
+import "./setupTests";
+import "../../dist/browser-matrix"; // uses browser-matrix instead of the src
+import {MockStorageApi} from "../MockStorageApi";
+import {WebStorageSessionStore} from "../../src/store/session/webstorage";
+import MockHttpBackend from "matrix-mock-request";
+import {LocalStorageCryptoStore} from "../../src/crypto/store/localStorage-crypto-store";
+import * as utils from "../test-utils";
+
+const USER_ID = "@user:test.server";
+const DEVICE_ID = "device_id";
+const ACCESS_TOKEN = "access_token";
+const ROOM_ID = "!room_id:server.test";
+
+/* global matrixcs */
+
+describe("Browserify Test", function() {
+    let client;
+    let httpBackend;
+
+    async function createTestClient() {
+        const sessionStoreBackend = new MockStorageApi();
+        const sessionStore = new WebStorageSessionStore(sessionStoreBackend);
+        const httpBackend = new MockHttpBackend();
+
+        const options = {
+            baseUrl: "http://" + USER_ID + ".test.server",
+            userId: USER_ID,
+            accessToken: ACCESS_TOKEN,
+            deviceId: DEVICE_ID,
+            sessionStore: sessionStore,
+            request: httpBackend.requestFn,
+            cryptoStore: new LocalStorageCryptoStore(sessionStoreBackend),
+        };
+
+        const client = matrixcs.createClient(options);
+
+        httpBackend.when("GET", "/pushrules").respond(200, {});
+        httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+
+        return { client, httpBackend };
+    }
+
+    beforeEach(async () => {
+        ({client, httpBackend} = await createTestClient());
+        await client.startClient();
+    });
+
+    afterEach(async () => {
+        client.stopClient();
+        await httpBackend.stop();
+    });
+
+    it("Sync", async function() {
+        const event = utils.mkMembership({
+            room: ROOM_ID,
+            mship: "join",
+            user: "@other_user:server.test",
+            name: "Displayname",
+        });
+
+        const syncData = {
+            next_batch: "batch1",
+            rooms: {
+                join: {},
+            },
+        };
+        syncData.rooms.join[ROOM_ID] = {
+            timeline: {
+                events: [
+                    event,
+                ],
+                limited: false,
+            },
+        };
+
+        httpBackend.when("GET", "/sync").respond(200, syncData);
+        await Promise.race([
+            Promise.all([
+                httpBackend.flushAllExpected(),
+            ]),
+            new Promise((_, reject) => {
+                client.once("sync.unexpectedError", reject);
+            }),
+        ]);
+    }, 20000); // additional timeout as this test can take quite a while
+});

spec/integ/devicelist-integ-spec.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2017 Vector Creations Ltd
 Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,11 +16,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import expect from 'expect';
-import Promise from 'bluebird';
-
-import TestClient from '../TestClient';
-import testUtils from '../test-utils';
+import {TestClient} from '../TestClient';
+import * as testUtils from '../test-utils';
+import {logger} from '../../src/logger';
 
 const ROOM_ID = "!room:id";
 
@@ -71,7 +70,7 @@ function getSyncResponse(roomMembers) {
 
 describe("DeviceList management:", function() {
     if (!global.Olm) {
-        console.warn('not running deviceList tests: Olm not present');
+        logger.warn('not running deviceList tests: Olm not present');
         return;
     }
 
@@ -87,8 +86,6 @@ describe("DeviceList management:", function() {
     }
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
-
         // we create our own sessionStoreBackend so that we can use it for
         // another TestClient.
         sessionStoreBackend = new testUtils.MockStorageApi();
@@ -101,13 +98,14 @@ describe("DeviceList management:", function() {
     });
 
     it("Alice shouldn't do a second /query for non-e2e-capable devices", function() {
+        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
         return aliceTestClient.start().then(function() {
             const syncResponse = getSyncResponse(['@bob:xyz']);
             aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
 
             return aliceTestClient.flushSync();
         }).then(function() {
-            console.log("Forcing alice to download our device keys");
+            logger.log("Forcing alice to download our device keys");
 
             aliceTestClient.httpBackend.when('POST', '/keys/query').respond(200, {
                 device_keys: {
@@ -120,7 +118,7 @@ describe("DeviceList management:", function() {
                 aliceTestClient.httpBackend.flush('/keys/query', 1),
             ]);
         }).then(function() {
-            console.log("Telling alice to send a megolm message");
+            logger.log("Telling alice to send a megolm message");
 
             aliceTestClient.httpBackend.when(
                 'PUT', '/send/',
@@ -142,7 +140,8 @@ describe("DeviceList management:", function() {
 
     it("We should not get confused by out-of-order device query responses",
        () => {
-           // https://github.com/vector-im/riot-web/issues/3126
+           // https://github.com/vector-im/element-web/issues/3126
+           aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
            return aliceTestClient.start().then(() => {
                aliceTestClient.httpBackend.when('GET', '/sync').respond(
                    200, getSyncResponse(['@bob:xyz', '@chris:abc']));
@@ -272,7 +271,7 @@ describe("DeviceList management:", function() {
            });
        }).timeout(3000);
 
-    // https://github.com/vector-im/riot-web/issues/4983
+    // https://github.com/vector-im/element-web/issues/4983
     describe("Alice should know she has stale device lists", () => {
         beforeEach(async function() {
             await aliceTestClient.start();

spec/integ/matrix-client-crypto.spec.js

@@ -2,6 +2,7 @@
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
 Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -24,18 +25,14 @@ limitations under the License.
  * See also `megolm.spec.js`.
  */
 
-"use strict";
-import 'source-map-support/register';
-
 // load olm before the sdk if possible
 import '../olm-loader';
 
-import expect from 'expect';
-const sdk = require("../..");
-import Promise from 'bluebird';
-const utils = require("../../lib/utils");
-const testUtils = require("../test-utils");
-const TestClient = require('../TestClient').default;
+import {logger} from '../../src/logger';
+import * as testUtils from "../test-utils";
+import * as utils from "../../src/utils";
+import {TestClient} from "../TestClient";
+import {CRYPTO_ENABLED} from "../../src/client";
 
 let aliTestClient;
 const roomId = "!room:localhost";
@@ -55,7 +52,7 @@ function bobUploadsDeviceKeys() {
         bobTestClient.client.uploadKeys(),
         bobTestClient.httpBackend.flush(),
     ]).then(() => {
-        expect(Object.keys(bobTestClient.deviceKeys).length).toNotEqual(0);
+        expect(Object.keys(bobTestClient.deviceKeys).length).not.toEqual(0);
     });
 }
 
@@ -72,7 +69,11 @@ function expectAliQueryKeys() {
     bobKeys[bobDeviceId] = bobTestClient.deviceKeys;
     aliTestClient.httpBackend.when("POST", "/keys/query")
             .respond(200, function(path, content) {
-        expect(content.device_keys[bobUserId]).toEqual({});
+        expect(content.device_keys[bobUserId]).toEqual(
+            [],
+            "Expected Alice to key query for " + bobUserId + ", got " +
+            Object.keys(content.device_keys),
+        );
         const result = {};
         result[bobUserId] = bobKeys;
         return {device_keys: result};
@@ -91,12 +92,16 @@ function expectBobQueryKeys() {
 
     const aliKeys = {};
     aliKeys[aliDeviceId] = aliTestClient.deviceKeys;
-    console.log("query result will be", aliKeys);
+    logger.log("query result will be", aliKeys);
 
     bobTestClient.httpBackend.when(
         "POST", "/keys/query",
     ).respond(200, function(path, content) {
-        expect(content.device_keys[aliUserId]).toEqual({});
+        expect(content.device_keys[aliUserId]).toEqual(
+            [],
+            "Expected Bob to key query for " + aliUserId + ", got " +
+            Object.keys(content.device_keys),
+        );
         const result = {};
         result[aliUserId] = aliKeys;
         return {device_keys: result};
@@ -195,7 +200,7 @@ function aliSendsFirstMessage() {
         expectAliQueryKeys()
             .then(expectAliClaimKeys)
             .then(expectAliSendMessageRequest),
-    ]).spread(function(_, ciphertext) {
+    ]).then(function([_, ciphertext]) {
         return ciphertext;
     });
 }
@@ -210,7 +215,7 @@ function aliSendsMessage() {
     return Promise.all([
         sendMessage(aliTestClient.client),
         expectAliSendMessageRequest(),
-    ]).spread(function(_, ciphertext) {
+    ]).then(function([_, ciphertext]) {
         return ciphertext;
     });
 }
@@ -226,7 +231,7 @@ function bobSendsReplyMessage() {
         sendMessage(bobTestClient.client),
         expectBobQueryKeys()
             .then(expectBobSendMessageRequest),
-    ]).spread(function(_, ciphertext) {
+    ]).then(function([_, ciphertext]) {
         return ciphertext;
     });
 }
@@ -271,16 +276,17 @@ function sendMessage(client) {
 
 function expectSendMessageRequest(httpBackend) {
     const path = "/send/m.room.encrypted/";
-    const deferred = Promise.defer();
-    httpBackend.when("PUT", path).respond(200, function(path, content) {
-        deferred.resolve(content);
-        return {
-            event_id: "asdfgh",
-        };
+    const prom = new Promise((resolve) => {
+        httpBackend.when("PUT", path).respond(200, function(path, content) {
+            resolve(content);
+            return {
+                event_id: "asdfgh",
+            };
+        });
     });
 
     // it can take a while to process the key query
-    return httpBackend.flush(path, 1).then(() => deferred.promise);
+    return httpBackend.flush(path, 1).then(() => prom);
 }
 
 function aliRecvMessage() {
@@ -326,7 +332,7 @@ function recvMessage(httpBackend, client, sender, message) {
             if (event.getType() == "m.room.member") {
                 return;
             }
-            console.log(client.credentials.userId + " received event",
+            logger.log(client.credentials.userId + " received event",
                         event);
 
             client.removeListener("event", onEvent);
@@ -392,13 +398,11 @@ function firstSync(testClient) {
 
 
 describe("MatrixClient crypto", function() {
-    if (!sdk.CRYPTO_ENABLED) {
+    if (!CRYPTO_ENABLED) {
         return;
     }
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
-
         aliTestClient = new TestClient(aliUserId, aliDeviceId, aliAccessToken);
         await aliTestClient.client.initCrypto();
 
@@ -421,15 +425,14 @@ describe("MatrixClient crypto", function() {
             .then(bobUploadsDeviceKeys);
     });
 
-    it("Ali downloads Bobs device keys", function(done) {
-        Promise.resolve()
+    it("Ali downloads Bobs device keys", function() {
+        return Promise.resolve()
             .then(bobUploadsDeviceKeys)
-            .then(aliDownloadsKeys)
-            .nodeify(done);
+            .then(aliDownloadsKeys);
     });
 
-    it("Ali gets keys with an invalid signature", function(done) {
-        Promise.resolve()
+    it("Ali gets keys with an invalid signature", function() {
+        return Promise.resolve()
             .then(bobUploadsDeviceKeys)
             .then(function() {
                 // tamper bob's keys
@@ -446,11 +449,10 @@ describe("MatrixClient crypto", function() {
             }).then((devices) => {
                 // should get an empty list
                 expect(devices).toEqual([]);
-            })
-            .nodeify(done);
+            });
     });
 
-    it("Ali gets keys with an incorrect userId", function(done) {
+    it("Ali gets keys with an incorrect userId", function() {
         const eveUserId = "@eve:localhost";
 
         const bobDeviceKeys = {
@@ -479,7 +481,7 @@ describe("MatrixClient crypto", function() {
             return {device_keys: result};
         });
 
-        Promise.all([
+        return Promise.all([
             aliTestClient.client.downloadKeys([bobUserId, eveUserId]),
             aliTestClient.httpBackend.flush("/keys/query", 1),
         ]).then(function() {
@@ -487,14 +489,14 @@ describe("MatrixClient crypto", function() {
                 aliTestClient.client.getStoredDevicesForUser(bobUserId),
                 aliTestClient.client.getStoredDevicesForUser(eveUserId),
             ]);
-        }).spread((bobDevices, eveDevices) => {
+        }).then(([bobDevices, eveDevices]) => {
             // should get an empty list
             expect(bobDevices).toEqual([]);
             expect(eveDevices).toEqual([]);
-        }).nodeify(done);
+        });
     });
 
-    it("Ali gets keys with an incorrect deviceId", function(done) {
+    it("Ali gets keys with an incorrect deviceId", function() {
         const bobDeviceKeys = {
             algorithms: ['m.olm.v1.curve25519-aes-sha2', 'm.megolm.v1.aes-sha2'],
             device_id: 'bad_device',
@@ -521,7 +523,7 @@ describe("MatrixClient crypto", function() {
             return {device_keys: result};
         });
 
-        Promise.all([
+        return Promise.all([
             aliTestClient.client.downloadKeys([bobUserId]),
             aliTestClient.httpBackend.flush("/keys/query", 1),
         ]).then(function() {
@@ -529,7 +531,7 @@ describe("MatrixClient crypto", function() {
         }).then((devices) => {
             // should get an empty list
             expect(devices).toEqual([]);
-        }).nodeify(done);
+        });
     });
 
 
@@ -539,21 +541,22 @@ describe("MatrixClient crypto", function() {
             .then(() => bobTestClient.awaitOneTimeKeyUpload())
             .then((keys) => {
                 expect(Object.keys(keys).length).toEqual(5);
-                expect(Object.keys(bobTestClient.deviceKeys).length).toNotEqual(0);
+                expect(Object.keys(bobTestClient.deviceKeys).length).not.toEqual(0);
             });
     });
 
-    it("Ali sends a message", function(done) {
-        Promise.resolve()
+    it("Ali sends a message", function() {
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
             .then(() => firstSync(aliTestClient))
             .then(aliEnablesEncryption)
-            .then(aliSendsFirstMessage)
-            .nodeify(done);
+            .then(aliSendsFirstMessage);
     });
 
     it("Bob receives a message", function() {
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
         return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
@@ -564,6 +567,7 @@ describe("MatrixClient crypto", function() {
     });
 
     it("Bob receives a message with a bogus sender", function() {
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
         return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
@@ -596,7 +600,7 @@ describe("MatrixClient crypto", function() {
 
                 const eventPromise = new Promise((resolve, reject) => {
                     const onEvent = function(event) {
-                        console.log(bobUserId + " received event",
+                        logger.log(bobUserId + " received event",
                                     event);
                         resolve(event);
                     };
@@ -616,8 +620,9 @@ describe("MatrixClient crypto", function() {
             });
     });
 
-    it("Ali blocks Bob's device", function(done) {
-        Promise.resolve()
+    it("Ali blocks Bob's device", function() {
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
             .then(() => firstSync(aliTestClient))
@@ -632,11 +637,12 @@ describe("MatrixClient crypto", function() {
                           expect(sentContent.ciphertext).toEqual({});
                       });
                 return Promise.all([p1, p2]);
-            }).nodeify(done);
+            });
     });
 
-    it("Bob receives two pre-key messages", function(done) {
-        Promise.resolve()
+    it("Bob receives two pre-key messages", function() {
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
             .then(() => firstSync(aliTestClient))
@@ -644,11 +650,12 @@ describe("MatrixClient crypto", function() {
             .then(aliSendsFirstMessage)
             .then(bobRecvMessage)
             .then(aliSendsMessage)
-            .then(bobRecvMessage)
-            .nodeify(done);
+            .then(bobRecvMessage);
     });
 
     it("Bob replies to the message", function() {
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        bobTestClient.expectKeyQuery({device_keys: {[bobUserId]: {}}});
         return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
@@ -659,13 +666,14 @@ describe("MatrixClient crypto", function() {
             .then(bobRecvMessage)
             .then(bobEnablesEncryption)
             .then(bobSendsReplyMessage).then(function(ciphertext) {
-                expect(ciphertext.type).toEqual(1);
+                expect(ciphertext.type).toEqual(1, "Unexpected cipghertext type.");
             }).then(aliRecvMessage);
     });
 
     it("Ali does a key query when encryption is enabled", function() {
         // enabling encryption in the room should make alice download devices
         // for both members.
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
         return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => firstSync(aliTestClient))
@@ -696,7 +704,6 @@ describe("MatrixClient crypto", function() {
             }).then(() => {
                 aliTestClient.expectKeyQuery({
                     device_keys: {
-                        [aliUserId]: {},
                         [bobUserId]: {},
                     },
                 });
@@ -719,7 +726,7 @@ describe("MatrixClient crypto", function() {
 
         return Promise.resolve()
             .then(() => {
-                console.log(aliTestClient + ': starting');
+                logger.log(aliTestClient + ': starting');
                 httpBackend.when("GET", "/pushrules").respond(200, {});
                 httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
                 aliTestClient.expectDeviceKeyUpload();
@@ -731,16 +738,16 @@ describe("MatrixClient crypto", function() {
                 aliTestClient.client.startClient({});
 
                 return httpBackend.flushAllExpected().then(() => {
-                    console.log(aliTestClient + ': started');
+                    logger.log(aliTestClient + ': started');
                 });
             })
             .then(() => httpBackend.when("POST", "/keys/upload")
                 .respond(200, (path, content) => {
                     expect(content.one_time_keys).toBeTruthy();
-                    expect(content.one_time_keys).toNotEqual({});
+                    expect(content.one_time_keys).not.toEqual({});
                     expect(Object.keys(content.one_time_keys).length)
-                        .toBeGreaterThanOrEqualTo(1);
-                    console.log('received %i one-time keys',
+                        .toBeGreaterThanOrEqual(1);
+                    logger.log('received %i one-time keys',
                                 Object.keys(content.one_time_keys).length);
                     // cancel futher calls by telling the client
                     // we have more than we need

spec/integ/matrix-client-event-emitter.spec.js

@@ -1,28 +1,16 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const HttpBackend = require("matrix-mock-request");
-const utils = require("../test-utils");
-
-import expect from 'expect';
-import Promise from 'bluebird';
+import * as utils from "../test-utils";
+import {TestClient} from "../TestClient";
 
 describe("MatrixClient events", function() {
-    const baseUrl = "http://localhost.or.something";
     let client;
     let httpBackend;
     const selfUserId = "@alice:localhost";
     const selfAccessToken = "aseukfgwef";
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        httpBackend = new HttpBackend();
-        sdk.request(httpBackend.requestFn);
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: selfUserId,
-            accessToken: selfAccessToken,
-        });
+        const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
+        client = testClient.client;
+        httpBackend = testClient.httpBackend;
         httpBackend.when("GET", "/pushrules").respond(200, {});
         httpBackend.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
     });
@@ -157,14 +145,14 @@ describe("MatrixClient events", function() {
                     return;
                 }
 
-                expect(event.event).toEqual(SYNC_DATA.presence.events[0]);
+                expect(event.event).toMatch(SYNC_DATA.presence.events[0]);
                 expect(user.presence).toEqual(
                     SYNC_DATA.presence.events[0].content.presence,
                 );
             });
             client.startClient();
 
-            httpBackend.flushAllExpected().done(function() {
+            httpBackend.flushAllExpected().then(function() {
                 expect(fired).toBe(true, "User.presence didn't fire.");
                 done();
             });
@@ -219,7 +207,7 @@ describe("MatrixClient events", function() {
             client.on("RoomState.events", function(event, state) {
                 eventsInvokeCount++;
                 const index = roomStateEventTypes.indexOf(event.getType());
-                expect(index).toNotEqual(
+                expect(index).not.toEqual(
                     -1, "Unexpected room state event type: " + event.getType(),
                 );
                 if (index >= 0) {
@@ -302,11 +290,32 @@ describe("MatrixClient events", function() {
         });
 
         it("should emit Session.logged_out on M_UNKNOWN_TOKEN", function() {
-            httpBackend.when("GET", "/sync").respond(401, { errcode: 'M_UNKNOWN_TOKEN' });
+            const error = { errcode: 'M_UNKNOWN_TOKEN' };
+            httpBackend.when("GET", "/sync").respond(401, error);
 
             let sessionLoggedOutCount = 0;
-            client.on("Session.logged_out", function(event, member) {
+            client.on("Session.logged_out", function(errObj) {
                 sessionLoggedOutCount++;
+                expect(errObj.data).toEqual(error);
+            });
+
+            client.startClient();
+
+            return httpBackend.flushAllExpected().then(function() {
+                expect(sessionLoggedOutCount).toEqual(
+                    1, "Session.logged_out fired wrong number of times",
+                );
+            });
+        });
+
+        it("should emit Session.logged_out on M_UNKNOWN_TOKEN (soft logout)", function() {
+            const error = { errcode: 'M_UNKNOWN_TOKEN', soft_logout: true };
+            httpBackend.when("GET", "/sync").respond(401, error);
+
+            let sessionLoggedOutCount = 0;
+            client.on("Session.logged_out", function(errObj) {
+                sessionLoggedOutCount++;
+                expect(errObj.data).toEqual(error);
             });
 
             client.startClient();

spec/integ/matrix-client-event-timeline.spec.js

@@ -1,12 +1,8 @@
-"use strict";
-import 'source-map-support/register';
-import Promise from 'bluebird';
-const sdk = require("../..");
-const HttpBackend = require("matrix-mock-request");
-const utils = require("../test-utils");
-const EventTimeline = sdk.EventTimeline;
+import * as utils from "../test-utils";
+import {EventTimeline} from "../../src/matrix";
+import {logger} from "../../src/logger";
+import {TestClient} from "../TestClient";
 
-const baseUrl = "http://localhost.or.something";
 const userId = "@alice:localhost";
 const userName = "Alice";
 const accessToken = "aseukfgwef";
@@ -82,18 +78,19 @@ function startClient(httpBackend, client) {
     client.startClient();
 
     // set up a promise which will resolve once the client is initialised
-    const deferred = Promise.defer();
-    client.on("sync", function(state) {
-        console.log("sync", state);
-        if (state != "SYNCING") {
-            return;
-        }
-        deferred.resolve();
+    const prom = new Promise((resolve) => {
+        client.on("sync", function(state) {
+            logger.log("sync", state);
+            if (state != "SYNCING") {
+                return;
+            }
+            resolve();
+        });
     });
 
     return Promise.all([
         httpBackend.flushAllExpected(),
-        deferred.promise,
+        prom,
     ]);
 }
 
@@ -102,9 +99,9 @@ describe("getEventTimeline support", function() {
     let client;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        httpBackend = new HttpBackend();
-        sdk.request(httpBackend.requestFn);
+        const testClient = new TestClient(userId, "DEVICE", accessToken);
+        client = testClient.client;
+        httpBackend = testClient.httpBackend;
     });
 
     afterEach(function() {
@@ -114,53 +111,44 @@ describe("getEventTimeline support", function() {
         return httpBackend.stop();
     });
 
-    it("timeline support must be enabled to work", function(done) {
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            accessToken: accessToken,
-        });
-
-        startClient(httpBackend, client,
-        ).then(function() {
+    it("timeline support must be enabled to work", function() {
+        return startClient(httpBackend, client).then(function() {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
             expect(function() {
                 client.getEventTimeline(timelineSet, "event");
             }).toThrow();
-        }).nodeify(done);
+        });
     });
 
     it("timeline support works when enabled", function() {
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            accessToken: accessToken,
-            timelineSupport: true,
-        });
+        const testClient = new TestClient(
+            userId,
+            "DEVICE",
+            accessToken,
+            undefined,
+            {timelineSupport: true},
+        );
+        client = testClient.client;
+        httpBackend = testClient.httpBackend;
 
         return startClient(httpBackend, client).then(() => {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
             expect(function() {
                 client.getEventTimeline(timelineSet, "event");
-            }).toNotThrow();
+            }).not.toThrow();
         });
     });
 
 
     it("scrollback should be able to scroll back to before a gappy /sync",
-      function(done) {
+      function() {
         // need a client with timelineSupport disabled to make this work
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            accessToken: accessToken,
-        });
+
         let room;
 
-        startClient(httpBackend, client,
-        ).then(function() {
+        return startClient(httpBackend, client).then(function() {
             room = client.getRoom(roomId);
 
             httpBackend.when("GET", "/sync").respond(200, {
@@ -216,27 +204,24 @@ describe("getEventTimeline support", function() {
             expect(room.timeline[0].event).toEqual(EVENTS[0]);
             expect(room.timeline[1].event).toEqual(EVENTS[1]);
             expect(room.oldState.paginationToken).toEqual("pagin_end");
-        }).nodeify(done);
+        });
     });
 });
 
-import expect from 'expect';
-
 describe("MatrixClient event timelines", function() {
     let client = null;
     let httpBackend = null;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        httpBackend = new HttpBackend();
-        sdk.request(httpBackend.requestFn);
-
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            accessToken: accessToken,
-            timelineSupport: true,
-        });
+        const testClient = new TestClient(
+            userId,
+            "DEVICE",
+            accessToken,
+            undefined,
+            {timelineSupport: true},
+        );
+        client = testClient.client;
+        httpBackend = testClient.httpBackend;
 
         return startClient(httpBackend, client);
     });
@@ -348,25 +333,25 @@ describe("MatrixClient event timelines", function() {
                     };
                 });
 
-            const deferred = Promise.defer();
-            client.on("sync", function() {
-                client.getEventTimeline(timelineSet, EVENTS[2].event_id,
-                ).then(function(tl) {
-                    expect(tl.getEvents().length).toEqual(4);
-                    expect(tl.getEvents()[0].event).toEqual(EVENTS[1]);
-                    expect(tl.getEvents()[1].event).toEqual(EVENTS[2]);
-                    expect(tl.getEvents()[3].event).toEqual(EVENTS[3]);
-                    expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
-                        .toEqual("start_token");
-                    // expect(tl.getPaginationToken(EventTimeline.FORWARDS))
-                    //    .toEqual("s_5_4");
-                }).done(() => deferred.resolve(),
-                        (e) => deferred.reject(e));
+            const prom = new Promise((resolve, reject) => {
+                client.on("sync", function() {
+                    client.getEventTimeline(timelineSet, EVENTS[2].event_id,
+                    ).then(function(tl) {
+                        expect(tl.getEvents().length).toEqual(4);
+                        expect(tl.getEvents()[0].event).toEqual(EVENTS[1]);
+                        expect(tl.getEvents()[1].event).toEqual(EVENTS[2]);
+                        expect(tl.getEvents()[3].event).toEqual(EVENTS[3]);
+                        expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                            .toEqual("start_token");
+                        // expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                        //    .toEqual("s_5_4");
+                    }).then(resolve, reject);
+                });
             });
 
             return Promise.all([
                 httpBackend.flushAllExpected(),
-                deferred.promise,
+                prom,
             ]);
         });
 
@@ -669,11 +654,11 @@ describe("MatrixClient event timelines", function() {
                 // initiate the send, and set up checks to be done when it completes
                 // - but note that it won't complete until after the /sync does, below.
                 client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
-                    console.log("sendTextMessage completed");
+                    logger.log("sendTextMessage completed");
                     expect(res.event_id).toEqual(event.event_id);
                     return client.getEventTimeline(timelineSet, event.event_id);
                 }).then(function(tl) {
-                    console.log("getEventTimeline completed (2)");
+                    logger.log("getEventTimeline completed (2)");
                     expect(tl.getEvents().length).toEqual(2);
                     expect(tl.getEvents()[1].getContent().body).toEqual("a body");
                 }),
@@ -684,7 +669,7 @@ describe("MatrixClient event timelines", function() {
                 ]).then(function() {
                     return client.getEventTimeline(timelineSet, event.event_id);
                 }).then(function(tl) {
-                    console.log("getEventTimeline completed (1)");
+                    logger.log("getEventTimeline completed (1)");
                     expect(tl.getEvents().length).toEqual(2);
                     expect(tl.getEvents()[1].event).toEqual(event);
 
@@ -696,7 +681,7 @@ describe("MatrixClient event timelines", function() {
     });
 
 
-    it("should handle gappy syncs after redactions", function(done) {
+    it("should handle gappy syncs after redactions", function() {
         // https://github.com/vector-im/vector-web/issues/1389
 
         // a state event, followed by a redaction thereof
@@ -728,7 +713,7 @@ describe("MatrixClient event timelines", function() {
         };
         httpBackend.when("GET", "/sync").respond(200, syncData);
 
-        Promise.all([
+        return Promise.all([
             httpBackend.flushAllExpected(),
             utils.syncPromise(client),
         ]).then(function() {
@@ -764,6 +749,6 @@ describe("MatrixClient event timelines", function() {
             const room = client.getRoom(roomId);
             const tl = room.getLiveTimeline();
             expect(tl.getEvents().length).toEqual(1);
-        }).nodeify(done);
+        });
     });
 });

spec/integ/matrix-client-methods.spec.js

@@ -1,42 +1,23 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const HttpBackend = require("matrix-mock-request");
-const publicGlobals = require("../../lib/matrix");
-const Room = publicGlobals.Room;
-const MatrixInMemoryStore = publicGlobals.MatrixInMemoryStore;
-const Filter = publicGlobals.Filter;
-const utils = require("../test-utils");
-const MockStorageApi = require("../MockStorageApi");
-
-import expect from 'expect';
+import * as utils from "../test-utils";
+import {CRYPTO_ENABLED} from "../../src/client";
+import {Filter, MemoryStore, Room} from "../../src/matrix";
+import {TestClient} from "../TestClient";
 
 describe("MatrixClient", function() {
-    const baseUrl = "http://localhost.or.something";
     let client = null;
     let httpBackend = null;
     let store = null;
-    let sessionStore = null;
     const userId = "@alice:localhost";
     const accessToken = "aseukfgwef";
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        httpBackend = new HttpBackend();
-        store = new MatrixInMemoryStore();
+        store = new MemoryStore();
 
-        const mockStorage = new MockStorageApi();
-        sessionStore = new sdk.WebStorageSessionStore(mockStorage);
-
-        sdk.request(httpBackend.requestFn);
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            deviceId: "aliceDevice",
-            accessToken: accessToken,
+        const testClient = new TestClient(userId, "aliceDevice", accessToken, undefined, {
             store: store,
-            sessionStore: sessionStore,
         });
+        httpBackend = testClient.httpBackend;
+        client = testClient.client;
     });
 
     afterEach(function() {
@@ -46,9 +27,9 @@ describe("MatrixClient", function() {
 
     describe("uploadContent", function() {
         const buf = new Buffer('hello world');
-        it("should upload the file", function(done) {
+        it("should upload the file", function() {
             httpBackend.when(
-                "POST", "/_matrix/media/v1/upload",
+                "POST", "/_matrix/media/r0/upload",
             ).check(function(req) {
                 expect(req.rawData).toEqual(buf);
                 expect(req.queryParams.filename).toEqual("hi.txt");
@@ -74,25 +55,26 @@ describe("MatrixClient", function() {
             expect(uploads[0].promise).toBe(prom);
             expect(uploads[0].loaded).toEqual(0);
 
-            prom.then(function(response) {
+            const prom2 = prom.then(function(response) {
                 // for backwards compatibility, we return the raw JSON
                 expect(response).toEqual("content");
 
                 const uploads = client.getCurrentUploads();
                 expect(uploads.length).toEqual(0);
-            }).nodeify(done);
+            });
 
             httpBackend.flush();
+            return prom2;
         });
 
-        it("should parse the response if rawResponse=false", function(done) {
+        it("should parse the response if rawResponse=false", function() {
             httpBackend.when(
-                "POST", "/_matrix/media/v1/upload",
+                "POST", "/_matrix/media/r0/upload",
             ).check(function(req) {
                 expect(req.opts.json).toBeFalsy();
             }).respond(200, { "content_uri": "uri" });
 
-            client.uploadContent({
+            const prom = client.uploadContent({
                 stream: buf,
                 name: "hi.txt",
                 type: "text/plain",
@@ -100,14 +82,15 @@ describe("MatrixClient", function() {
                 rawResponse: false,
             }).then(function(response) {
                 expect(response.content_uri).toEqual("uri");
-            }).nodeify(done);
+            });
 
             httpBackend.flush();
+            return prom;
         });
 
-        it("should parse errors into a MatrixError", function(done) {
+        it("should parse errors into a MatrixError", function() {
             httpBackend.when(
-                "POST", "/_matrix/media/v1/upload",
+                "POST", "/_matrix/media/r0/upload",
             ).check(function(req) {
                 expect(req.rawData).toEqual(buf);
                 expect(req.opts.json).toBeFalsy();
@@ -116,7 +99,7 @@ describe("MatrixClient", function() {
                 "error": "broken",
             });
 
-            client.uploadContent({
+            const prom = client.uploadContent({
                 stream: buf,
                 name: "hi.txt",
                 type: "text/plain",
@@ -126,12 +109,13 @@ describe("MatrixClient", function() {
                 expect(error.httpStatus).toEqual(400);
                 expect(error.errcode).toEqual("M_SNAFU");
                 expect(error.message).toEqual("broken");
-            }).nodeify(done);
+            });
 
             httpBackend.flush();
+            return prom;
         });
 
-        it("should return a promise which can be cancelled", function(done) {
+        it("should return a promise which can be cancelled", function() {
             const prom = client.uploadContent({
                 stream: buf,
                 name: "hi.txt",
@@ -143,17 +127,18 @@ describe("MatrixClient", function() {
             expect(uploads[0].promise).toBe(prom);
             expect(uploads[0].loaded).toEqual(0);
 
-            prom.then(function(response) {
+            const prom2 = prom.then(function(response) {
                 throw Error("request not aborted");
             }, function(error) {
                 expect(error).toEqual("aborted");
 
                 const uploads = client.getCurrentUploads();
                 expect(uploads.length).toEqual(0);
-            }).nodeify(done);
+            });
 
             const r = client.cancelUpload(prom);
             expect(r).toBe(true);
+            return prom2;
         });
     });
 
@@ -180,7 +165,7 @@ describe("MatrixClient", function() {
                 event_format: "client",
             });
             store.storeFilter(filter);
-            client.getFilter(userId, filterId, true).done(function(gotFilter) {
+            client.getFilter(userId, filterId, true).then(function(gotFilter) {
                 expect(gotFilter).toEqual(filter);
                 done();
             });
@@ -201,7 +186,7 @@ describe("MatrixClient", function() {
                 event_format: "client",
             });
             store.storeFilter(storeFilter);
-            client.getFilter(userId, filterId, false).done(function(gotFilter) {
+            client.getFilter(userId, filterId, false).then(function(gotFilter) {
                 expect(gotFilter.getDefinition()).toEqual(httpFilterDefinition);
                 done();
             });
@@ -219,7 +204,7 @@ describe("MatrixClient", function() {
             httpBackend.when(
                 "GET", "/user/" + encodeURIComponent(userId) + "/filter/" + filterId,
             ).respond(200, httpFilterDefinition);
-            client.getFilter(userId, filterId, true).done(function(gotFilter) {
+            client.getFilter(userId, filterId, true).then(function(gotFilter) {
                 expect(gotFilter.getDefinition()).toEqual(httpFilterDefinition);
                 expect(store.getFilter(userId, filterId)).toBeTruthy();
                 done();
@@ -247,7 +232,7 @@ describe("MatrixClient", function() {
                 filter_id: filterId,
             });
 
-            client.createFilter(filterDefinition).done(function(gotFilter) {
+            client.createFilter(filterDefinition).then(function(gotFilter) {
                 expect(gotFilter.getDefinition()).toEqual(filterDefinition);
                 expect(store.getFilter(userId, filterId)).toEqual(gotFilter);
                 done();
@@ -294,7 +279,7 @@ describe("MatrixClient", function() {
                 });
             }).respond(200, response);
 
-            httpBackend.flush().done(function() {
+            httpBackend.flush().then(function() {
                 done();
             });
         });
@@ -302,7 +287,7 @@ describe("MatrixClient", function() {
 
 
     describe("downloadKeys", function() {
-        if (!sdk.CRYPTO_ENABLED) {
+        if (!CRYPTO_ENABLED) {
             return;
         }
 
@@ -310,7 +295,7 @@ describe("MatrixClient", function() {
             return client.initCrypto();
         });
 
-        it("should do an HTTP request and then store the keys", function(done) {
+        it("should do an HTTP request and then store the keys", function() {
             const ed25519key = "7wG2lzAqbjcyEkOP7O4gU7ItYcn+chKzh5sT/5r2l78";
             // ed25519key = client.getDeviceEd25519Key();
             const borisKeys = {
@@ -355,15 +340,15 @@ describe("MatrixClient", function() {
                 return client._crypto._olmDevice.sign(anotherjson.stringify(b));
             };
 
-            console.log("Ed25519: " + ed25519key);
-            console.log("boris:", sign(borisKeys.dev1));
-            console.log("chaz:", sign(chazKeys.dev2));
+            logger.log("Ed25519: " + ed25519key);
+            logger.log("boris:", sign(borisKeys.dev1));
+            logger.log("chaz:", sign(chazKeys.dev2));
             */
 
             httpBackend.when("POST", "/keys/query").check(function(req) {
                 expect(req.data).toEqual({device_keys: {
-                    'boris': {},
-                    'chaz': {},
+                    'boris': [],
+                    'chaz': [],
                 }});
             }).respond(200, {
                 device_keys: {
@@ -372,7 +357,7 @@ describe("MatrixClient", function() {
                 },
             });
 
-            client.downloadKeys(["boris", "chaz"]).then(function(res) {
+            const prom = client.downloadKeys(["boris", "chaz"]).then(function(res) {
                 assertObjectContains(res.boris.dev1, {
                     verified: 0, // DeviceVerification.UNVERIFIED
                     keys: { "ed25519:dev1": ed25519key },
@@ -386,26 +371,26 @@ describe("MatrixClient", function() {
                     algorithms: ["2"],
                     unsigned: { "ghi": "def" },
                 });
-            }).nodeify(done);
+            });
 
             httpBackend.flush();
+            return prom;
         });
     });
 
     describe("deleteDevice", function() {
         const auth = {a: 1};
-        it("should pass through an auth dict", function(done) {
+        it("should pass through an auth dict", function() {
             httpBackend.when(
-                "DELETE", "/_matrix/client/unstable/devices/my_device",
+                "DELETE", "/_matrix/client/r0/devices/my_device",
             ).check(function(req) {
                 expect(req.data).toEqual({auth: auth});
             }).respond(200);
 
-            client.deleteDevice(
-                "my_device", auth,
-            ).nodeify(done);
+            const prom = client.deleteDevice("my_device", auth);
 
             httpBackend.flush();
+            return prom;
         });
     });
 });

spec/integ/matrix-client-opts.spec.js

@@ -1,12 +1,9 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const MatrixClient = sdk.MatrixClient;
-const HttpBackend = require("matrix-mock-request");
-const utils = require("../test-utils");
-
-import expect from 'expect';
-import Promise from 'bluebird';
+import * as utils from "../test-utils";
+import HttpBackend from "matrix-mock-request";
+import {MatrixClient} from "../../src/matrix";
+import {MatrixScheduler} from "../../src/scheduler";
+import {MemoryStore} from "../../src/store/memory";
+import {MatrixError} from "../../src/http-api";
 
 describe("MatrixClient opts", function() {
     const baseUrl = "http://localhost.or.something";
@@ -58,7 +55,6 @@ describe("MatrixClient opts", function() {
     };
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
         httpBackend = new HttpBackend();
     });
 
@@ -75,7 +71,7 @@ describe("MatrixClient opts", function() {
                 baseUrl: baseUrl,
                 userId: userId,
                 accessToken: accessToken,
-                scheduler: new sdk.MatrixScheduler(),
+                scheduler: new MatrixScheduler(),
             });
         });
 
@@ -88,7 +84,7 @@ describe("MatrixClient opts", function() {
             httpBackend.when("PUT", "/txn1").respond(200, {
                 event_id: eventId,
             });
-            client.sendTextMessage("!foo:bar", "a body", "txn1").done(function(res) {
+            client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                 expect(res.event_id).toEqual(eventId);
                 done();
             });
@@ -101,7 +97,7 @@ describe("MatrixClient opts", function() {
                 "m.room.create",
             ];
             client.on("event", function(event) {
-                expect(expectedEventTypes.indexOf(event.getType())).toNotEqual(
+                expect(expectedEventTypes.indexOf(event.getType())).not.toEqual(
                     -1, "Recv unexpected event type: " + event.getType(),
                 );
                 expectedEventTypes.splice(
@@ -128,7 +124,7 @@ describe("MatrixClient opts", function() {
         beforeEach(function() {
             client = new MatrixClient({
                 request: httpBackend.requestFn,
-                store: new sdk.MatrixInMemoryStore(),
+                store: new MemoryStore(),
                 baseUrl: baseUrl,
                 userId: userId,
                 accessToken: accessToken,
@@ -137,11 +133,11 @@ describe("MatrixClient opts", function() {
         });
 
         it("shouldn't retry sending events", function(done) {
-            httpBackend.when("PUT", "/txn1").fail(500, {
+            httpBackend.when("PUT", "/txn1").fail(500, new MatrixError({
                 errcode: "M_SOMETHING",
                 error: "Ruh roh",
-            });
-            client.sendTextMessage("!foo:bar", "a body", "txn1").done(function(res) {
+            }));
+            client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                 expect(false).toBe(true, "sendTextMessage resolved but shouldn't");
             }, function(err) {
                 expect(err.errcode).toEqual("M_SOMETHING");
@@ -159,16 +155,16 @@ describe("MatrixClient opts", function() {
             });
             let sentA = false;
             let sentB = false;
-            client.sendTextMessage("!foo:bar", "a body", "txn1").done(function(res) {
+            client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                 sentA = true;
                 expect(sentB).toBe(true);
             });
-            client.sendTextMessage("!foo:bar", "b body", "txn2").done(function(res) {
+            client.sendTextMessage("!foo:bar", "b body", "txn2").then(function(res) {
                 sentB = true;
                 expect(sentA).toBe(false);
             });
-            httpBackend.flush("/txn2", 1).done(function() {
-                httpBackend.flush("/txn1", 1).done(function() {
+            httpBackend.flush("/txn2", 1).then(function() {
+                httpBackend.flush("/txn1", 1).then(function() {
                     done();
                 });
             });
@@ -178,7 +174,7 @@ describe("MatrixClient opts", function() {
             httpBackend.when("PUT", "/txn1").respond(200, {
                 event_id: "foo",
             });
-            client.sendTextMessage("!foo:bar", "a body", "txn1").done(function(res) {
+            client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                 expect(res.event_id).toEqual("foo");
                 done();
             });

spec/integ/matrix-client-retrying.spec.js

@@ -1,16 +1,9 @@
-"use strict";
-import 'source-map-support/register';
-import Promise from 'bluebird';
-
-const sdk = require("../..");
-const HttpBackend = require("matrix-mock-request");
-const utils = require("../test-utils");
-const EventStatus = sdk.EventStatus;
-
-import expect from 'expect';
+import {EventStatus} from "../../src/matrix";
+import {MatrixScheduler} from "../../src/scheduler";
+import {Room} from "../../src/models/room";
+import {TestClient} from "../TestClient";
 
 describe("MatrixClient retrying", function() {
-    const baseUrl = "http://localhost.or.something";
     let client = null;
     let httpBackend = null;
     let scheduler;
@@ -20,17 +13,17 @@ describe("MatrixClient retrying", function() {
     let room;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        httpBackend = new HttpBackend();
-        sdk.request(httpBackend.requestFn);
-        scheduler = new sdk.MatrixScheduler();
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            accessToken: accessToken,
-            scheduler: scheduler,
-        });
-        room = new sdk.Room(roomId);
+        scheduler = new MatrixScheduler();
+        const testClient = new TestClient(
+            userId,
+            "DEVICE",
+            accessToken,
+            undefined,
+            {scheduler},
+        );
+        httpBackend = testClient.httpBackend;
+        client = testClient.client;
+        room = new Room(roomId);
         client.store.storeRoom(room);
     });
 
@@ -97,7 +90,7 @@ describe("MatrixClient retrying", function() {
         // wait for the localecho of ev1 to be updated
         const p3 = new Promise((resolve, reject) => {
             room.on("Room.localEchoUpdated", (ev0) => {
-                if(ev0 === ev1) {
+                if (ev0 === ev1) {
                     resolve();
                 }
             });

spec/integ/matrix-client-room-timeline.spec.js

@@ -1,15 +1,9 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const EventStatus = sdk.EventStatus;
-const HttpBackend = require("matrix-mock-request");
-const utils = require("../test-utils");
+import * as utils from "../test-utils";
+import {EventStatus} from "../../src/models/event";
+import {TestClient} from "../TestClient";
 
-import Promise from 'bluebird';
-import expect from 'expect';
 
 describe("MatrixClient room timelines", function() {
-    const baseUrl = "http://localhost.or.something";
     let client = null;
     let httpBackend = null;
     const userId = "@alice:localhost";
@@ -103,17 +97,18 @@ describe("MatrixClient room timelines", function() {
         });
     }
 
-    beforeEach(function(done) {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        httpBackend = new HttpBackend();
-        sdk.request(httpBackend.requestFn);
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            accessToken: accessToken,
-            // these tests should work with or without timelineSupport
-            timelineSupport: true,
-        });
+    beforeEach(function() {
+        // these tests should work with or without timelineSupport
+        const testClient = new TestClient(
+            userId,
+            "DEVICE",
+            accessToken,
+            undefined,
+            {timelineSupport: true},
+        );
+        httpBackend = testClient.httpBackend;
+        client = testClient.client;
+
         setNextSyncData();
         httpBackend.when("GET", "/pushrules").respond(200, {});
         httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
@@ -122,9 +117,9 @@ describe("MatrixClient room timelines", function() {
             return NEXT_SYNC_DATA;
         });
         client.startClient();
-        httpBackend.flush("/pushrules").then(function() {
+        return httpBackend.flush("/pushrules").then(function() {
             return httpBackend.flush("/filter");
-        }).nodeify(done);
+        });
     });
 
     afterEach(function() {
@@ -153,7 +148,7 @@ describe("MatrixClient room timelines", function() {
                 expect(member.userId).toEqual(userId);
                 expect(member.name).toEqual(userName);
 
-                httpBackend.flush("/sync", 1).done(function() {
+                httpBackend.flush("/sync", 1).then(function() {
                     done();
                 });
             });
@@ -179,10 +174,10 @@ describe("MatrixClient room timelines", function() {
                     return;
                 }
                 const room = client.getRoom(roomId);
-                client.sendTextMessage(roomId, "I am a fish", "txn1").done(
+                client.sendTextMessage(roomId, "I am a fish", "txn1").then(
                 function() {
                     expect(room.timeline[1].getId()).toEqual(eventId);
-                    httpBackend.flush("/sync", 1).done(function() {
+                    httpBackend.flush("/sync", 1).then(function() {
                         expect(room.timeline[1].getId()).toEqual(eventId);
                         done();
                     });
@@ -212,10 +207,10 @@ describe("MatrixClient room timelines", function() {
                 }
                 const room = client.getRoom(roomId);
                 const promise = client.sendTextMessage(roomId, "I am a fish", "txn1");
-                httpBackend.flush("/sync", 1).done(function() {
+                httpBackend.flush("/sync", 1).then(function() {
                     expect(room.timeline.length).toEqual(2);
                     httpBackend.flush("/txn1", 1);
-                    promise.done(function() {
+                    promise.then(function() {
                         expect(room.timeline.length).toEqual(2);
                         expect(room.timeline[1].getId()).toEqual(eventId);
                         done();
@@ -250,7 +245,7 @@ describe("MatrixClient room timelines", function() {
                 const room = client.getRoom(roomId);
                 expect(room.timeline.length).toEqual(1);
 
-                client.scrollback(room).done(function() {
+                client.scrollback(room).then(function() {
                     expect(room.timeline.length).toEqual(1);
                     expect(room.oldState.paginationToken).toBe(null);
 
@@ -314,7 +309,7 @@ describe("MatrixClient room timelines", function() {
                 // sync response
                 expect(room.timeline.length).toEqual(1);
 
-                client.scrollback(room).done(function() {
+                client.scrollback(room).then(function() {
                     expect(room.timeline.length).toEqual(5);
                     const joinMsg = room.timeline[0];
                     expect(joinMsg.sender.name).toEqual("Old Alice");
@@ -352,7 +347,7 @@ describe("MatrixClient room timelines", function() {
                 const room = client.getRoom(roomId);
                 expect(room.timeline.length).toEqual(1);
 
-                client.scrollback(room).done(function() {
+                client.scrollback(room).then(function() {
                     expect(room.timeline.length).toEqual(3);
                     expect(room.timeline[0].event).toEqual(sbEvents[1]);
                     expect(room.timeline[1].event).toEqual(sbEvents[0]);
@@ -383,11 +378,11 @@ describe("MatrixClient room timelines", function() {
                 const room = client.getRoom(roomId);
                 expect(room.oldState.paginationToken).toBeTruthy();
 
-                client.scrollback(room, 1).done(function() {
+                client.scrollback(room, 1).then(function() {
                     expect(room.oldState.paginationToken).toEqual(sbEndTok);
                 });
 
-                httpBackend.flush("/messages", 1).done(function() {
+                httpBackend.flush("/messages", 1).then(function() {
                     // still have a sync to flush
                     httpBackend.flush("/sync", 1).then(() => {
                         done();

spec/integ/matrix-client-syncing.spec.js

@@ -1,16 +1,9 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const HttpBackend = require("matrix-mock-request");
-const utils = require("../test-utils");
-const MatrixEvent = sdk.MatrixEvent;
-const EventTimeline = sdk.EventTimeline;
-
-import expect from 'expect';
-import Promise from 'bluebird';
+import {MatrixEvent} from "../../src/models/event";
+import {EventTimeline} from "../../src/models/event-timeline";
+import * as utils from "../test-utils";
+import {TestClient} from "../TestClient";
 
 describe("MatrixClient syncing", function() {
-    const baseUrl = "http://localhost.or.something";
     let client = null;
     let httpBackend = null;
     const selfUserId = "@alice:localhost";
@@ -23,14 +16,9 @@ describe("MatrixClient syncing", function() {
     const roomTwo = "!bar:localhost";
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        httpBackend = new HttpBackend();
-        sdk.request(httpBackend.requestFn);
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: selfUserId,
-            accessToken: selfAccessToken,
-        });
+        const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
+        httpBackend = testClient.httpBackend;
+        client = testClient.client;
         httpBackend.when("GET", "/pushrules").respond(200, {});
         httpBackend.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
     });
@@ -53,7 +41,7 @@ describe("MatrixClient syncing", function() {
 
             client.startClient();
 
-            httpBackend.flushAllExpected().done(function() {
+            httpBackend.flushAllExpected().then(function() {
                 done();
             });
         });
@@ -67,7 +55,7 @@ describe("MatrixClient syncing", function() {
 
             client.startClient();
 
-            httpBackend.flushAllExpected().done(function() {
+            httpBackend.flushAllExpected().then(function() {
                 done();
             });
         });
@@ -528,7 +516,7 @@ describe("MatrixClient syncing", function() {
                 awaitSyncEvent(),
             ]).then(function() {
                 const room = client.getRoom(roomTwo);
-                expect(room).toExist();
+                expect(room).toBeDefined();
                 const tok = room.getLiveTimeline()
                     .getPaginationToken(EventTimeline.BACKWARDS);
                 expect(tok).toEqual("roomtwotok");
@@ -693,12 +681,12 @@ describe("MatrixClient syncing", function() {
                             include_leave: true }});
             }).respond(200, { filter_id: "another_id" });
 
-            const defer = Promise.defer();
-
-            httpBackend.when("GET", "/sync").check(function(req) {
-                expect(req.queryParams.filter).toEqual("another_id");
-                defer.resolve();
-            }).respond(200, {});
+            const prom = new Promise((resolve) => {
+                httpBackend.when("GET", "/sync").check(function(req) {
+                    expect(req.queryParams.filter).toEqual("another_id");
+                    resolve();
+                }).respond(200, {});
+            });
 
             client.syncLeftRooms();
 
@@ -709,7 +697,7 @@ describe("MatrixClient syncing", function() {
                     // flush the syncs
                     return httpBackend.flushAllExpected();
                 }),
-                defer.promise,
+                prom,
             ]);
         });
 

spec/integ/megolm-integ.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2016 OpenMarket Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,15 +15,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-"use strict";
-
-const anotherjson = require('another-json');
-import Promise from 'bluebird';
-import expect from 'expect';
-
-const utils = require('../../lib/utils');
-const testUtils = require('../test-utils');
-const TestClient = require('../TestClient').default;
+import anotherjson from "another-json";
+import * as utils from "../../src/utils";
+import * as testUtils from "../test-utils";
+import {TestClient} from "../TestClient";
+import {logger} from "../../src/logger";
 
 const ROOM_ID = "!room:id";
 
@@ -203,7 +200,7 @@ function getSyncResponse(roomMembers) {
 
 describe("megolm", function() {
     if (!global.Olm) {
-        console.warn('not running megolm tests: Olm not present');
+        logger.warn('not running megolm tests: Olm not present');
         return;
     }
     const Olm = global.Olm;
@@ -282,8 +279,6 @@ describe("megolm", function() {
     }
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
-
         aliceTestClient = new TestClient(
             "@alice:localhost", "xzcvb", "akjgkrgjs",
         );
@@ -351,7 +346,7 @@ describe("megolm", function() {
     });
 
     it("Alice receives a megolm message before the session keys", function() {
-        // https://github.com/vector-im/riot-web/issues/2273
+        // https://github.com/vector-im/element-web/issues/2273
         let roomKeyEncrypted;
 
         return aliceTestClient.start().then(() => {
@@ -416,7 +411,7 @@ describe("megolm", function() {
 
             return new Promise((resolve, reject) => {
                 event.once('Event.decrypted', (ev) => {
-                    console.log(`${Date.now()} event ${event.getId()} now decrypted`);
+                    logger.log(`${Date.now()} event ${event.getId()} now decrypted`);
                     resolve(ev);
                 });
             });
@@ -499,6 +494,7 @@ describe("megolm", function() {
     it('Alice sends a megolm message', function() {
         let p2pSession;
 
+        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
         return aliceTestClient.start().then(() => {
             // establish an olm session with alice
             return createOlmSession(testOlmAccount, aliceTestClient);
@@ -554,7 +550,7 @@ describe("megolm", function() {
             ).respond(200, function(path, content) {
                 const ct = content.ciphertext;
                 const r = inboundGroupSession.decrypt(ct);
-                console.log('Decrypted received megolm message', r);
+                logger.log('Decrypted received megolm message', r);
 
                 expect(r.message_index).toEqual(0);
                 const decrypted = JSON.parse(r.plaintext);
@@ -581,6 +577,7 @@ describe("megolm", function() {
     });
 
     it("We shouldn't attempt to send to blocked devices", function() {
+        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
         return aliceTestClient.start().then(() => {
             // establish an olm session with alice
             return createOlmSession(testOlmAccount, aliceTestClient);
@@ -598,7 +595,7 @@ describe("megolm", function() {
 
             return aliceTestClient.flushSync();
         }).then(function() {
-            console.log('Forcing alice to download our device keys');
+            logger.log('Forcing alice to download our device keys');
 
             aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
                 200, getTestKeysQueryResponse('@bob:xyz'),
@@ -609,15 +606,18 @@ describe("megolm", function() {
                 aliceTestClient.httpBackend.flush('/keys/query', 1),
             ]);
         }).then(function() {
-            console.log('Telling alice to block our device');
+            logger.log('Telling alice to block our device');
             aliceTestClient.client.setDeviceBlocked('@bob:xyz', 'DEVICE_ID');
 
-            console.log('Telling alice to send a megolm message');
+            logger.log('Telling alice to send a megolm message');
             aliceTestClient.httpBackend.when(
                 'PUT', '/send/',
             ).respond(200, {
                 event_id: '$event_id',
             });
+            aliceTestClient.httpBackend.when(
+                'PUT', '/sendToDevice/org.matrix.room_key.withheld/',
+            ).respond(200, {});
 
             return Promise.all([
                 aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
@@ -634,6 +634,7 @@ describe("megolm", function() {
         let p2pSession;
         let megolmSessionId;
 
+        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
         return aliceTestClient.start().then(() => {
             // establish an olm session with alice
             return createOlmSession(testOlmAccount, aliceTestClient);
@@ -653,7 +654,7 @@ describe("megolm", function() {
 
             return aliceTestClient.flushSync();
         }).then(function() {
-            console.log("Fetching bob's devices and marking known");
+            logger.log("Fetching bob's devices and marking known");
 
             aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
                 200, getTestKeysQueryResponse('@bob:xyz'),
@@ -666,17 +667,17 @@ describe("megolm", function() {
                 aliceTestClient.client.setDeviceKnown('@bob:xyz', 'DEVICE_ID');
             });
         }).then(function() {
-            console.log('Telling alice to send a megolm message');
+            logger.log('Telling alice to send a megolm message');
 
             aliceTestClient.httpBackend.when(
                 'PUT', '/sendToDevice/m.room.encrypted/',
             ).respond(200, function(path, content) {
-                console.log('sendToDevice: ', content);
+                logger.log('sendToDevice: ', content);
                 const m = content.messages['@bob:xyz'].DEVICE_ID;
                 const ct = m.ciphertext[testSenderKey];
                 expect(ct.type).toEqual(1); // normal message
                 const decrypted = JSON.parse(p2pSession.decrypt(ct.type, ct.body));
-                console.log('decrypted sendToDevice:', decrypted);
+                logger.log('decrypted sendToDevice:', decrypted);
                 expect(decrypted.type).toEqual('m.room_key');
                 megolmSessionId = decrypted.content.session_id;
                 return {};
@@ -685,7 +686,7 @@ describe("megolm", function() {
             aliceTestClient.httpBackend.when(
                 'PUT', '/send/',
             ).respond(200, function(path, content) {
-                console.log('/send:', content);
+                logger.log('/send:', content);
                 expect(content.session_id).toEqual(megolmSessionId);
                 return {
                     event_id: '$event_id',
@@ -701,19 +702,22 @@ describe("megolm", function() {
                 }),
             ]);
         }).then(function() {
-            console.log('Telling alice to block our device');
+            logger.log('Telling alice to block our device');
             aliceTestClient.client.setDeviceBlocked('@bob:xyz', 'DEVICE_ID');
 
-            console.log('Telling alice to send another megolm message');
+            logger.log('Telling alice to send another megolm message');
             aliceTestClient.httpBackend.when(
                 'PUT', '/send/',
             ).respond(200, function(path, content) {
-                console.log('/send:', content);
-                expect(content.session_id).toNotEqual(megolmSessionId);
+                logger.log('/send:', content);
+                expect(content.session_id).not.toEqual(megolmSessionId);
                 return {
                     event_id: '$event_id',
                 };
             });
+            aliceTestClient.httpBackend.when(
+                'PUT', '/sendToDevice/org.matrix.room_key.withheld/',
+            ).respond(200, {});
 
             return Promise.all([
                 aliceTestClient.client.sendTextMessage(ROOM_ID, 'test2'),
@@ -722,7 +726,7 @@ describe("megolm", function() {
         });
     });
 
-    // https://github.com/vector-im/riot-web/issues/2676
+    // https://github.com/vector-im/element-web/issues/2676
     it("Alice should send to her other devices", function() {
         // for this test, we make the testOlmAccount be another of Alice's devices.
         // it ought to get included in messages Alice sends.
@@ -789,7 +793,7 @@ describe("megolm", function() {
             aliceTestClient.httpBackend.when(
                 'PUT', '/sendToDevice/m.room.encrypted/',
             ).respond(200, function(path, content) {
-                console.log("sendToDevice: ", content);
+                logger.log("sendToDevice: ", content);
                 const m = content.messages[aliceTestClient.userId].DEVICE_ID;
                 const ct = m.ciphertext[testSenderKey];
                 expect(ct.type).toEqual(0); // pre-key message
@@ -809,7 +813,7 @@ describe("megolm", function() {
             ).respond(200, function(path, content) {
                 const ct = content.ciphertext;
                 const r = inboundGroupSession.decrypt(ct);
-                console.log('Decrypted received megolm message', r);
+                logger.log('Decrypted received megolm message', r);
                 decrypted = JSON.parse(r.plaintext);
 
                 return {
@@ -843,6 +847,7 @@ describe("megolm", function() {
         let downloadPromise;
         let sendPromise;
 
+        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
         return aliceTestClient.start().then(() => {
             // establish an olm session with alice
             return createOlmSession(testOlmAccount, aliceTestClient);
@@ -861,7 +866,7 @@ describe("megolm", function() {
             return aliceTestClient.flushSync();
         }).then(function() {
             // this will block
-            console.log('Forcing alice to download our device keys');
+            logger.log('Forcing alice to download our device keys');
             downloadPromise = aliceTestClient.client.downloadKeys(['@bob:xyz']);
 
             // so will this.
@@ -886,6 +891,7 @@ describe("megolm", function() {
     it("Alice exports megolm keys and imports them to a new device", function() {
         let messageEncrypted;
 
+        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
         return aliceTestClient.start().then(() => {
             // establish an olm session with alice
             return createOlmSession(testOlmAccount, aliceTestClient);
@@ -965,4 +971,60 @@ describe("megolm", function() {
             expect(event.getContent().body).toEqual('42');
         });
     });
+
+    it("Alice receives an untrusted megolm key, only to receive the trusted one shortly after", function() {
+        const testClient = new TestClient(
+            "@alice:localhost", "device2", "access_token2",
+        );
+        const groupSession = new Olm.OutboundGroupSession();
+        groupSession.create();
+        const inboundGroupSession = new Olm.InboundGroupSession();
+        inboundGroupSession.create(groupSession.session_key());
+        const rawEvent = encryptMegolmEvent({
+            senderKey: testSenderKey,
+            groupSession: groupSession,
+            room_id: ROOM_ID,
+        });
+        return testClient.client.initCrypto().then(() => {
+            const keys = [{
+                room_id: ROOM_ID,
+                algorithm: 'm.megolm.v1.aes-sha2',
+                session_id: groupSession.session_id(),
+                session_key: inboundGroupSession.export_session(0),
+                sender_key: testSenderKey,
+            }];
+            return testClient.client.importRoomKeys(keys, { untrusted: true });
+        }).then(() => {
+            const event = testUtils.mkEvent({
+                event: true,
+                ...rawEvent,
+                room: ROOM_ID,
+            });
+            return event.attemptDecryption(testClient.client._crypto, true).then(() => {
+                expect(event.isKeySourceUntrusted()).toBeTruthy();
+            });
+        }).then(() => {
+            const event = testUtils.mkEvent({
+                type: 'm.room_key',
+                content: {
+                    room_id: ROOM_ID,
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    session_id: groupSession.session_id(),
+                    session_key: groupSession.session_key(),
+                },
+                event: true,
+            });
+            event._senderCurve25519Key = testSenderKey;
+            return testClient.client._crypto._onRoomKeyEvent(event);
+        }).then(() => {
+            const event = testUtils.mkEvent({
+                event: true,
+                ...rawEvent,
+                room: ROOM_ID,
+            });
+            return event.attemptDecryption(testClient.client._crypto, true).then(() => {
+                expect(event.isKeySourceUntrusted()).toBeFalsy();
+            });
+        });
+    });
 });

spec/olm-loader.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector creations Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,11 +15,21 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// try to load the olm library.
+import {logger} from '../src/logger';
+import * as utils from "../src/utils";
 
+// try to load the olm library.
 try {
     global.Olm = require('olm');
-    console.log('loaded libolm');
+    logger.log('loaded libolm');
 } catch (e) {
-    console.warn("unable to run crypto tests: libolm not available");
+    logger.warn("unable to run crypto tests: libolm not available");
+}
+
+// also try to set node crypto
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    logger.log('nodejs was compiled without crypto support: some tests will fail');
 }

spec/test-utils.js

@@ -1,12 +1,8 @@
-"use strict";
-import expect from 'expect';
-import Promise from 'bluebird';
-
 // load olm before the sdk if possible
 import './olm-loader';
 
-import sdk from '..';
-const MatrixEvent = sdk.MatrixEvent;
+import {logger} from '../src/logger';
+import {MatrixEvent} from "../src/models/event";
 
 /**
  * Return a promise that is resolved when the client next emits a
@@ -15,7 +11,7 @@ const MatrixEvent = sdk.MatrixEvent;
  * @param {Number=} count Number of syncs to wait for (default 1)
  * @return {Promise} Resolves once the client has emitted a SYNCING event
  */
-module.exports.syncPromise = function(client, count) {
+export function syncPromise(client, count) {
     if (count === undefined) {
         count = 1;
     }
@@ -25,8 +21,8 @@ module.exports.syncPromise = function(client, count) {
 
     const p = new Promise((resolve, reject) => {
         const cb = (state) => {
-            console.log(`${Date.now()} syncPromise(${count}): ${state}`);
-            if (state == 'SYNCING') {
+            logger.log(`${Date.now()} syncPromise(${count}): ${state}`);
+            if (state === 'SYNCING') {
                 resolve();
             } else {
                 client.once('sync', cb);
@@ -36,21 +32,9 @@ module.exports.syncPromise = function(client, count) {
     });
 
     return p.then(() => {
-        return module.exports.syncPromise(client, count-1);
+        return syncPromise(client, count-1);
     });
-};
-
-/**
- * Perform common actions before each test case, e.g. printing the test case
- * name to stdout.
- * @param {Mocha.Context} context  The test context
- */
-module.exports.beforeEach = function(context) {
-    const desc = context.currentTest.fullTitle();
-
-    console.log(desc);
-    console.log(new Array(1 + desc.length).join("="));
-};
+}
 
 /**
  * Create a spy for an object and automatically spy its methods.
@@ -58,7 +42,7 @@ module.exports.beforeEach = function(context) {
  * @param {string} name The name of the class
  * @return {Object} An instantiated object with spied methods/properties.
  */
-module.exports.mock = function(constr, name) {
+export function mock(constr, name) {
     // Based on
     // http://eclipsesource.com/blogs/2014/03/27/mocks-in-jasmine-tests/
     const HelperConstr = new Function(); // jshint ignore:line
@@ -70,7 +54,7 @@ module.exports.mock = function(constr, name) {
     for (const key in constr.prototype) { // eslint-disable-line guard-for-in
         try {
             if (constr.prototype[key] instanceof Function) {
-                result[key] = expect.createSpy();
+                result[key] = jest.fn();
             }
         } catch (ex) {
             // Direct access to some non-function fields of DOM prototypes may
@@ -79,7 +63,7 @@ module.exports.mock = function(constr, name) {
         }
     }
     return result;
-};
+}
 
 /**
  * Create an Event.
@@ -92,7 +76,7 @@ module.exports.mock = function(constr, name) {
  * @param {boolean} opts.event True to make a MatrixEvent.
  * @return {Object} a JSON object representing this event.
  */
-module.exports.mkEvent = function(opts) {
+export function mkEvent(opts) {
     if (!opts.type || !opts.content) {
         throw new Error("Missing .type or .content =>" + JSON.stringify(opts));
     }
@@ -111,14 +95,14 @@ module.exports.mkEvent = function(opts) {
         event.state_key = "";
     }
     return opts.event ? new MatrixEvent(event) : event;
-};
+}
 
 /**
  * Create an m.presence event.
  * @param {Object} opts Values for the presence.
  * @return {Object|MatrixEvent} The event
  */
-module.exports.mkPresence = function(opts) {
+export function mkPresence(opts) {
     if (!opts.user) {
         throw new Error("Missing user");
     }
@@ -134,7 +118,7 @@ module.exports.mkPresence = function(opts) {
         },
     };
     return opts.event ? new MatrixEvent(event) : event;
-};
+}
 
 /**
  * Create an m.room.member event.
@@ -149,7 +133,7 @@ module.exports.mkPresence = function(opts) {
  * @param {boolean} opts.event True to make a MatrixEvent.
  * @return {Object|MatrixEvent} The event
  */
-module.exports.mkMembership = function(opts) {
+export function mkMembership(opts) {
     opts.type = "m.room.member";
     if (!opts.skey) {
         opts.skey = opts.sender || opts.user;
@@ -166,8 +150,8 @@ module.exports.mkMembership = function(opts) {
     if (opts.url) {
         opts.content.avatar_url = opts.url;
     }
-    return module.exports.mkEvent(opts);
-};
+    return mkEvent(opts);
+}
 
 /**
  * Create an m.room.message event.
@@ -178,7 +162,7 @@ module.exports.mkMembership = function(opts) {
  * @param {boolean} opts.event True to make a MatrixEvent.
  * @return {Object|MatrixEvent} The event
  */
-module.exports.mkMessage = function(opts) {
+export function mkMessage(opts) {
     opts.type = "m.room.message";
     if (!opts.msg) {
         opts.msg = "Random->" + Math.random();
@@ -190,8 +174,8 @@ module.exports.mkMessage = function(opts) {
         msgtype: "m.text",
         body: opts.msg,
     };
-    return module.exports.mkEvent(opts);
-};
+    return mkEvent(opts);
+}
 
 
 /**
@@ -199,10 +183,10 @@ module.exports.mkMessage = function(opts) {
  *
  * @constructor
  */
-module.exports.MockStorageApi = function() {
+export function MockStorageApi() {
     this.data = {};
-};
-module.exports.MockStorageApi.prototype = {
+}
+MockStorageApi.prototype = {
     get length() {
         return Object.keys(this.data).length;
     },
@@ -227,17 +211,158 @@ module.exports.MockStorageApi.prototype = {
  * @param {MatrixEvent} event
  * @returns {Promise} promise which resolves (to `event`) when the event has been decrypted
  */
-module.exports.awaitDecryption = function(event) {
+export function awaitDecryption(event) {
     if (!event.isBeingDecrypted()) {
         return Promise.resolve(event);
     }
 
-    console.log(`${Date.now()} event ${event.getId()} is being decrypted; waiting`);
+    logger.log(`${Date.now()} event ${event.getId()} is being decrypted; waiting`);
 
     return new Promise((resolve, reject) => {
         event.once('Event.decrypted', (ev) => {
-            console.log(`${Date.now()} event ${event.getId()} now decrypted`);
+            logger.log(`${Date.now()} event ${event.getId()} now decrypted`);
             resolve(ev);
         });
     });
+}
+
+
+export function HttpResponse(
+    httpLookups, acceptKeepalives, ignoreUnhandledSync,
+) {
+    this.httpLookups = httpLookups;
+    this.acceptKeepalives = acceptKeepalives === undefined ? true : acceptKeepalives;
+    this.ignoreUnhandledSync = ignoreUnhandledSync;
+    this.pendingLookup = null;
+}
+
+HttpResponse.prototype.request = function(
+    cb, method, path, qp, data, prefix,
+) {
+    if (path === HttpResponse.KEEP_ALIVE_PATH && this.acceptKeepalives) {
+        return Promise.resolve();
+    }
+    const next = this.httpLookups.shift();
+    const logLine = (
+        "MatrixClient[UT] RECV " + method + " " + path + "  " +
+            "EXPECT " + (next ? next.method : next) + " " + (next ? next.path : next)
+    );
+    logger.log(logLine);
+
+    if (!next) { // no more things to return
+        if (method === "GET" && path === "/sync" && this.ignoreUnhandledSync) {
+            logger.log("MatrixClient[UT] Ignoring.");
+            return new Promise(() => {});
+        }
+        if (this.pendingLookup) {
+            if (this.pendingLookup.method === method
+                && this.pendingLookup.path === path) {
+                return this.pendingLookup.promise;
+            }
+            // >1 pending thing, and they are different, whine.
+            expect(false).toBe(
+                true, ">1 pending request. You should probably handle them. " +
+                    "PENDING: " + JSON.stringify(this.pendingLookup) + " JUST GOT: " +
+                    method + " " + path,
+            );
+        }
+        this.pendingLookup = {
+            promise: new Promise(() => {}),
+            method: method,
+            path: path,
+        };
+        return this.pendingLookup.promise;
+    }
+    if (next.path === path && next.method === method) {
+        logger.log(
+            "MatrixClient[UT] Matched. Returning " +
+                (next.error ? "BAD" : "GOOD") + " response",
+        );
+        if (next.expectBody) {
+            expect(next.expectBody).toEqual(data);
+        }
+        if (next.expectQueryParams) {
+            Object.keys(next.expectQueryParams).forEach(function(k) {
+                expect(qp[k]).toEqual(next.expectQueryParams[k]);
+            });
+        }
+
+        if (next.thenCall) {
+            process.nextTick(next.thenCall, 0); // next tick so we return first.
+        }
+
+        if (next.error) {
+            return Promise.reject({
+                errcode: next.error.errcode,
+                httpStatus: next.error.httpStatus,
+                name: next.error.errcode,
+                message: "Expected testing error",
+                data: next.error,
+            });
+        }
+        return Promise.resolve(next.data);
+    } else if (method === "GET" && path === "/sync" && this.ignoreUnhandledSync) {
+        logger.log("MatrixClient[UT] Ignoring.");
+        this.httpLookups.unshift(next);
+        return new Promise(() => {});
+    }
+    expect(true).toBe(false, "Expected different request. " + logLine);
+    return new Promise(() => {});
 };
+
+HttpResponse.KEEP_ALIVE_PATH = "/_matrix/client/versions";
+
+HttpResponse.PUSH_RULES_RESPONSE = {
+    method: "GET",
+    path: "/pushrules/",
+    data: {},
+};
+
+HttpResponse.USER_ID = "@alice:bar";
+
+HttpResponse.filterResponse = function(userId) {
+    const filterPath = "/user/" + encodeURIComponent(userId) + "/filter";
+    return {
+        method: "POST",
+        path: filterPath,
+        data: { filter_id: "f1lt3r" },
+    };
+};
+
+HttpResponse.SYNC_DATA = {
+    next_batch: "s_5_3",
+    presence: { events: [] },
+    rooms: {},
+};
+
+HttpResponse.SYNC_RESPONSE = {
+    method: "GET",
+    path: "/sync",
+    data: HttpResponse.SYNC_DATA,
+};
+
+HttpResponse.defaultResponses = function(userId) {
+    return [
+        HttpResponse.PUSH_RULES_RESPONSE,
+        HttpResponse.filterResponse(userId),
+        HttpResponse.SYNC_RESPONSE,
+    ];
+};
+
+export function setHttpResponses(
+    client, responses, acceptKeepalives, ignoreUnhandledSyncs,
+) {
+    const httpResponseObj = new HttpResponse(
+        responses, acceptKeepalives, ignoreUnhandledSyncs,
+    );
+
+    const httpReq = httpResponseObj.request.bind(httpResponseObj);
+    client._http = [
+        "authedRequest", "authedRequestWithPrefix", "getContentUri",
+        "request", "requestWithPrefix", "uploadContent",
+    ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
+    client._http.authedRequest.mockImplementation(httpReq);
+    client._http.authedRequestWithPrefix.mockImplementation(httpReq);
+    client._http.requestWithPrefix.mockImplementation(httpReq);
+    client._http.request.mockImplementation(httpReq);
+}

spec/unit/ReEmitter.spec.ts

@@ -0,0 +1,72 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventEmitter } from "events";
+import { ReEmitter } from "../../src/ReEmitter";
+
+const EVENTNAME = "UnknownEntry";
+
+class EventSource extends EventEmitter {
+    doTheThing() {
+        this.emit(EVENTNAME, "foo", "bar");
+    }
+
+    doAnError() {
+        this.emit('error');
+    }
+}
+
+class EventTarget extends EventEmitter {
+
+}
+
+describe("ReEmitter", function() {
+    it("Re-Emits events with the same args", function() {
+        const src = new EventSource();
+        const tgt = new EventTarget();
+
+        const handler = jest.fn();
+        tgt.on(EVENTNAME, handler);
+
+        const reEmitter = new ReEmitter(tgt);
+        reEmitter.reEmit(src, [EVENTNAME]);
+
+        src.doTheThing();
+
+        // Args should be the args passed to 'emit' after the event name, and
+        // also the source object of the event which re-emitter adds
+        expect(handler).toHaveBeenCalledWith("foo", "bar", src);
+    });
+
+    it("Doesn't throw if no handler for 'error' event", function() {
+        const src = new EventSource();
+        const tgt = new EventTarget();
+
+        const reEmitter = new ReEmitter(tgt);
+        reEmitter.reEmit(src, ['error']);
+
+        // without the workaround in ReEmitter, this would throw
+        src.doAnError();
+
+        const handler = jest.fn();
+        tgt.on('error', handler);
+
+        src.doAnError();
+
+        // Now we've attached an error handler, it should be called
+        expect(handler).toHaveBeenCalled();
+    });
+});

spec/unit/autodiscovery.spec.js

@@ -0,0 +1,662 @@
+/*
+Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import MockHttpBackend from "matrix-mock-request";
+import * as sdk from "../../src";
+import {AutoDiscovery} from "../../src/autodiscovery";
+
+describe("AutoDiscovery", function() {
+    let httpBackend = null;
+
+    beforeEach(function() {
+        httpBackend = new MockHttpBackend();
+        sdk.request(httpBackend.requestFn);
+    });
+
+    it("should throw an error when no domain is specified", function() {
+        return Promise.all([
+            AutoDiscovery.findClientConfig(/* no args */).then(() => {
+                throw new Error("Expected a failure, not success with no args");
+            }, () => {
+                return true;
+            }),
+
+            AutoDiscovery.findClientConfig("").then(() => {
+                throw new Error("Expected a failure, not success with an empty string");
+            }, () => {
+                return true;
+            }),
+
+            AutoDiscovery.findClientConfig(null).then(() => {
+                throw new Error("Expected a failure, not success with null");
+            }, () => {
+                return true;
+            }),
+
+            AutoDiscovery.findClientConfig(true).then(() => {
+                throw new Error("Expected a failure, not success with a non-string");
+            }, () => {
+                return true;
+            }),
+        ]);
+    });
+
+    it("should return PROMPT when .well-known 404s", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(404, {});
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_PROMPT when .well-known returns a 500 error", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(500, {});
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_PROMPT",
+                        error: AutoDiscovery.ERROR_INVALID,
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_PROMPT when .well-known returns a 400 error", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(400, {});
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_PROMPT",
+                        error: AutoDiscovery.ERROR_INVALID,
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_PROMPT when .well-known returns an empty body", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, "");
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_PROMPT",
+                        error: AutoDiscovery.ERROR_INVALID,
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_PROMPT when .well-known returns not-JSON", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, "abc");
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_PROMPT",
+                        error: AutoDiscovery.ERROR_INVALID,
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_PROMPT when .well-known does not have a base_url for " +
+        "m.homeserver (empty string)", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                base_url: "",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_PROMPT",
+                        error: AutoDiscovery.ERROR_INVALID_HS_BASE_URL,
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_PROMPT when .well-known does not have a base_url for " +
+        "m.homeserver (no property)", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {},
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_PROMPT",
+                        error: AutoDiscovery.ERROR_INVALID_HS_BASE_URL,
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_ERROR when .well-known has an invalid base_url for " +
+        "m.homeserver (disallowed scheme)", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                base_url: "mxc://example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_ERROR",
+                        error: AutoDiscovery.ERROR_INVALID_HS_BASE_URL,
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_ERROR when .well-known has an invalid base_url for " +
+        "m.homeserver (verification failure: 404)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").respond(404, {});
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                base_url: "https://example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_ERROR",
+                        error: AutoDiscovery.ERROR_INVALID_HOMESERVER,
+                        base_url: "https://example.org",
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_ERROR when .well-known has an invalid base_url for " +
+        "m.homeserver (verification failure: 500)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").respond(500, {});
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                base_url: "https://example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_ERROR",
+                        error: AutoDiscovery.ERROR_INVALID_HOMESERVER,
+                        base_url: "https://example.org",
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_ERROR when .well-known has an invalid base_url for " +
+        "m.homeserver (verification failure: 200 but wrong content)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
+            not_matrix_versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                base_url: "https://example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_ERROR",
+                        error: AutoDiscovery.ERROR_INVALID_HOMESERVER,
+                        base_url: "https://example.org",
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return SUCCESS when .well-known has a verifiably accurate base_url for " +
+        "m.homeserver", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri).toEqual("https://example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                base_url: "https://example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "SUCCESS",
+                        error: null,
+                        base_url: "https://example.org",
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return SUCCESS with the right homeserver URL", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "SUCCESS",
+                        error: null,
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return SUCCESS / FAIL_PROMPT when the identity server configuration " +
+        "is wrong (missing base_url)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+            "m.identity_server": {
+                not_base_url: "https://identity.example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "SUCCESS",
+                        error: null,
+
+                        // We still expect the base_url to be here for debugging purposes.
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "FAIL_PROMPT",
+                        error: AutoDiscovery.ERROR_INVALID_IS_BASE_URL,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return SUCCESS / FAIL_PROMPT when the identity server configuration " +
+        "is wrong (empty base_url)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+            "m.identity_server": {
+                base_url: "",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "SUCCESS",
+                        error: null,
+
+                        // We still expect the base_url to be here for debugging purposes.
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "FAIL_PROMPT",
+                        error: AutoDiscovery.ERROR_INVALID_IS_BASE_URL,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return SUCCESS / FAIL_PROMPT when the identity server configuration " +
+        "is wrong (validation error: 404)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/_matrix/identity/api/v1").respond(404, {});
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+            "m.identity_server": {
+                base_url: "https://identity.example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "SUCCESS",
+                        error: null,
+
+                        // We still expect the base_url to be here for debugging purposes.
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "FAIL_PROMPT",
+                        error: AutoDiscovery.ERROR_INVALID_IDENTITY_SERVER,
+                        base_url: "https://identity.example.org",
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return SUCCESS / FAIL_PROMPT when the identity server configuration " +
+        "is wrong (validation error: 500)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/_matrix/identity/api/v1").respond(500, {});
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+            "m.identity_server": {
+                base_url: "https://identity.example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "SUCCESS",
+                        error: null,
+
+                        // We still expect the base_url to be here for debugging purposes
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "FAIL_PROMPT",
+                        error: AutoDiscovery.ERROR_INVALID_IDENTITY_SERVER,
+                        base_url: "https://identity.example.org",
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return SUCCESS when the identity server configuration is " +
+        "verifiably accurate", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/_matrix/identity/api/v1").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://identity.example.org/_matrix/identity/api/v1");
+        }).respond(200, {});
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+            "m.identity_server": {
+                base_url: "https://identity.example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "SUCCESS",
+                        error: null,
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "SUCCESS",
+                        error: null,
+                        base_url: "https://identity.example.org",
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return SUCCESS and preserve non-standard keys from the " +
+        ".well-known response", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/_matrix/identity/api/v1").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://identity.example.org/_matrix/identity/api/v1");
+        }).respond(200, {});
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+            "m.identity_server": {
+                base_url: "https://identity.example.org",
+            },
+            "org.example.custom.property": {
+                cupcakes: "yes",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "SUCCESS",
+                        error: null,
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "SUCCESS",
+                        error: null,
+                        base_url: "https://identity.example.org",
+                    },
+                    "org.example.custom.property": {
+                        cupcakes: "yes",
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+});

spec/unit/content-repo.spec.js

@@ -1,22 +1,13 @@
-"use strict";
-import 'source-map-support/register';
-const ContentRepo = require("../../lib/content-repo");
-const testUtils = require("../test-utils");
-
-import expect from 'expect';
+import {getHttpUriForMxc} from "../../src/content-repo";
 
 describe("ContentRepo", function() {
     const baseUrl = "https://my.home.server";
 
-    beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
-    });
-
     describe("getHttpUriForMxc", function() {
         it("should do nothing to HTTP URLs when allowing direct links", function() {
             const httpUrl = "http://example.com/image.jpeg";
             expect(
-                ContentRepo.getHttpUriForMxc(
+                getHttpUriForMxc(
                     baseUrl, httpUrl, undefined, undefined, undefined, true,
                 ),
             ).toEqual(httpUrl);
@@ -24,26 +15,26 @@ describe("ContentRepo", function() {
 
         it("should return the empty string HTTP URLs by default", function() {
             const httpUrl = "http://example.com/image.jpeg";
-            expect(ContentRepo.getHttpUriForMxc(baseUrl, httpUrl)).toEqual("");
+            expect(getHttpUriForMxc(baseUrl, httpUrl)).toEqual("");
         });
 
         it("should return a download URL if no width/height/resize are specified",
         function() {
             const mxcUri = "mxc://server.name/resourceid";
-            expect(ContentRepo.getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
-                baseUrl + "/_matrix/media/v1/download/server.name/resourceid",
+            expect(getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
+                baseUrl + "/_matrix/media/r0/download/server.name/resourceid",
             );
         });
 
         it("should return the empty string for null input", function() {
-            expect(ContentRepo.getHttpUriForMxc(null)).toEqual("");
+            expect(getHttpUriForMxc(null)).toEqual("");
         });
 
         it("should return a thumbnail URL if a width/height/resize is specified",
         function() {
             const mxcUri = "mxc://server.name/resourceid";
-            expect(ContentRepo.getHttpUriForMxc(baseUrl, mxcUri, 32, 64, "crop")).toEqual(
-                baseUrl + "/_matrix/media/v1/thumbnail/server.name/resourceid" +
+            expect(getHttpUriForMxc(baseUrl, mxcUri, 32, 64, "crop")).toEqual(
+                baseUrl + "/_matrix/media/r0/thumbnail/server.name/resourceid" +
                 "?width=32&height=64&method=crop",
             );
         });
@@ -51,8 +42,8 @@ describe("ContentRepo", function() {
         it("should put fragments from mxc:// URIs after any query parameters",
         function() {
             const mxcUri = "mxc://server.name/resourceid#automade";
-            expect(ContentRepo.getHttpUriForMxc(baseUrl, mxcUri, 32)).toEqual(
-                baseUrl + "/_matrix/media/v1/thumbnail/server.name/resourceid" +
+            expect(getHttpUriForMxc(baseUrl, mxcUri, 32)).toEqual(
+                baseUrl + "/_matrix/media/r0/thumbnail/server.name/resourceid" +
                 "?width=32#automade",
             );
         });
@@ -60,35 +51,8 @@ describe("ContentRepo", function() {
         it("should put fragments from mxc:// URIs at the end of the HTTP URI",
         function() {
             const mxcUri = "mxc://server.name/resourceid#automade";
-            expect(ContentRepo.getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
-                baseUrl + "/_matrix/media/v1/download/server.name/resourceid#automade",
-            );
-        });
-    });
-
-    describe("getIdenticonUri", function() {
-        it("should do nothing for null input", function() {
-            expect(ContentRepo.getIdenticonUri(null)).toEqual(null);
-        });
-
-        it("should set w/h by default to 96", function() {
-            expect(ContentRepo.getIdenticonUri(baseUrl, "foobar")).toEqual(
-                baseUrl + "/_matrix/media/v1/identicon/foobar" +
-                "?width=96&height=96",
-            );
-        });
-
-        it("should be able to set custom w/h", function() {
-            expect(ContentRepo.getIdenticonUri(baseUrl, "foobar", 32, 64)).toEqual(
-                baseUrl + "/_matrix/media/v1/identicon/foobar" +
-                "?width=32&height=64",
-            );
-        });
-
-        it("should URL encode the identicon string", function() {
-            expect(ContentRepo.getIdenticonUri(baseUrl, "foo#bar", 32, 64)).toEqual(
-                baseUrl + "/_matrix/media/v1/identicon/foo%23bar" +
-                "?width=32&height=64",
+            expect(getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
+                baseUrl + "/_matrix/media/r0/download/server.name/resourceid#automade",
             );
         });
     });

spec/unit/crypto.spec.js

@@ -1,30 +1,92 @@
-import 'source-map-support/register';
-import Crypto from '../../lib/crypto';
-import expect from 'expect';
-
-import WebStorageSessionStore from '../../lib/store/session/webstorage';
-import MemoryCryptoStore from '../../lib/crypto/store/memory-crypto-store.js';
-import MockStorageApi from '../MockStorageApi';
-
-const EventEmitter = require("events").EventEmitter;
-
-const sdk = require("../..");
+import '../olm-loader';
+import {Crypto} from "../../src/crypto";
+import {WebStorageSessionStore} from "../../src/store/session/webstorage";
+import {MemoryCryptoStore} from "../../src/crypto/store/memory-crypto-store";
+import {MockStorageApi} from "../MockStorageApi";
+import {TestClient} from "../TestClient";
+import {MatrixEvent} from "../../src/models/event";
+import {Room} from "../../src/models/room";
+import * as olmlib from "../../src/crypto/olmlib";
+import {sleep} from "../../src/utils";
+import {EventEmitter} from "events";
+import {CRYPTO_ENABLED} from "../../src/client";
+import {DeviceInfo} from "../../src/crypto/deviceinfo";
 
 const Olm = global.Olm;
 
 describe("Crypto", function() {
-    if (!sdk.CRYPTO_ENABLED) {
+    if (!CRYPTO_ENABLED) {
         return;
     }
 
-    beforeEach(function(done) {
-        Olm.init().then(done);
+    beforeAll(function() {
+        return Olm.init();
     });
 
     it("Crypto exposes the correct olm library version", function() {
         expect(Crypto.getOlmVersion()[0]).toEqual(3);
     });
 
+    describe("encrypted events", function() {
+        it("provides encryption information", async function() {
+            const client = (new TestClient(
+                "@alice:example.com", "deviceid",
+            )).client;
+            await client.initCrypto();
+
+            // unencrypted event
+            const event = {
+                getId: () => "$event_id",
+                getSenderKey: () => null,
+                getWireContent: () => {return {};},
+            };
+
+            let encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeFalsy();
+
+            // unknown sender (e.g. deleted device), forwarded megolm key (untrusted)
+            event.getSenderKey = () => 'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI';
+            event.getWireContent = () => {return {algorithm: olmlib.MEGOLM_ALGORITHM};};
+            event.getForwardingCurve25519KeyChain = () => ["not empty"];
+            event.isKeySourceUntrusted = () => false;
+            event.getClaimedEd25519Key =
+                () => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeFalsy();
+            expect(encryptionInfo.sender).toBeFalsy();
+
+            // known sender, megolm key from backup
+            event.getForwardingCurve25519KeyChain = () => [];
+            event.isKeySourceUntrusted = () => true;
+            const device = new DeviceInfo("FLIBBLE");
+            device.keys["curve25519:FLIBBLE"] =
+                'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI';
+            device.keys["ed25519:FLIBBLE"] =
+                'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
+            client._crypto._deviceList.getDeviceByIdentityKey = () => device;
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeFalsy();
+            expect(encryptionInfo.sender).toBeTruthy();
+            expect(encryptionInfo.mismatchedSender).toBeFalsy();
+
+            // known sender, trusted megolm key, but bad ed25519key
+            event.isKeySourceUntrusted = () => false;
+            device.keys["ed25519:FLIBBLE"] =
+                'BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB';
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeTruthy();
+            expect(encryptionInfo.sender).toBeTruthy();
+            expect(encryptionInfo.mismatchedSender).toBeTruthy();
+
+            client.stopClient();
+        });
+    });
 
     describe('Session management', function() {
         const otkResponse = {
@@ -68,8 +130,9 @@ describe("Crypto", function() {
             });
 
             mockBaseApis = {
-                sendToDevice: expect.createSpy(),
-                getKeyBackupVersion: expect.createSpy(),
+                sendToDevice: jest.fn(),
+                getKeyBackupVersion: jest.fn(),
+                isGuest: jest.fn(),
             };
             mockRoomList = {};
 
@@ -101,18 +164,238 @@ describe("Crypto", function() {
             });
 
             fakeEmitter.emit('toDeviceEvent', {
-                getType: expect.createSpy().andReturn('m.room.message'),
-                getContent: expect.createSpy().andReturn({
+                getId: jest.fn().mockReturnValue("$wedged"),
+                getType: jest.fn().mockReturnValue('m.room.message'),
+                getContent: jest.fn().mockReturnValue({
                     msgtype: 'm.bad.encrypted',
                 }),
-                getWireContent: expect.createSpy().andReturn({
+                getWireContent: jest.fn().mockReturnValue({
                     algorithm: 'm.olm.v1.curve25519-aes-sha2',
                     sender_key: 'this is a key',
                 }),
-                getSender: expect.createSpy().andReturn('@bob:home.server'),
+                getSender: jest.fn().mockReturnValue('@bob:home.server'),
             });
 
             await prom;
         });
     });
+
+    describe('Key requests', function() {
+        let aliceClient;
+        let bobClient;
+
+        beforeEach(async function() {
+            aliceClient = (new TestClient(
+                "@alice:example.com", "alicedevice",
+            )).client;
+            bobClient = (new TestClient(
+                "@bob:example.com", "bobdevice",
+            )).client;
+            await aliceClient.initCrypto();
+            await bobClient.initCrypto();
+        });
+
+        afterEach(async function() {
+            aliceClient.stopClient();
+            bobClient.stopClient();
+        });
+
+        it(
+            "does not cancel keyshare requests if some messages are not decrypted",
+            async function() {
+                function awaitEvent(emitter, event) {
+                    return new Promise((resolve, reject) => {
+                        emitter.once(event, (result) => {
+                            resolve(result);
+                        });
+                    });
+                }
+
+                async function keyshareEventForEvent(event, index) {
+                    const eventContent = event.getWireContent();
+                    const key = await aliceClient._crypto._olmDevice
+                        .getInboundGroupSessionKey(
+                            roomId, eventContent.sender_key, eventContent.session_id,
+                            index,
+                        );
+                    const ksEvent = new MatrixEvent({
+                        type: "m.forwarded_room_key",
+                        sender: "@alice:example.com",
+                        content: {
+                            algorithm: olmlib.MEGOLM_ALGORITHM,
+                            room_id: roomId,
+                            sender_key: eventContent.sender_key,
+                            sender_claimed_ed25519_key: key.sender_claimed_ed25519_key,
+                            session_id: eventContent.session_id,
+                            session_key: key.key,
+                            chain_index: key.chain_index,
+                            forwarding_curve25519_key_chain:
+                            key.forwarding_curve_key_chain,
+                        },
+                    });
+                    // make onRoomKeyEvent think this was an encrypted event
+                    ksEvent._senderCurve25519Key = "akey";
+                    return ksEvent;
+                }
+
+                const encryptionCfg = {
+                    "algorithm": "m.megolm.v1.aes-sha2",
+                };
+                const roomId = "!someroom";
+                const aliceRoom = new Room(roomId, aliceClient, "@alice:example.com", {});
+                const bobRoom = new Room(roomId, bobClient, "@bob:example.com", {});
+                aliceClient.store.storeRoom(aliceRoom);
+                bobClient.store.storeRoom(bobRoom);
+                await aliceClient.setRoomEncryption(roomId, encryptionCfg);
+                await bobClient.setRoomEncryption(roomId, encryptionCfg);
+                const events = [
+                    new MatrixEvent({
+                        type: "m.room.message",
+                        sender: "@alice:example.com",
+                        room_id: roomId,
+                        event_id: "$1",
+                        content: {
+                            msgtype: "m.text",
+                            body: "1",
+                        },
+                    }),
+                    new MatrixEvent({
+                        type: "m.room.message",
+                        sender: "@alice:example.com",
+                        room_id: roomId,
+                        event_id: "$2",
+                        content: {
+                            msgtype: "m.text",
+                            body: "2",
+                        },
+                    }),
+                ];
+                await Promise.all(events.map(async (event) => {
+                    // alice encrypts each event, and then bob tries to decrypt
+                    // them without any keys, so that they'll be in pending
+                    await aliceClient._crypto.encryptEvent(event, aliceRoom);
+                    event._clearEvent = {};
+                    event._senderCurve25519Key = null;
+                    event._claimedEd25519Key = null;
+                    try {
+                        await bobClient._crypto.decryptEvent(event);
+                    } catch (e) {
+                        // we expect this to fail because we don't have the
+                        // decryption keys yet
+                    }
+                }));
+
+                const bobDecryptor = bobClient._crypto._getRoomDecryptor(
+                    roomId, olmlib.MEGOLM_ALGORITHM,
+                );
+
+                let eventPromise = Promise.all(events.map((ev) => {
+                    return awaitEvent(ev, "Event.decrypted");
+                }));
+
+                // keyshare the session key starting at the second message, so
+                // the first message can't be decrypted yet, but the second one
+                // can
+                let ksEvent = await keyshareEventForEvent(events[1], 1);
+                await bobDecryptor.onRoomKeyEvent(ksEvent);
+                await eventPromise;
+                expect(events[0].getContent().msgtype).toBe("m.bad.encrypted");
+                expect(events[1].getContent().msgtype).not.toBe("m.bad.encrypted");
+
+                const cryptoStore = bobClient._cryptoStore;
+                const eventContent = events[0].getWireContent();
+                const senderKey = eventContent.sender_key;
+                const sessionId = eventContent.session_id;
+                const roomKeyRequestBody = {
+                    algorithm: olmlib.MEGOLM_ALGORITHM,
+                    room_id: roomId,
+                    sender_key: senderKey,
+                    session_id: sessionId,
+                };
+                // the room key request should still be there, since we haven't
+                // decrypted everything
+                expect(await cryptoStore.getOutgoingRoomKeyRequest(roomKeyRequestBody))
+                    .toBeDefined();
+
+                // keyshare the session key starting at the first message, so
+                // that it can now be decrypted
+                eventPromise = awaitEvent(events[0], "Event.decrypted");
+                ksEvent = await keyshareEventForEvent(events[0], 0);
+                await bobDecryptor.onRoomKeyEvent(ksEvent);
+                await eventPromise;
+                expect(events[0].getContent().msgtype).not.toBe("m.bad.encrypted");
+                await sleep(1);
+                // the room key request should be gone since we've now decrypted everything
+                expect(await cryptoStore.getOutgoingRoomKeyRequest(roomKeyRequestBody))
+                    .toBeFalsy();
+            },
+        );
+
+        it("creates a new keyshare request if we request a keyshare", async function() {
+            // make sure that cancelAndResend... creates a new keyshare request
+            // if there wasn't an already-existing one
+            const event = new MatrixEvent({
+                sender: "@bob:example.com",
+                room_id: "!someroom",
+                content: {
+                    algorithm: olmlib.MEGOLM_ALGORITHM,
+                    session_id: "sessionid",
+                    sender_key: "senderkey",
+                },
+            });
+            await aliceClient.cancelAndResendEventRoomKeyRequest(event);
+            const cryptoStore = aliceClient._cryptoStore;
+            const roomKeyRequestBody = {
+                algorithm: olmlib.MEGOLM_ALGORITHM,
+                room_id: "!someroom",
+                session_id: "sessionid",
+                sender_key: "senderkey",
+            };
+            expect(await cryptoStore.getOutgoingRoomKeyRequest(roomKeyRequestBody))
+                .toBeDefined();
+        });
+
+        it("uses a new txnid for re-requesting keys", async function() {
+            jest.useFakeTimers();
+
+            const event = new MatrixEvent({
+                sender: "@bob:example.com",
+                room_id: "!someroom",
+                content: {
+                    algorithm: olmlib.MEGOLM_ALGORITHM,
+                    session_id: "sessionid",
+                    sender_key: "senderkey",
+                },
+            });
+            // replace Alice's sendToDevice function with a mock
+            aliceClient.sendToDevice = jest.fn().mockResolvedValue(undefined);
+            aliceClient.startClient();
+
+            // make a room key request, and record the transaction ID for the
+            // sendToDevice call
+            await aliceClient.cancelAndResendEventRoomKeyRequest(event);
+            // key requests get queued until the sync has finished, but we don't
+            // let the client set up enough for that to happen, so gut-wrench a bit
+            // to force it to send now.
+            aliceClient._crypto._outgoingRoomKeyRequestManager.sendQueuedRequests();
+            jest.runAllTimers();
+            await Promise.resolve();
+            expect(aliceClient.sendToDevice).toBeCalledTimes(1);
+            const txnId = aliceClient.sendToDevice.mock.calls[0][2];
+
+            // give the room key request manager time to update the state
+            // of the request
+            await Promise.resolve();
+
+            // cancel and resend the room key request
+            await aliceClient.cancelAndResendEventRoomKeyRequest(event);
+            jest.runAllTimers();
+            await Promise.resolve();
+            // cancelAndResend will call sendToDevice twice:
+            // the first call to sendToDevice will be the cancellation
+            // the second call to sendToDevice will be the key request
+            expect(aliceClient.sendToDevice).toBeCalledTimes(3);
+            expect(aliceClient.sendToDevice.mock.calls[2][2]).not.toBe(txnId);
+        });
+    });
 });

spec/unit/crypto/CrossSigningInfo.spec.js

@@ -0,0 +1,258 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import '../../olm-loader';
+import {
+    CrossSigningInfo,
+    createCryptoStoreCacheCallbacks,
+} from '../../../src/crypto/CrossSigning';
+import {
+    IndexedDBCryptoStore,
+} from '../../../src/crypto/store/indexeddb-crypto-store';
+import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
+import 'fake-indexeddb/auto';
+import 'jest-localstorage-mock';
+import {OlmDevice} from "../../../src/crypto/OlmDevice";
+import {logger} from '../../../src/logger';
+
+const userId = "@alice:example.com";
+
+// Private key for tests only
+const testKey = new Uint8Array([
+    0xda, 0x5a, 0x27, 0x60, 0xe3, 0x3a, 0xc5, 0x82,
+    0x9d, 0x12, 0xc3, 0xbe, 0xe8, 0xaa, 0xc2, 0xef,
+    0xae, 0xb1, 0x05, 0xc1, 0xe7, 0x62, 0x78, 0xa6,
+    0xd7, 0x1f, 0xf8, 0x2c, 0x51, 0x85, 0xf0, 0x1d,
+]);
+
+const types = [
+    { type: "master", shouldCache: true },
+    { type: "self_signing", shouldCache: true },
+    { type: "user_signing", shouldCache: true },
+    { type: "invalid", shouldCache: false },
+];
+
+const badKey = Uint8Array.from(testKey);
+badKey[0] ^= 1;
+
+const masterKeyPub = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk";
+
+describe("CrossSigningInfo.getCrossSigningKey", function() {
+    if (!global.Olm) {
+        logger.warn('Not running megolm backup unit tests: libolm not present');
+        return;
+    }
+
+    beforeAll(function() {
+        return global.Olm.init();
+    });
+
+    it("should throw if no callback is provided", async () => {
+        const info = new CrossSigningInfo(userId);
+        await expect(info.getCrossSigningKey("master")).rejects.toThrow();
+    });
+
+    it.each(types)("should throw if the callback returns falsey",
+                   async ({type, shouldCache}) => {
+        const info = new CrossSigningInfo(userId, {
+            getCrossSigningKey: () => false,
+        });
+        await expect(info.getCrossSigningKey(type)).rejects.toThrow("falsey");
+    });
+
+    it("should throw if the expected key doesn't come back", async () => {
+        const info = new CrossSigningInfo(userId, {
+            getCrossSigningKey: () => masterKeyPub,
+        });
+        await expect(info.getCrossSigningKey("master", "")).rejects.toThrow();
+    });
+
+    it("should return a key from its callback", async () => {
+        const info = new CrossSigningInfo(userId, {
+            getCrossSigningKey: () => testKey,
+        });
+        const [pubKey, pkSigning] = await info.getCrossSigningKey("master", masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        // check that the pkSigning object corresponds to the pubKey
+        const signature = pkSigning.sign("message");
+        const util = new global.Olm.Utility();
+        try {
+            util.ed25519_verify(pubKey, "message", signature);
+        } finally {
+            util.free();
+        }
+    });
+
+    it.each(types)("should request a key from the cache callback (if set)" +
+                   " and does not call app if one is found" +
+                   " %o",
+                   async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockImplementation(() => {
+            if (shouldCache) {
+                return Promise.reject(new Error("Regular callback called"));
+            } else {
+                return Promise.resolve(testKey);
+            }
+        });
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(testKey);
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(getCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+        if (shouldCache) {
+            expect(getCrossSigningKeyCache.mock.calls[0][0]).toBe(type);
+        }
+    });
+
+    it.each(types)("should store a key with the cache callback (if set)",
+                   async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
+        const storeCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { storeCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(storeCrossSigningKeyCache.mock.calls.length).toEqual(shouldCache ? 1 : 0);
+        if (shouldCache) {
+            expect(storeCrossSigningKeyCache.mock.calls[0][0]).toBe(type);
+            expect(storeCrossSigningKeyCache.mock.calls[0][1]).toBe(testKey);
+        }
+    });
+
+    it.each(types)("does not store a bad key to the cache",
+                   async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(badKey);
+        const storeCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { storeCrossSigningKeyCache },
+        );
+        await expect(info.getCrossSigningKey(type, masterKeyPub)).rejects.toThrow();
+        expect(storeCrossSigningKeyCache.mock.calls.length).toEqual(0);
+    });
+
+    it.each(types)("does not store a value to the cache if it came from the cache",
+        async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockImplementation(() => {
+            if (shouldCache) {
+                return Promise.reject(new Error("Regular callback called"));
+            } else {
+                return Promise.resolve(testKey);
+            }
+        });
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(testKey);
+        const storeCrossSigningKeyCache = jest.fn().mockRejectedValue(
+            new Error("Tried to store a value from cache"),
+        );
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache, storeCrossSigningKeyCache },
+        );
+        expect(storeCrossSigningKeyCache.mock.calls.length).toBe(0);
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+    });
+
+    it.each(types)("requests a key from the cache callback (if set) and then calls app" +
+        " if one is not found", async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+        const storeCrossSigningKeyCache = jest.fn();
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache, storeCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(getCrossSigningKey.mock.calls.length).toBe(1);
+        expect(getCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+
+        /* Also expect that the cache gets updated */
+        expect(storeCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+    });
+
+    it.each(types)("requests a key from the cache callback (if set) and then" +
+        " calls app if that key doesn't match", async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(badKey);
+        const storeCrossSigningKeyCache = jest.fn();
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache, storeCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(getCrossSigningKey.mock.calls.length).toBe(1);
+        expect(getCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+
+        /* Also expect that the cache gets updated */
+        expect(storeCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+    });
+});
+
+/*
+ * Note that MemoryStore is weird.  It's only used for testing - as far as I can tell,
+ * it's not possible to get one in normal execution unless you hack as we do here.
+ */
+describe.each([
+    ["IndexedDBCryptoStore",
+     () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+    ["LocalStorageCryptoStore",
+     () => new IndexedDBCryptoStore(undefined, "tests")],
+    ["MemoryCryptoStore", () => {
+        const store = new IndexedDBCryptoStore(undefined, "tests");
+        store._backend = new MemoryCryptoStore();
+        store._backendPromise = Promise.resolve(store._backend);
+        return store;
+    }],
+])("CrossSigning > createCryptoStoreCacheCallbacks [%s]", function(name, dbFactory) {
+    let store;
+
+    beforeAll(() => {
+        store = dbFactory();
+    });
+
+    beforeEach(async () => {
+        await store.deleteAllData();
+    });
+
+    it("should cache data to the store and retrieve it", async () => {
+        await store.startup();
+        const olmDevice = new OlmDevice(store);
+        const { getCrossSigningKeyCache, storeCrossSigningKeyCache } =
+              createCryptoStoreCacheCallbacks(store, olmDevice);
+        await storeCrossSigningKeyCache("self_signing", testKey);
+
+        // If we've not saved anything, don't expect anything
+        // Definitely don't accidentally return the wrong key for the type
+        const nokey = await getCrossSigningKeyCache("self", "");
+        expect(nokey).toBeNull();
+
+        const key = await getCrossSigningKeyCache("self_signing", "");
+        expect(new Uint8Array(key)).toEqual(testKey);
+    });
+});

spec/unit/crypto/DeviceList.spec.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2017 Vector Creations Ltd
-Copyright 2018 New Vector Ltd
+Copyright 2018, 2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,15 +16,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import DeviceList from '../../../lib/crypto/DeviceList';
-import MockStorageApi from '../../MockStorageApi';
-import WebStorageSessionStore from '../../../lib/store/session/webstorage';
-import MemoryCryptoStore from '../../../lib/crypto/store/memory-crypto-store.js';
-import testUtils from '../../test-utils';
-import utils from '../../../lib/utils';
-
-import expect from 'expect';
-import Promise from 'bluebird';
+import {logger} from "../../../src/logger";
+import * as utils from "../../../src/utils";
+import {MemoryCryptoStore} from "../../../src/crypto/store/memory-crypto-store";
+import {DeviceList} from "../../../src/crypto/DeviceList";
 
 const signedDeviceList = {
     "failures": {},
@@ -57,18 +53,13 @@ const signedDeviceList = {
 
 describe('DeviceList', function() {
     let downloadSpy;
-    let sessionStore;
     let cryptoStore;
     let deviceLists = [];
 
     beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
-
         deviceLists = [];
 
-        downloadSpy = expect.createSpy();
-        const mockStorage = new MockStorageApi();
-        sessionStore = new WebStorageSessionStore(mockStorage);
+        downloadSpy = jest.fn();
         cryptoStore = new MemoryCryptoStore();
     });
 
@@ -81,11 +72,13 @@ describe('DeviceList', function() {
     function createTestDeviceList() {
         const baseApis = {
             downloadKeysForUsers: downloadSpy,
+            getUserId: () => '@test1:sw1v.org',
+            deviceId: 'HGKAWHRVJQ',
         };
         const mockOlm = {
             verifySignature: function(key, message, signature) {},
         };
-        const dl = new DeviceList(baseApis, cryptoStore, sessionStore, mockOlm);
+        const dl = new DeviceList(baseApis, cryptoStore, mockOlm);
         deviceLists.push(dl);
         return dl;
     }
@@ -95,8 +88,8 @@ describe('DeviceList', function() {
 
         dl.startTrackingDeviceList('@test1:sw1v.org');
 
-        const queryDefer1 = Promise.defer();
-        downloadSpy.andReturn(queryDefer1.promise);
+        const queryDefer1 = utils.defer();
+        downloadSpy.mockReturnValue(queryDefer1.promise);
 
         const prom1 = dl.refreshOutdatedDeviceLists();
         expect(downloadSpy).toHaveBeenCalledWith(['@test1:sw1v.org'], {});
@@ -114,16 +107,16 @@ describe('DeviceList', function() {
 
         dl.startTrackingDeviceList('@test1:sw1v.org');
 
-        const queryDefer1 = Promise.defer();
-        downloadSpy.andReturn(queryDefer1.promise);
+        const queryDefer1 = utils.defer();
+        downloadSpy.mockReturnValue(queryDefer1.promise);
 
         const prom1 = dl.refreshOutdatedDeviceLists();
         expect(downloadSpy).toHaveBeenCalledWith(['@test1:sw1v.org'], {});
-        downloadSpy.reset();
+        downloadSpy.mockReset();
 
         // outdated notif arrives while the request is in flight.
-        const queryDefer2 = Promise.defer();
-        downloadSpy.andReturn(queryDefer2.promise);
+        const queryDefer2 = utils.defer();
+        downloadSpy.mockReturnValue(queryDefer2.promise);
 
         dl.invalidateUserDeviceList('@test1:sw1v.org');
         dl.refreshOutdatedDeviceLists();
@@ -139,11 +132,11 @@ describe('DeviceList', function() {
         }).then(() => {
             // uh-oh; user restarts before second request completes. The new instance
             // should know we never got a complete device list.
-            console.log("Creating new devicelist to simulate app reload");
-            downloadSpy.reset();
+            logger.log("Creating new devicelist to simulate app reload");
+            downloadSpy.mockReset();
             const dl2 = createTestDeviceList();
-            const queryDefer3 = Promise.defer();
-            downloadSpy.andReturn(queryDefer3.promise);
+            const queryDefer3 = utils.defer();
+            downloadSpy.mockReturnValue(queryDefer3.promise);
 
             const prom3 = dl2.refreshOutdatedDeviceLists();
             expect(downloadSpy).toHaveBeenCalledWith(['@test1:sw1v.org'], {});

spec/unit/crypto/algorithms/megolm.spec.js

@@ -1,22 +1,16 @@
-try {
-    global.Olm = require('olm');
-} catch (e) {
-    console.warn("unable to run megolm tests: libolm not available");
-}
+import '../../../olm-loader';
+import * as algorithms from "../../../../src/crypto/algorithms";
+import {MemoryCryptoStore} from "../../../../src/crypto/store/memory-crypto-store";
+import {MockStorageApi} from "../../../MockStorageApi";
+import * as testUtils from "../../../test-utils";
+import {OlmDevice} from "../../../../src/crypto/OlmDevice";
+import {Crypto} from "../../../../src/crypto";
+import {logger} from "../../../../src/logger";
+import {MatrixEvent} from "../../../../src/models/event";
+import {TestClient} from "../../../TestClient";
+import {Room} from "../../../../src/models/room";
+import * as olmlib from "../../../../src/crypto/olmlib";
 
-import expect from 'expect';
-import Promise from 'bluebird';
-
-import sdk from '../../../..';
-import algorithms from '../../../../lib/crypto/algorithms';
-import WebStorageSessionStore from '../../../../lib/store/session/webstorage';
-import MemoryCryptoStore from '../../../../lib/crypto/store/memory-crypto-store.js';
-import MockStorageApi from '../../../MockStorageApi';
-import testUtils from '../../../test-utils';
-import OlmDevice from '../../../../lib/crypto/OlmDevice';
-import Crypto from '../../../../lib/crypto';
-
-const MatrixEvent = sdk.MatrixEvent;
 const MegolmDecryption = algorithms.DECRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
 const MegolmEncryption = algorithms.ENCRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
 
@@ -26,28 +20,27 @@ const Olm = global.Olm;
 
 describe("MegolmDecryption", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm unit tests: libolm not present');
+        logger.warn('Not running megolm unit tests: libolm not present');
         return;
     }
 
+    beforeAll(function() {
+        return Olm.init();
+    });
+
     let megolmDecryption;
     let mockOlmLib;
     let mockCrypto;
     let mockBaseApis;
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
-
-        await Olm.init();
-
         mockCrypto = testUtils.mock(Crypto, 'Crypto');
         mockBaseApis = {};
 
         const mockStorage = new MockStorageApi();
-        const sessionStore = new WebStorageSessionStore(mockStorage);
         const cryptoStore = new MemoryCryptoStore(mockStorage);
 
-        const olmDevice = new OlmDevice(sessionStore, cryptoStore);
+        const olmDevice = new OlmDevice(cryptoStore);
 
         megolmDecryption = new MegolmDecryption({
             userId: '@user:id',
@@ -60,9 +53,9 @@ describe("MegolmDecryption", function() {
 
         // we stub out the olm encryption bits
         mockOlmLib = {};
-        mockOlmLib.ensureOlmSessionsForDevices = expect.createSpy();
+        mockOlmLib.ensureOlmSessionsForDevices = jest.fn();
         mockOlmLib.encryptMessageForDevice =
-            expect.createSpy().andReturn(Promise.resolve());
+            jest.fn().mockResolvedValue(undefined);
         megolmDecryption.olmlib = mockOlmLib;
     });
 
@@ -140,22 +133,22 @@ describe("MegolmDecryption", function() {
 
                 // set up some pre-conditions for the share call
                 const deviceInfo = {};
-                mockCrypto.getStoredDevice.andReturn(deviceInfo);
+                mockCrypto.getStoredDevice.mockReturnValue(deviceInfo);
 
-                mockOlmLib.ensureOlmSessionsForDevices.andReturn(
-                    Promise.resolve({'@alice:foo': {'alidevice': {
+                mockOlmLib.ensureOlmSessionsForDevices.mockResolvedValue({
+                    '@alice:foo': {'alidevice': {
                         sessionId: 'alisession',
-                    }}}),
-                );
+                    }},
+                });
 
                 const awaitEncryptForDevice = new Promise((res, rej) => {
-                    mockOlmLib.encryptMessageForDevice.andCall(() => {
+                    mockOlmLib.encryptMessageForDevice.mockImplementation(() => {
                         res();
                         return Promise.resolve();
                     });
                 });
 
-                mockBaseApis.sendToDevice = expect.createSpy();
+                mockBaseApis.sendToDevice = jest.fn();
 
                 // do the share
                 megolmDecryption.shareKeysWithDevice(keyRequest);
@@ -165,21 +158,20 @@ describe("MegolmDecryption", function() {
             }).then(() => {
                 // check that it called encryptMessageForDevice with
                 // appropriate args.
-                expect(mockOlmLib.encryptMessageForDevice.calls.length)
-                    .toEqual(1);
+                expect(mockOlmLib.encryptMessageForDevice).toBeCalledTimes(1);
 
-                const call = mockOlmLib.encryptMessageForDevice.calls[0];
-                const payload = call.arguments[6];
+                const call = mockOlmLib.encryptMessageForDevice.mock.calls[0];
+                const payload = call[6];
 
                 expect(payload.type).toEqual("m.forwarded_room_key");
-                expect(payload.content).toInclude({
+                expect(payload.content).toMatchObject({
                     sender_key: "SENDER_CURVE25519",
                     sender_claimed_ed25519_key: "SENDER_ED25519",
                     session_id: groupSession.session_id(),
                     chain_index: 0,
                     forwarding_curve25519_key_chain: [],
                 });
-                expect(payload.content.session_key).toExist();
+                expect(payload.content.session_key).toBeDefined();
             });
         });
 
@@ -206,13 +198,12 @@ describe("MegolmDecryption", function() {
                 origin_server_ts: 1507753886000,
             });
 
-            const successHandler = expect.createSpy();
-            const failureHandler = expect.createSpy()
-                .andCall((err) => {
-                    expect(err.toString()).toMatch(
-                        /Duplicate message index, possible replay attack/,
-                    );
-                });
+            const successHandler = jest.fn();
+            const failureHandler = jest.fn((err) => {
+                expect(err.toString()).toMatch(
+                    /Duplicate message index, possible replay attack/,
+                );
+            });
 
             return megolmDecryption.decryptEvent(event1).then((res) => {
                 const event2 = new MatrixEvent({
@@ -233,7 +224,7 @@ describe("MegolmDecryption", function() {
                 successHandler,
                 failureHandler,
             ).then(() => {
-                expect(successHandler).toNotHaveBeenCalled();
+                expect(successHandler).not.toHaveBeenCalled();
                 expect(failureHandler).toHaveBeenCalled();
             });
         });
@@ -268,14 +259,13 @@ describe("MegolmDecryption", function() {
 
         it("re-uses sessions for sequential messages", async function() {
             const mockStorage = new MockStorageApi();
-            const sessionStore = new WebStorageSessionStore(mockStorage);
             const cryptoStore = new MemoryCryptoStore(mockStorage);
 
-            const olmDevice = new OlmDevice(sessionStore, cryptoStore);
-            olmDevice.verifySignature = expect.createSpy();
+            const olmDevice = new OlmDevice(cryptoStore);
+            olmDevice.verifySignature = jest.fn();
             await olmDevice.init();
 
-            mockBaseApis.claimOneTimeKeys = expect.createSpy().andReturn(Promise.resolve({
+            mockBaseApis.claimOneTimeKeys = jest.fn().mockReturnValue(Promise.resolve({
                 one_time_keys: {
                     '@alice:home.server': {
                         aliceDevice: {
@@ -291,22 +281,26 @@ describe("MegolmDecryption", function() {
                     },
                 },
             }));
-            mockBaseApis.sendToDevice = expect.createSpy().andReturn(Promise.resolve());
+            mockBaseApis.sendToDevice = jest.fn().mockResolvedValue(undefined);
 
-            mockCrypto.downloadKeys.andReturn(Promise.resolve({
+            mockCrypto.downloadKeys.mockReturnValue(Promise.resolve({
                 '@alice:home.server': {
                     aliceDevice: {
                         deviceId: 'aliceDevice',
-                        isBlocked: expect.createSpy().andReturn(false),
-                        isUnverified: expect.createSpy().andReturn(false),
-                        getIdentityKey: expect.createSpy().andReturn(
+                        isBlocked: jest.fn().mockReturnValue(false),
+                        isUnverified: jest.fn().mockReturnValue(false),
+                        getIdentityKey: jest.fn().mockReturnValue(
                             'YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE',
                         ),
-                        getFingerprint: expect.createSpy().andReturn(''),
+                        getFingerprint: jest.fn().mockReturnValue(''),
                     },
                 },
             }));
 
+            mockCrypto.checkDeviceTrust.mockReturnValue({
+                isVerified: () => false,
+            });
+
             const megolmEncryption = new MegolmEncryption({
                 userId: '@user:id',
                 crypto: mockCrypto,
@@ -318,10 +312,10 @@ describe("MegolmDecryption", function() {
                 },
             });
             const mockRoom = {
-                getEncryptionTargetMembers: expect.createSpy().andReturn(
+                getEncryptionTargetMembers: jest.fn().mockReturnValue(
                     [{userId: "@alice:home.server"}],
                 ),
-                getBlacklistUnverifiedDevices: expect.createSpy().andReturn(false),
+                getBlacklistUnverifiedDevices: jest.fn().mockReturnValue(false),
             };
             const ct1 = await megolmEncryption.encryptMessage(mockRoom, "a.fake.type", {
                 body: "Some text",
@@ -329,28 +323,372 @@ describe("MegolmDecryption", function() {
             expect(mockRoom.getEncryptionTargetMembers).toHaveBeenCalled();
 
             // this should have claimed a key for alice as it's starting a new session
-            expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalled(
-                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519',
+            expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalledWith(
+                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519', 2000,
             );
             expect(mockCrypto.downloadKeys).toHaveBeenCalledWith(
                 ['@alice:home.server'], false,
             );
             expect(mockBaseApis.sendToDevice).toHaveBeenCalled();
-            expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalled(
-                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519',
+            expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalledWith(
+                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519', 2000,
             );
 
-            mockBaseApis.claimOneTimeKeys.reset();
+            mockBaseApis.claimOneTimeKeys.mockReset();
 
             const ct2 = await megolmEncryption.encryptMessage(mockRoom, "a.fake.type", {
                 body: "Some more text",
             });
 
             // this should *not* have claimed a key as it should be using the same session
-            expect(mockBaseApis.claimOneTimeKeys).toNotHaveBeenCalled();
+            expect(mockBaseApis.claimOneTimeKeys).not.toHaveBeenCalled();
 
             // likewise they should show the same session ID
             expect(ct2.session_id).toEqual(ct1.session_id);
         });
     });
+
+    it("notifies devices that have been blocked", async function() {
+        const aliceClient = (new TestClient(
+            "@alice:example.com", "alicedevice",
+        )).client;
+        const bobClient1 = (new TestClient(
+            "@bob:example.com", "bobdevice1",
+        )).client;
+        const bobClient2 = (new TestClient(
+            "@bob:example.com", "bobdevice2",
+        )).client;
+        await Promise.all([
+            aliceClient.initCrypto(),
+            bobClient1.initCrypto(),
+            bobClient2.initCrypto(),
+        ]);
+        const aliceDevice = aliceClient._crypto._olmDevice;
+        const bobDevice1 = bobClient1._crypto._olmDevice;
+        const bobDevice2 = bobClient2._crypto._olmDevice;
+
+        const encryptionCfg = {
+            "algorithm": "m.megolm.v1.aes-sha2",
+        };
+        const roomId = "!someroom";
+        const room = new Room(roomId, aliceClient, "@alice:example.com", {});
+        room.getEncryptionTargetMembers = async function() {
+            return [{userId: "@bob:example.com"}];
+        };
+        room.setBlacklistUnverifiedDevices(true);
+        aliceClient.store.storeRoom(room);
+        await aliceClient.setRoomEncryption(roomId, encryptionCfg);
+
+        const BOB_DEVICES = {
+            bobdevice1: {
+                user_id: "@bob:example.com",
+                device_id: "bobdevice1",
+                algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                keys: {
+                    "ed25519:Dynabook": bobDevice1.deviceEd25519Key,
+                    "curve25519:Dynabook": bobDevice1.deviceCurve25519Key,
+                },
+                verified: 0,
+            },
+            bobdevice2: {
+                user_id: "@bob:example.com",
+                device_id: "bobdevice2",
+                algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                keys: {
+                    "ed25519:Dynabook": bobDevice2.deviceEd25519Key,
+                    "curve25519:Dynabook": bobDevice2.deviceCurve25519Key,
+                },
+                verified: -1,
+            },
+        };
+
+        aliceClient._crypto._deviceList.storeDevicesForUser(
+            "@bob:example.com", BOB_DEVICES,
+        );
+        aliceClient._crypto._deviceList.downloadKeys = async function(userIds) {
+            return this._getDevicesFromStore(userIds);
+        };
+
+        let run = false;
+        aliceClient.sendToDevice = async (msgtype, contentMap) => {
+            run = true;
+            expect(msgtype).toBe("org.matrix.room_key.withheld");
+            delete contentMap["@bob:example.com"].bobdevice1.session_id;
+            delete contentMap["@bob:example.com"].bobdevice2.session_id;
+            expect(contentMap).toStrictEqual({
+                '@bob:example.com': {
+                    bobdevice1: {
+                        algorithm: "m.megolm.v1.aes-sha2",
+                        room_id: roomId,
+                        code: 'm.unverified',
+                        reason:
+                        'The sender has disabled encrypting to unverified devices.',
+                        sender_key: aliceDevice.deviceCurve25519Key,
+                    },
+                    bobdevice2: {
+                        algorithm: "m.megolm.v1.aes-sha2",
+                        room_id: roomId,
+                        code: 'm.blacklisted',
+                        reason: 'The sender has blocked you.',
+                        sender_key: aliceDevice.deviceCurve25519Key,
+                    },
+                },
+            });
+        };
+
+        const event = new MatrixEvent({
+            type: "m.room.message",
+            sender: "@alice:example.com",
+            room_id: roomId,
+            event_id: "$event",
+            content: {
+                msgtype: "m.text",
+                body: "secret",
+            },
+        });
+        await aliceClient._crypto.encryptEvent(event, room);
+
+        expect(run).toBe(true);
+
+        aliceClient.stopClient();
+        bobClient1.stopClient();
+        bobClient2.stopClient();
+    });
+
+    it("notifies devices when unable to create olm session", async function() {
+        const aliceClient = (new TestClient(
+            "@alice:example.com", "alicedevice",
+        )).client;
+        const bobClient = (new TestClient(
+            "@bob:example.com", "bobdevice",
+        )).client;
+        await Promise.all([
+            aliceClient.initCrypto(),
+            bobClient.initCrypto(),
+        ]);
+        const aliceDevice = aliceClient._crypto._olmDevice;
+        const bobDevice = bobClient._crypto._olmDevice;
+
+        const encryptionCfg = {
+            "algorithm": "m.megolm.v1.aes-sha2",
+        };
+        const roomId = "!someroom";
+        const aliceRoom = new Room(roomId, aliceClient, "@alice:example.com", {});
+        const bobRoom = new Room(roomId, bobClient, "@bob:example.com", {});
+        aliceClient.store.storeRoom(aliceRoom);
+        bobClient.store.storeRoom(bobRoom);
+        await aliceClient.setRoomEncryption(roomId, encryptionCfg);
+        await bobClient.setRoomEncryption(roomId, encryptionCfg);
+
+        aliceRoom.getEncryptionTargetMembers = async () => {
+            return [
+                {
+                    userId: "@alice:example.com",
+                    membership: "join",
+                },
+                {
+                    userId: "@bob:example.com",
+                    membership: "join",
+                },
+            ];
+        };
+        const BOB_DEVICES = {
+            bobdevice: {
+                user_id: "@bob:example.com",
+                device_id: "bobdevice",
+                algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                keys: {
+                    "ed25519:bobdevice": bobDevice.deviceEd25519Key,
+                    "curve25519:bobdevice": bobDevice.deviceCurve25519Key,
+                },
+                known: true,
+                verified: 1,
+            },
+        };
+
+        aliceClient._crypto._deviceList.storeDevicesForUser(
+            "@bob:example.com", BOB_DEVICES,
+        );
+        aliceClient._crypto._deviceList.downloadKeys = async function(userIds) {
+            return this._getDevicesFromStore(userIds);
+        };
+
+        aliceClient.claimOneTimeKeys = async () => {
+            // Bob has no one-time keys
+            return {
+                one_time_keys: {},
+            };
+        };
+
+        const sendPromise = new Promise((resolve, reject) => {
+            aliceClient.sendToDevice = async (msgtype, contentMap) => {
+                expect(msgtype).toBe("org.matrix.room_key.withheld");
+                expect(contentMap).toStrictEqual({
+                    '@bob:example.com': {
+                        bobdevice: {
+                            algorithm: "m.megolm.v1.aes-sha2",
+                            code: 'm.no_olm',
+                            reason: 'Unable to establish a secure channel.',
+                            sender_key: aliceDevice.deviceCurve25519Key,
+                        },
+                    },
+                });
+                resolve();
+            };
+        });
+
+        const event = new MatrixEvent({
+            type: "m.room.message",
+            sender: "@alice:example.com",
+            room_id: roomId,
+            event_id: "$event",
+            content: {},
+        });
+        await aliceClient._crypto.encryptEvent(event, aliceRoom);
+        await sendPromise;
+    });
+
+    it("throws an error describing why it doesn't have a key", async function() {
+        const aliceClient = (new TestClient(
+            "@alice:example.com", "alicedevice",
+        )).client;
+        const bobClient = (new TestClient(
+            "@bob:example.com", "bobdevice",
+        )).client;
+        await Promise.all([
+            aliceClient.initCrypto(),
+            bobClient.initCrypto(),
+        ]);
+        const bobDevice = bobClient._crypto._olmDevice;
+
+        const roomId = "!someroom";
+
+        aliceClient._crypto._onToDeviceEvent(new MatrixEvent({
+            type: "org.matrix.room_key.withheld",
+            sender: "@bob:example.com",
+            content: {
+                algorithm: "m.megolm.v1.aes-sha2",
+                room_id: roomId,
+                session_id: "session_id",
+                sender_key: bobDevice.deviceCurve25519Key,
+                code: "m.blacklisted",
+                reason: "You have been blocked",
+            },
+        }));
+
+        await expect(aliceClient._crypto.decryptEvent(new MatrixEvent({
+            type: "m.room.encrypted",
+            sender: "@bob:example.com",
+            event_id: "$event",
+            room_id: roomId,
+            content: {
+                algorithm: "m.megolm.v1.aes-sha2",
+                ciphertext: "blablabla",
+                device_id: "bobdevice",
+                sender_key: bobDevice.deviceCurve25519Key,
+                session_id: "session_id",
+            },
+        }))).rejects.toThrow("The sender has blocked you.");
+    });
+
+    it("throws an error describing the lack of an olm session", async function() {
+        const aliceClient = (new TestClient(
+            "@alice:example.com", "alicedevice",
+        )).client;
+        const bobClient = (new TestClient(
+            "@bob:example.com", "bobdevice",
+        )).client;
+        await Promise.all([
+            aliceClient.initCrypto(),
+            bobClient.initCrypto(),
+        ]);
+        aliceClient._crypto.downloadKeys = async () => {};
+        const bobDevice = bobClient._crypto._olmDevice;
+
+        const roomId = "!someroom";
+
+        const now = Date.now();
+
+        aliceClient._crypto._onToDeviceEvent(new MatrixEvent({
+            type: "org.matrix.room_key.withheld",
+            sender: "@bob:example.com",
+            content: {
+                algorithm: "m.megolm.v1.aes-sha2",
+                room_id: roomId,
+                session_id: "session_id",
+                sender_key: bobDevice.deviceCurve25519Key,
+                code: "m.no_olm",
+                reason: "Unable to establish a secure channel.",
+            },
+        }));
+
+        await new Promise((resolve) => {
+            setTimeout(resolve, 100);
+        });
+
+        await expect(aliceClient._crypto.decryptEvent(new MatrixEvent({
+            type: "m.room.encrypted",
+            sender: "@bob:example.com",
+            event_id: "$event",
+            room_id: roomId,
+            content: {
+                algorithm: "m.megolm.v1.aes-sha2",
+                ciphertext: "blablabla",
+                device_id: "bobdevice",
+                sender_key: bobDevice.deviceCurve25519Key,
+                session_id: "session_id",
+            },
+            origin_server_ts: now,
+        }))).rejects.toThrow("The sender was unable to establish a secure channel.");
+    });
+
+    it("throws an error to indicate a wedged olm session", async function() {
+        const aliceClient = (new TestClient(
+            "@alice:example.com", "alicedevice",
+        )).client;
+        const bobClient = (new TestClient(
+            "@bob:example.com", "bobdevice",
+        )).client;
+        await Promise.all([
+            aliceClient.initCrypto(),
+            bobClient.initCrypto(),
+        ]);
+        const bobDevice = bobClient._crypto._olmDevice;
+        aliceClient._crypto.downloadKeys = async () => {};
+
+        const roomId = "!someroom";
+
+        const now = Date.now();
+
+        // pretend we got an event that we can't decrypt
+        aliceClient._crypto._onToDeviceEvent(new MatrixEvent({
+            type: "m.room.encrypted",
+            sender: "@bob:example.com",
+            content: {
+                msgtype: "m.bad.encrypted",
+                algorithm: "m.megolm.v1.aes-sha2",
+                session_id: "session_id",
+                sender_key: bobDevice.deviceCurve25519Key,
+            },
+        }));
+
+        await new Promise((resolve) => {
+            setTimeout(resolve, 100);
+        });
+
+        await expect(aliceClient._crypto.decryptEvent(new MatrixEvent({
+            type: "m.room.encrypted",
+            sender: "@bob:example.com",
+            event_id: "$event",
+            room_id: roomId,
+            content: {
+                algorithm: "m.megolm.v1.aes-sha2",
+                ciphertext: "blablabla",
+                device_id: "bobdevice",
+                sender_key: bobDevice.deviceCurve25519Key,
+                session_id: "session_id",
+            },
+            origin_server_ts: now,
+        }))).rejects.toThrow("The secure channel with the sender was corrupted.");
+    });
 });

spec/unit/crypto/algorithms/olm.spec.js

@@ -1,5 +1,6 @@
 /*
-Copyright 2018 New Vector Ltd
+Copyright 2018,2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,25 +15,18 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-try {
-    global.Olm = require('olm');
-} catch (e) {
-    console.warn("unable to run megolm tests: libolm not available");
-}
-
-import expect from 'expect';
-import WebStorageSessionStore from '../../../../lib/store/session/webstorage';
-import MemoryCryptoStore from '../../../../lib/crypto/store/memory-crypto-store.js';
-import MockStorageApi from '../../../MockStorageApi';
-import testUtils from '../../../test-utils';
-
-import OlmDevice from '../../../../lib/crypto/OlmDevice';
+import '../../../olm-loader';
+import {MemoryCryptoStore} from "../../../../src/crypto/store/memory-crypto-store";
+import {MockStorageApi} from "../../../MockStorageApi";
+import {logger} from "../../../../src/logger";
+import {OlmDevice} from "../../../../src/crypto/OlmDevice";
+import * as olmlib from "../../../../src/crypto/olmlib";
+import {DeviceInfo} from "../../../../src/crypto/deviceinfo";
 
 function makeOlmDevice() {
     const mockStorage = new MockStorageApi();
-    const sessionStore = new WebStorageSessionStore(mockStorage);
     const cryptoStore = new MemoryCryptoStore(mockStorage);
-    const olmDevice = new OlmDevice(sessionStore, cryptoStore);
+    const olmDevice = new OlmDevice(cryptoStore);
     return olmDevice;
 }
 
@@ -47,20 +41,20 @@ async function setupSession(initiator, opponent) {
     return sid;
 }
 
-describe("OlmDecryption", function() {
+describe("OlmDevice", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm unit tests: libolm not present');
+        logger.warn('Not running megolm unit tests: libolm not present');
         return;
     }
 
+    beforeAll(function() {
+        return global.Olm.init();
+    });
+
     let aliceOlmDevice;
     let bobOlmDevice;
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
-
-        await global.Olm.init();
-
         aliceOlmDevice = makeOlmDevice();
         bobOlmDevice = makeOlmDevice();
         await aliceOlmDevice.init();
@@ -86,5 +80,115 @@ describe("OlmDecryption", function() {
                 "The olm or proteus is an aquatic salamander in the family Proteidae",
             );
         });
+
+        it('exports picked account and olm sessions', async function() {
+            const sessionId = await setupSession(aliceOlmDevice, bobOlmDevice);
+
+            const exported = await bobOlmDevice.export();
+            // At this moment only Alice (the “initiator” in setupSession) has a session
+            expect(exported.sessions).toEqual([]);
+
+            const MESSAGE = (
+                "The olm or proteus is an aquatic salamander"
+                + " in the family Proteidae"
+            );
+            const ciphertext = await aliceOlmDevice.encryptMessage(
+                bobOlmDevice.deviceCurve25519Key,
+                sessionId,
+                MESSAGE,
+            );
+
+            const bobRecreatedOlmDevice = makeOlmDevice();
+            bobRecreatedOlmDevice.init({ fromExportedDevice: exported });
+
+            const decrypted = await bobRecreatedOlmDevice.createInboundSession(
+                aliceOlmDevice.deviceCurve25519Key,
+                ciphertext.type,
+                ciphertext.body,
+            );
+            expect(decrypted.payload).toEqual(MESSAGE);
+
+            const exportedAgain = await bobRecreatedOlmDevice.export();
+            // this time we expect Bob to have a session to export
+            expect(exportedAgain.sessions).toHaveLength(1);
+
+            const MESSAGE_2 = (
+                "In contrast to most amphibians,"
+                + " the olm is entirely aquatic"
+            );
+            const ciphertext2 = await aliceOlmDevice.encryptMessage(
+                bobOlmDevice.deviceCurve25519Key,
+                sessionId,
+                MESSAGE_2,
+            );
+
+            const bobRecreatedAgainOlmDevice = makeOlmDevice();
+            bobRecreatedAgainOlmDevice.init({ fromExportedDevice: exportedAgain });
+
+            // Note: "decrypted_2" does not have the same structure as "decrypted"
+            const decrypted2 = await bobRecreatedAgainOlmDevice.decryptMessage(
+                aliceOlmDevice.deviceCurve25519Key,
+                decrypted.session_id,
+                ciphertext2.type,
+                ciphertext2.body,
+            );
+            expect(decrypted2).toEqual(MESSAGE_2);
+        });
+
+        it("creates only one session at a time", async function() {
+            // if we call ensureOlmSessionsForDevices multiple times, it should
+            // only try to create one session at a time, even if the server is
+            // slow
+            let count = 0;
+            const baseApis = {
+                claimOneTimeKeys: () => {
+                    // simulate a very slow server (.5 seconds to respond)
+                    count++;
+                    return new Promise((resolve, reject) => {
+                        setTimeout(reject, 500);
+                    });
+                },
+            };
+            const devicesByUser = {
+                "@bob:example.com": [
+                    DeviceInfo.fromStorage({
+                        keys: {
+                            "curve25519:ABCDEFG": "akey",
+                        },
+                    }, "ABCDEFG"),
+                ],
+            };
+            function alwaysSucceed(promise) {
+                // swallow any exception thrown by a promise, so that
+                // Promise.all doesn't abort
+                return promise.catch(() => {});
+            }
+
+            // start two tasks that try to ensure that there's an olm session
+            const promises = Promise.all([
+                alwaysSucceed(olmlib.ensureOlmSessionsForDevices(
+                    aliceOlmDevice, baseApis, devicesByUser,
+                )),
+                alwaysSucceed(olmlib.ensureOlmSessionsForDevices(
+                    aliceOlmDevice, baseApis, devicesByUser,
+                )),
+            ]);
+
+            await new Promise((resolve) => {
+                setTimeout(resolve, 200);
+            });
+
+            // after .2s, both tasks should have started, but one should be
+            // waiting on the other before trying to create a session, so
+            // claimOneTimeKeys should have only been called once
+            expect(count).toBe(1);
+
+            await promises;
+
+            // after waiting for both tasks to complete, the first task should
+            // have failed, so the second task should have tried to create a
+            // new session and will have called claimOneTimeKeys
+            expect(count).toBe(2);
+        });
     });
 });

spec/unit/crypto/backup.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,29 +14,23 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-try {
-    global.Olm = require('olm');
-} catch (e) {
-    console.warn("unable to run megolm backup tests: libolm not available");
-}
 
-import expect from 'expect';
-import Promise from 'bluebird';
-
-import sdk from '../../..';
-import algorithms from '../../../lib/crypto/algorithms';
-import WebStorageSessionStore from '../../../lib/store/session/webstorage';
-import MemoryCryptoStore from '../../../lib/crypto/store/memory-crypto-store.js';
-import MockStorageApi from '../../MockStorageApi';
-import testUtils from '../../test-utils';
-
-import OlmDevice from '../../../lib/crypto/OlmDevice';
-import Crypto from '../../../lib/crypto';
+import '../../olm-loader';
+import {logger} from "../../../src/logger";
+import * as olmlib from "../../../src/crypto/olmlib";
+import {MatrixClient} from "../../../src/client";
+import {MatrixEvent} from "../../../src/models/event";
+import * as algorithms from "../../../src/crypto/algorithms";
+import {WebStorageSessionStore} from "../../../src/store/session/webstorage";
+import {MemoryCryptoStore} from "../../../src/crypto/store/memory-crypto-store";
+import {MockStorageApi} from "../../MockStorageApi";
+import * as testUtils from "../../test-utils";
+import {OlmDevice} from "../../../src/crypto/OlmDevice";
+import {Crypto} from "../../../src/crypto";
+import {resetCrossSigningKeys} from "./crypto-utils";
 
 const Olm = global.Olm;
 
-const MatrixClient = sdk.MatrixClient;
-const MatrixEvent = sdk.MatrixEvent;
 const MegolmDecryption = algorithms.DECRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
 
 const ROOM_ID = '!ROOM:ID';
@@ -77,20 +72,38 @@ const KEY_BACKUP_DATA = {
     },
 };
 
+const BACKUP_INFO = {
+    algorithm: "m.megolm_backup.v1",
+    version: 1,
+    auth_data: {
+        public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+    },
+};
+
+const keys = {};
+
+function getCrossSigningKey(type) {
+    return keys[type];
+}
+
+function saveCrossSigningKeys(k) {
+    Object.assign(keys, k);
+}
+
 function makeTestClient(sessionStore, cryptoStore) {
     const scheduler = [
         "getQueueForEvent", "queueEvent", "removeEventFromQueue",
         "setProcessFunction",
-    ].reduce((r, k) => {r[k] = expect.createSpy(); return r;}, {});
+    ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
     const store = [
         "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
         "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom",
         "storeUser", "getFilterIdByName", "setFilterIdByName", "getFilter",
         "storeFilter", "getSyncAccumulator", "startup", "deleteAllData",
-    ].reduce((r, k) => {r[k] = expect.createSpy(); return r;}, {});
-    store.getSavedSync = expect.createSpy().andReturn(Promise.resolve(null));
-    store.getSavedSyncToken = expect.createSpy().andReturn(Promise.resolve(null));
-    store.setSyncData = expect.createSpy().andReturn(Promise.resolve(null));
+    ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
+    store.getSavedSync = jest.fn().mockReturnValue(Promise.resolve(null));
+    store.getSavedSyncToken = jest.fn().mockReturnValue(Promise.resolve(null));
+    store.setSyncData = jest.fn().mockReturnValue(Promise.resolve(null));
     return new MatrixClient({
         baseUrl: "https://my.home.server",
         idBaseUrl: "https://identity.server",
@@ -102,15 +115,20 @@ function makeTestClient(sessionStore, cryptoStore) {
         deviceId: "device",
         sessionStore: sessionStore,
         cryptoStore: cryptoStore,
+        cryptoCallbacks: { getCrossSigningKey, saveCrossSigningKeys },
     });
 }
 
 describe("MegolmBackup", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 
+    beforeAll(function() {
+        return Olm.init();
+    });
+
     let olmDevice;
     let mockOlmLib;
     let mockCrypto;
@@ -119,33 +137,24 @@ describe("MegolmBackup", function() {
     let cryptoStore;
     let megolmDecryption;
     beforeEach(async function() {
-        await Olm.init();
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
-
         mockCrypto = testUtils.mock(Crypto, 'Crypto');
         mockCrypto.backupKey = new Olm.PkEncryption();
         mockCrypto.backupKey.set_recipient_key(
             "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
         );
-        mockCrypto.backupInfo = {
-            algorithm: "m.megolm_backup.v1",
-            version: 1,
-            auth_data: {
-                public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
-            },
-        };
+        mockCrypto.backupInfo = BACKUP_INFO;
 
         mockStorage = new MockStorageApi();
         sessionStore = new WebStorageSessionStore(mockStorage);
         cryptoStore = new MemoryCryptoStore(mockStorage);
 
-        olmDevice = new OlmDevice(sessionStore, cryptoStore);
+        olmDevice = new OlmDevice(cryptoStore);
 
         // we stub out the olm encryption bits
         mockOlmLib = {};
-        mockOlmLib.ensureOlmSessionsForDevices = expect.createSpy();
+        mockOlmLib.ensureOlmSessionsForDevices = jest.fn();
         mockOlmLib.encryptMessageForDevice =
-            expect.createSpy().andReturn(Promise.resolve());
+            jest.fn().mockResolvedValue(undefined);
     });
 
     describe("backup", function() {
@@ -206,7 +215,7 @@ describe("MegolmBackup", function() {
             };
             mockCrypto.cancelRoomKeyRequest = function() {};
 
-            mockCrypto.backupGroupSession = expect.createSpy();
+            mockCrypto.backupGroupSession = jest.fn();
 
             return event.attemptDecryption(mockCrypto).then(() => {
                 return megolmDecryption.onRoomKeyEvent(event);
@@ -267,7 +276,7 @@ describe("MegolmBackup", function() {
                             callback, method, path, queryParams, data, opts,
                         ) {
                             ++numCalls;
-                            expect(numCalls).toBeLessThanOrEqualTo(1);
+                            expect(numCalls).toBeLessThanOrEqual(1);
                             if (numCalls >= 2) {
                                 // exit out of retry loop if there's something wrong
                                 reject(new Error("authedRequest called too many timmes"));
@@ -276,8 +285,8 @@ describe("MegolmBackup", function() {
                             expect(method).toBe("PUT");
                             expect(path).toBe("/room_keys/keys");
                             expect(queryParams.version).toBe(1);
-                            expect(data.rooms[ROOM_ID].sessions).toExist();
-                            expect(data.rooms[ROOM_ID].sessions).toIncludeKey(
+                            expect(data.rooms[ROOM_ID].sessions).toBeDefined();
+                            expect(data.rooms[ROOM_ID].sessions).toHaveProperty(
                                 groupSession.session_id(),
                             );
                             resolve();
@@ -296,6 +305,71 @@ describe("MegolmBackup", function() {
                 });
         });
 
+        it('signs backups with the cross-signing master key', async function() {
+            const groupSession = new Olm.OutboundGroupSession();
+            groupSession.create();
+            const ibGroupSession = new Olm.InboundGroupSession();
+            ibGroupSession.create(groupSession.session_key());
+
+            const client = makeTestClient(sessionStore, cryptoStore);
+
+            megolmDecryption = new MegolmDecryption({
+                userId: '@user:id',
+                crypto: mockCrypto,
+                olmDevice: olmDevice,
+                baseApis: client,
+                roomId: ROOM_ID,
+            });
+
+            megolmDecryption.olmlib = mockOlmLib;
+
+            await client.initCrypto();
+            let privateKeys;
+            client.uploadDeviceSigningKeys = async function(e) {return;};
+            client.uploadKeySignatures = async function(e) {return;};
+            client.on("crossSigning.saveCrossSigningKeys", function(e) {
+                privateKeys = e;
+            });
+            client.on("crossSigning.getKey", function(e) {
+                e.done(privateKeys[e.type]);
+            });
+            await resetCrossSigningKeys(client);
+            let numCalls = 0;
+            await new Promise((resolve, reject) => {
+                client._http.authedRequest = function(
+                    callback, method, path, queryParams, data, opts,
+                ) {
+                    ++numCalls;
+                    expect(numCalls).toBeLessThanOrEqual(1);
+                    if (numCalls >= 2) {
+                        // exit out of retry loop if there's something wrong
+                        reject(new Error("authedRequest called too many timmes"));
+                        return Promise.resolve({});
+                    }
+                    expect(method).toBe("POST");
+                    expect(path).toBe("/room_keys/version");
+                    try {
+                        // make sure auth_data is signed by the master key
+                        olmlib.pkVerify(
+                            data.auth_data, client.getCrossSigningId(), "@alice:bar",
+                        );
+                    } catch (e) {
+                        reject(e);
+                        return Promise.resolve({});
+                    }
+                    resolve();
+                    return Promise.resolve({});
+                };
+                client.createKeyBackupVersion({
+                    algorithm: "m.megolm_backup.v1",
+                    auth_data: {
+                        public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+                    },
+                });
+            });
+            expect(numCalls).toBe(1);
+        });
+
         it('retries when a backup fails', function() {
             const groupSession = new Olm.OutboundGroupSession();
             groupSession.create();
@@ -305,16 +379,16 @@ describe("MegolmBackup", function() {
             const scheduler = [
                 "getQueueForEvent", "queueEvent", "removeEventFromQueue",
                 "setProcessFunction",
-            ].reduce((r, k) => {r[k] = expect.createSpy(); return r;}, {});
+            ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
             const store = [
                 "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
                 "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom",
                 "storeUser", "getFilterIdByName", "setFilterIdByName", "getFilter",
                 "storeFilter", "getSyncAccumulator", "startup", "deleteAllData",
-            ].reduce((r, k) => {r[k] = expect.createSpy(); return r;}, {});
-            store.getSavedSync = expect.createSpy().andReturn(Promise.resolve(null));
-            store.getSavedSyncToken = expect.createSpy().andReturn(Promise.resolve(null));
-            store.setSyncData = expect.createSpy().andReturn(Promise.resolve(null));
+            ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
+            store.getSavedSync = jest.fn().mockReturnValue(Promise.resolve(null));
+            store.getSavedSyncToken = jest.fn().mockReturnValue(Promise.resolve(null));
+            store.setSyncData = jest.fn().mockReturnValue(Promise.resolve(null));
             const client = new MatrixClient({
                 baseUrl: "https://my.home.server",
                 idBaseUrl: "https://identity.server",
@@ -372,7 +446,7 @@ describe("MegolmBackup", function() {
                             callback, method, path, queryParams, data, opts,
                         ) {
                             ++numCalls;
-                            expect(numCalls).toBeLessThanOrEqualTo(2);
+                            expect(numCalls).toBeLessThanOrEqual(2);
                             if (numCalls >= 3) {
                                 // exit out of retry loop if there's something wrong
                                 reject(new Error("authedRequest called too many timmes"));
@@ -381,8 +455,8 @@ describe("MegolmBackup", function() {
                             expect(method).toBe("PUT");
                             expect(path).toBe("/room_keys/keys");
                             expect(queryParams.version).toBe(1);
-                            expect(data.rooms[ROOM_ID].sessions).toExist();
-                            expect(data.rooms[ROOM_ID].sessions).toIncludeKey(
+                            expect(data.rooms[ROOM_ID].sessions).toBeDefined();
+                            expect(data.rooms[ROOM_ID].sessions).toHaveProperty(
                                 groupSession.session_id(),
                             );
                             if (numCalls > 1) {
@@ -439,10 +513,12 @@ describe("MegolmBackup", function() {
                 "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d",
                 ROOM_ID,
                 SESSION_ID,
+                BACKUP_INFO,
             ).then(() => {
                 return megolmDecryption.decryptEvent(ENCRYPTED_EVENT);
             }).then((res) => {
                 expect(res.clearEvent.content).toEqual('testytest');
+                expect(res.untrusted).toBeTruthy(); // keys from backup are untrusted
             });
         });
 
@@ -460,11 +536,38 @@ describe("MegolmBackup", function() {
             };
             return client.restoreKeyBackupWithRecoveryKey(
                 "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d",
+                null, null, BACKUP_INFO,
             ).then(() => {
                 return megolmDecryption.decryptEvent(ENCRYPTED_EVENT);
             }).then((res) => {
                 expect(res.clearEvent.content).toEqual('testytest');
             });
         });
+
+        it('has working cache functions', async function() {
+            const key = Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]);
+            await client._crypto.storeSessionBackupPrivateKey(key);
+            const result = await client._crypto.getSessionBackupPrivateKey();
+            expect(new Uint8Array(result)).toEqual(key);
+        });
+
+        it('caches session backup keys as it encounters them', async function() {
+            const cachedNull = await client._crypto.getSessionBackupPrivateKey();
+            expect(cachedNull).toBeNull();
+            client._http.authedRequest = function() {
+                return Promise.resolve(KEY_BACKUP_DATA);
+            };
+            await new Promise((resolve) => {
+                client.restoreKeyBackupWithRecoveryKey(
+                    "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d",
+                    ROOM_ID,
+                    SESSION_ID,
+                    BACKUP_INFO,
+                    { cacheCompleteCallback: resolve },
+                );
+            });
+            const cachedKey = await client._crypto.getSessionBackupPrivateKey();
+            expect(cachedKey).not.toBeNull();
+        });
     });
 });

spec/unit/crypto/cross-signing.spec.js

@@ -0,0 +1,855 @@
+/*
+Copyright 2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import '../../olm-loader';
+import anotherjson from 'another-json';
+import * as olmlib from "../../../src/crypto/olmlib";
+import {TestClient} from '../../TestClient';
+import {HttpResponse, setHttpResponses} from '../../test-utils';
+import { resetCrossSigningKeys } from "./crypto-utils";
+import { MatrixError } from '../../../src/http-api';
+import {logger} from '../../../src/logger';
+
+async function makeTestClient(userInfo, options, keys) {
+    if (!keys) keys = {};
+
+    function getCrossSigningKey(type) {
+        return keys[type];
+    }
+
+    function saveCrossSigningKeys(k) {
+        Object.assign(keys, k);
+    }
+
+    if (!options) options = {};
+    options.cryptoCallbacks = Object.assign(
+        {}, { getCrossSigningKey, saveCrossSigningKeys }, options.cryptoCallbacks || {},
+    );
+    const client = (new TestClient(
+        userInfo.userId, userInfo.deviceId, undefined, undefined, options,
+    )).client;
+
+    await client.initCrypto();
+
+    return client;
+}
+
+describe("Cross Signing", function() {
+    if (!global.Olm) {
+        logger.warn('Not running megolm backup unit tests: libolm not present');
+        return;
+    }
+
+    beforeAll(function() {
+        return global.Olm.init();
+    });
+
+    it("should sign the master key with the device key", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = jest.fn(async (auth, keys) => {
+            await olmlib.verifySignature(
+                alice._crypto._olmDevice, keys.master_key, "@alice:example.com",
+                "Osborne2", alice._crypto._olmDevice.deviceEd25519Key,
+            );
+        });
+        alice.uploadKeySignatures = async () => {};
+        alice.setAccountData = async () => {};
+        alice.getAccountDataFromServer = async () => {};
+        // set Alice's cross-signing key
+        await alice.bootstrapCrossSigning({
+            authUploadDeviceSigningKeys: async func => await func({}),
+        });
+        expect(alice.uploadDeviceSigningKeys).toHaveBeenCalled();
+    });
+
+    it("should abort bootstrap if device signing auth fails", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = async (auth, keys) => {
+            const errorResponse = {
+                session: "sessionId",
+                flows: [
+                    {
+                        stages: [
+                            "m.login.password",
+                        ],
+                    },
+                ],
+                params: {},
+            };
+
+            // If we're not just polling for flows, add on error rejecting the
+            // auth attempt.
+            if (auth) {
+                Object.assign(errorResponse, {
+                    completed: [],
+                    error: "Invalid password",
+                    errcode: "M_FORBIDDEN",
+                });
+            }
+
+            const error = new MatrixError(errorResponse);
+            error.httpStatus == 401;
+            throw error;
+        };
+        alice.uploadKeySignatures = async () => {};
+        alice.setAccountData = async () => {};
+        alice.getAccountDataFromServer = async () => { };
+        const authUploadDeviceSigningKeys = async func => await func({});
+
+        // Try bootstrap, expecting `authUploadDeviceSigningKeys` to pass
+        // through failure, stopping before actually applying changes.
+        let bootstrapDidThrow = false;
+        try {
+            await alice.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys,
+            });
+        } catch (e) {
+            if (e.errcode === "M_FORBIDDEN") {
+                bootstrapDidThrow = true;
+            }
+        }
+        expect(bootstrapDidThrow).toBeTruthy();
+    });
+
+    it("should upload a signature when a user is verified", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = async () => {};
+        alice.uploadKeySignatures = async () => {};
+        // set Alice's cross-signing key
+        await resetCrossSigningKeys(alice);
+        // Alice downloads Bob's device key
+        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+            keys: {
+                master: {
+                    user_id: "@bob:example.com",
+                    usage: ["master"],
+                    keys: {
+                        "ed25519:bobs+master+pubkey": "bobs+master+pubkey",
+                    },
+                },
+            },
+        });
+        // Alice verifies Bob's key
+        const promise = new Promise((resolve, reject) => {
+            alice.uploadKeySignatures = (...args) => {
+                resolve(...args);
+            };
+        });
+        await alice.setDeviceVerified("@bob:example.com", "bobs+master+pubkey", true);
+        // Alice should send a signature of Bob's key to the server
+        await promise;
+    });
+
+    it("should get cross-signing keys from sync", async function() {
+        const masterKey = new Uint8Array([
+            0xda, 0x5a, 0x27, 0x60, 0xe3, 0x3a, 0xc5, 0x82,
+            0x9d, 0x12, 0xc3, 0xbe, 0xe8, 0xaa, 0xc2, 0xef,
+            0xae, 0xb1, 0x05, 0xc1, 0xe7, 0x62, 0x78, 0xa6,
+            0xd7, 0x1f, 0xf8, 0x2c, 0x51, 0x85, 0xf0, 0x1d,
+        ]);
+        const selfSigningKey = new Uint8Array([
+            0x1e, 0xf4, 0x01, 0x6d, 0x4f, 0xa1, 0x73, 0x66,
+            0x6b, 0xf8, 0x93, 0xf5, 0xb0, 0x4d, 0x17, 0xc0,
+            0x17, 0xb5, 0xa5, 0xf6, 0x59, 0x11, 0x8b, 0x49,
+            0x34, 0xf2, 0x4b, 0x64, 0x9b, 0x52, 0xf8, 0x5f,
+        ]);
+
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            {
+                cryptoCallbacks: {
+                    // will be called to sign our own device
+                    getCrossSigningKey: type => {
+                        if (type === 'master') {
+                            return masterKey;
+                        } else {
+                            return selfSigningKey;
+                        }
+                    },
+                },
+            },
+        );
+
+        const keyChangePromise = new Promise((resolve, reject) => {
+            alice.once("crossSigning.keysChanged", async (e) => {
+                resolve(e);
+                await alice.checkOwnCrossSigningTrust();
+            });
+        });
+
+        const uploadSigsPromise = new Promise((resolve, reject) => {
+            alice.uploadKeySignatures = jest.fn(async (content) => {
+                await olmlib.verifySignature(
+                    alice._crypto._olmDevice,
+                    content["@alice:example.com"][
+                        "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
+                    ],
+                    "@alice:example.com",
+                    "Osborne2", alice._crypto._olmDevice.deviceEd25519Key,
+                );
+                olmlib.pkVerify(
+                    content["@alice:example.com"]["Osborne2"],
+                    "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ",
+                    "@alice:example.com",
+                );
+                resolve();
+            });
+        });
+
+        const deviceInfo = alice._crypto._deviceList._devices["@alice:example.com"]
+            .Osborne2;
+        const aliceDevice = {
+            user_id: "@alice:example.com",
+            device_id: "Osborne2",
+        };
+        aliceDevice.keys = deviceInfo.keys;
+        aliceDevice.algorithms = deviceInfo.algorithms;
+        await alice._crypto._signObject(aliceDevice);
+        olmlib.pkSign(aliceDevice, selfSigningKey, "@alice:example.com");
+
+        // feed sync result that includes master key, ssk, device key
+        const responses = [
+            HttpResponse.PUSH_RULES_RESPONSE,
+            {
+                method: "POST",
+                path: "/keys/upload",
+                data: {
+                    one_time_key_counts: {
+                        curve25519: 100,
+                        signed_curve25519: 100,
+                    },
+                },
+            },
+            HttpResponse.filterResponse("@alice:example.com"),
+            {
+                method: "GET",
+                path: "/sync",
+                data: {
+                    next_batch: "abcdefg",
+                    device_lists: {
+                        changed: [
+                            "@alice:example.com",
+                            "@bob:example.com",
+                        ],
+                    },
+                },
+            },
+            {
+                method: "POST",
+                path: "/keys/query",
+                data: {
+                    "failures": {},
+                    "device_keys": {
+                        "@alice:example.com": {
+                            "Osborne2": aliceDevice,
+                        },
+                    },
+                    "master_keys": {
+                        "@alice:example.com": {
+                            user_id: "@alice:example.com",
+                            usage: ["master"],
+                            keys: {
+                                "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk":
+                                "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk",
+                            },
+                        },
+                    },
+                    "self_signing_keys": {
+                        "@alice:example.com": {
+                            user_id: "@alice:example.com",
+                            usage: ["self-signing"],
+                            keys: {
+                                "ed25519:EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ":
+                                "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ",
+                            },
+                            signatures: {
+                                "@alice:example.com": {
+                                    "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk":
+                                    "Wqx/HXR851KIi8/u/UX+fbAMtq9Uj8sr8FsOcqrLfVYa6lAmbXs"
+                                    + "Vhfy4AlZ3dnEtjgZx0U0QDrghEn2eYBeOCA",
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+            {
+                method: "POST",
+                path: "/keys/upload",
+                data: {
+                    one_time_key_counts: {
+                        curve25519: 100,
+                        signed_curve25519: 100,
+                    },
+                },
+            },
+        ];
+        setHttpResponses(alice, responses, true, true);
+
+        await alice.startClient();
+
+        // once ssk is confirmed, device key should be trusted
+        await keyChangePromise;
+        await uploadSigsPromise;
+
+        const aliceTrust = alice.checkUserTrust("@alice:example.com");
+        expect(aliceTrust.isCrossSigningVerified()).toBeTruthy();
+        expect(aliceTrust.isTofu()).toBeTruthy();
+        expect(aliceTrust.isVerified()).toBeTruthy();
+
+        const aliceDeviceTrust = alice.checkDeviceTrust("@alice:example.com", "Osborne2");
+        expect(aliceDeviceTrust.isCrossSigningVerified()).toBeTruthy();
+        expect(aliceDeviceTrust.isLocallyVerified()).toBeTruthy();
+        expect(aliceDeviceTrust.isTofu()).toBeTruthy();
+        expect(aliceDeviceTrust.isVerified()).toBeTruthy();
+    });
+
+    it("should use trust chain to determine device verification", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = async () => {};
+        alice.uploadKeySignatures = async () => {};
+        // set Alice's cross-signing key
+        await resetCrossSigningKeys(alice);
+        // Alice downloads Bob's ssk and device key
+        const bobMasterSigning = new global.Olm.PkSigning();
+        const bobMasterPrivkey = bobMasterSigning.generate_seed();
+        const bobMasterPubkey = bobMasterSigning.init_with_seed(bobMasterPrivkey);
+        const bobSigning = new global.Olm.PkSigning();
+        const bobPrivkey = bobSigning.generate_seed();
+        const bobPubkey = bobSigning.init_with_seed(bobPrivkey);
+        const bobSSK = {
+            user_id: "@bob:example.com",
+            usage: ["self_signing"],
+            keys: {
+                ["ed25519:" + bobPubkey]: bobPubkey,
+            },
+        };
+        const sskSig = bobMasterSigning.sign(anotherjson.stringify(bobSSK));
+        bobSSK.signatures = {
+            "@bob:example.com": {
+                ["ed25519:" + bobMasterPubkey]: sskSig,
+            },
+        };
+        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+            keys: {
+                master: {
+                    user_id: "@bob:example.com",
+                    usage: ["master"],
+                    keys: {
+                        ["ed25519:" + bobMasterPubkey]: bobMasterPubkey,
+                    },
+                },
+                self_signing: bobSSK,
+            },
+            firstUse: 1,
+            unsigned: {},
+        });
+        const bobDevice = {
+            user_id: "@bob:example.com",
+            device_id: "Dynabook",
+            algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
+            keys: {
+                "curve25519:Dynabook": "somePubkey",
+                "ed25519:Dynabook": "someOtherPubkey",
+            },
+        };
+        const sig = bobSigning.sign(anotherjson.stringify(bobDevice));
+        bobDevice.signatures = {
+            "@bob:example.com": {
+                ["ed25519:" + bobPubkey]: sig,
+            },
+        };
+        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+            Dynabook: bobDevice,
+        });
+        // Bob's device key should be TOFU
+        const bobTrust = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust.isVerified()).toBeFalsy();
+        expect(bobTrust.isTofu()).toBeTruthy();
+
+        const bobDeviceTrust = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust.isVerified()).toBeFalsy();
+        expect(bobDeviceTrust.isTofu()).toBeTruthy();
+
+        // Alice verifies Bob's SSK
+        alice.uploadKeySignatures = () => {};
+        await alice.setDeviceVerified("@bob:example.com", bobMasterPubkey, true);
+
+        // Bob's device key should be trusted
+        const bobTrust2 = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust2.isCrossSigningVerified()).toBeTruthy();
+        expect(bobTrust2.isTofu()).toBeTruthy();
+
+        const bobDeviceTrust2 = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust2.isCrossSigningVerified()).toBeTruthy();
+        expect(bobDeviceTrust2.isLocallyVerified()).toBeFalsy();
+        expect(bobDeviceTrust2.isTofu()).toBeTruthy();
+    });
+
+    it("should trust signatures received from other devices", async function() {
+        const aliceKeys = {};
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            null,
+            aliceKeys,
+        );
+        alice._crypto._deviceList.startTrackingDeviceList("@bob:example.com");
+        alice._crypto._deviceList.stopTrackingAllDeviceLists = () => {};
+        alice.uploadDeviceSigningKeys = async () => {};
+        alice.uploadKeySignatures = async () => {};
+
+        // set Alice's cross-signing key
+        await resetCrossSigningKeys(alice);
+
+        const selfSigningKey = new Uint8Array([
+            0x1e, 0xf4, 0x01, 0x6d, 0x4f, 0xa1, 0x73, 0x66,
+            0x6b, 0xf8, 0x93, 0xf5, 0xb0, 0x4d, 0x17, 0xc0,
+            0x17, 0xb5, 0xa5, 0xf6, 0x59, 0x11, 0x8b, 0x49,
+            0x34, 0xf2, 0x4b, 0x64, 0x9b, 0x52, 0xf8, 0x5f,
+        ]);
+
+        const keyChangePromise = new Promise((resolve, reject) => {
+            alice._crypto._deviceList.once("userCrossSigningUpdated", (userId) => {
+                if (userId === "@bob:example.com") {
+                    resolve();
+                }
+            });
+        });
+
+        const deviceInfo = alice._crypto._deviceList._devices["@alice:example.com"]
+            .Osborne2;
+        const aliceDevice = {
+            user_id: "@alice:example.com",
+            device_id: "Osborne2",
+        };
+        aliceDevice.keys = deviceInfo.keys;
+        aliceDevice.algorithms = deviceInfo.algorithms;
+        await alice._crypto._signObject(aliceDevice);
+
+        const bobOlmAccount = new global.Olm.Account();
+        bobOlmAccount.create();
+        const bobKeys = JSON.parse(bobOlmAccount.identity_keys());
+        const bobDevice = {
+            user_id: "@bob:example.com",
+            device_id: "Dynabook",
+            algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+            keys: {
+                "ed25519:Dynabook": bobKeys.ed25519,
+                "curve25519:Dynabook": bobKeys.curve25519,
+            },
+        };
+        const deviceStr = anotherjson.stringify(bobDevice);
+        bobDevice.signatures = {
+            "@bob:example.com": {
+                "ed25519:Dynabook": bobOlmAccount.sign(deviceStr),
+            },
+        };
+        olmlib.pkSign(bobDevice, selfSigningKey, "@bob:example.com");
+
+        const bobMaster = {
+            user_id: "@bob:example.com",
+            usage: ["master"],
+            keys: {
+                "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk":
+                "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk",
+            },
+        };
+        olmlib.pkSign(bobMaster, aliceKeys.user_signing, "@alice:example.com");
+
+        // Alice downloads Bob's keys
+        // - device key
+        // - ssk
+        // - master key signed by her usk (pretend that it was signed by another
+        //   of Alice's devices)
+        const responses = [
+            HttpResponse.PUSH_RULES_RESPONSE,
+            {
+                method: "POST",
+                path: "/keys/upload",
+                data: {
+                    one_time_key_counts: {
+                        curve25519: 100,
+                        signed_curve25519: 100,
+                    },
+                },
+            },
+            HttpResponse.filterResponse("@alice:example.com"),
+            {
+                method: "GET",
+                path: "/sync",
+                data: {
+                    next_batch: "abcdefg",
+                    device_lists: {
+                        changed: [
+                            "@bob:example.com",
+                        ],
+                    },
+                },
+            },
+            {
+                method: "POST",
+                path: "/keys/query",
+                data: {
+                    "failures": {},
+                    "device_keys": {
+                        "@alice:example.com": {
+                            "Osborne2": aliceDevice,
+                        },
+                        "@bob:example.com": {
+                            "Dynabook": bobDevice,
+                        },
+                    },
+                    "master_keys": {
+                        "@bob:example.com": bobMaster,
+                    },
+                    "self_signing_keys": {
+                        "@bob:example.com": {
+                            user_id: "@bob:example.com",
+                            usage: ["self-signing"],
+                            keys: {
+                                "ed25519:EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ":
+                                "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ",
+                            },
+                            signatures: {
+                                "@bob:example.com": {
+                                    "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk":
+                                    "2KLiufImvEbfJuAFvsaZD+PsL8ELWl7N1u9yr/9hZvwRghBfQMB"
+                                    + "LAI86b1kDV9+Cq1lt85ykReeCEzmTEPY2BQ",
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+            {
+                method: "POST",
+                path: "/keys/upload",
+                data: {
+                    one_time_key_counts: {
+                        curve25519: 100,
+                        signed_curve25519: 100,
+                    },
+                },
+            },
+        ];
+        setHttpResponses(alice, responses);
+
+        await alice.startClient();
+
+        await keyChangePromise;
+
+        // Bob's device key should be trusted
+        const bobTrust = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust.isCrossSigningVerified()).toBeTruthy();
+        expect(bobTrust.isTofu()).toBeTruthy();
+
+        const bobDeviceTrust = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust.isCrossSigningVerified()).toBeTruthy();
+        expect(bobDeviceTrust.isLocallyVerified()).toBeFalsy();
+        expect(bobDeviceTrust.isTofu()).toBeTruthy();
+    });
+
+    it("should dis-trust an unsigned device", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = async () => {};
+        alice.uploadKeySignatures = async () => {};
+        // set Alice's cross-signing key
+        await resetCrossSigningKeys(alice);
+        // Alice downloads Bob's ssk and device key
+        // (NOTE: device key is not signed by ssk)
+        const bobMasterSigning = new global.Olm.PkSigning();
+        const bobMasterPrivkey = bobMasterSigning.generate_seed();
+        const bobMasterPubkey = bobMasterSigning.init_with_seed(bobMasterPrivkey);
+        const bobSigning = new global.Olm.PkSigning();
+        const bobPrivkey = bobSigning.generate_seed();
+        const bobPubkey = bobSigning.init_with_seed(bobPrivkey);
+        const bobSSK = {
+            user_id: "@bob:example.com",
+            usage: ["self_signing"],
+            keys: {
+                ["ed25519:" + bobPubkey]: bobPubkey,
+            },
+        };
+        const sskSig = bobMasterSigning.sign(anotherjson.stringify(bobSSK));
+        bobSSK.signatures = {
+            "@bob:example.com": {
+                ["ed25519:" + bobMasterPubkey]: sskSig,
+            },
+        };
+        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+            keys: {
+                master: {
+                    user_id: "@bob:example.com",
+                    usage: ["master"],
+                    keys: {
+                        ["ed25519:" + bobMasterPubkey]: bobMasterPubkey,
+                    },
+                },
+                self_signing: bobSSK,
+            },
+            firstUse: 1,
+            unsigned: {},
+        });
+        const bobDevice = {
+            user_id: "@bob:example.com",
+            device_id: "Dynabook",
+            algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
+            keys: {
+                "curve25519:Dynabook": "somePubkey",
+                "ed25519:Dynabook": "someOtherPubkey",
+            },
+        };
+        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+            Dynabook: bobDevice,
+        });
+        // Bob's device key should be untrusted
+        const bobDeviceTrust = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust.isVerified()).toBeFalsy();
+        expect(bobDeviceTrust.isTofu()).toBeFalsy();
+
+        // Alice verifies Bob's SSK
+        await alice.setDeviceVerified("@bob:example.com", bobMasterPubkey, true);
+
+        // Bob's device key should be untrusted
+        const bobDeviceTrust2 = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust2.isVerified()).toBeFalsy();
+        expect(bobDeviceTrust2.isTofu()).toBeFalsy();
+    });
+
+    it("should dis-trust a user when their ssk changes", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = async () => {};
+        alice.uploadKeySignatures = async () => {};
+        await resetCrossSigningKeys(alice);
+        // Alice downloads Bob's keys
+        const bobMasterSigning = new global.Olm.PkSigning();
+        const bobMasterPrivkey = bobMasterSigning.generate_seed();
+        const bobMasterPubkey = bobMasterSigning.init_with_seed(bobMasterPrivkey);
+        const bobSigning = new global.Olm.PkSigning();
+        const bobPrivkey = bobSigning.generate_seed();
+        const bobPubkey = bobSigning.init_with_seed(bobPrivkey);
+        const bobSSK = {
+            user_id: "@bob:example.com",
+            usage: ["self_signing"],
+            keys: {
+                ["ed25519:" + bobPubkey]: bobPubkey,
+            },
+        };
+        const sskSig = bobMasterSigning.sign(anotherjson.stringify(bobSSK));
+        bobSSK.signatures = {
+            "@bob:example.com": {
+                ["ed25519:" + bobMasterPubkey]: sskSig,
+            },
+        };
+        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+            keys: {
+                master: {
+                    user_id: "@bob:example.com",
+                    usage: ["master"],
+                    keys: {
+                        ["ed25519:" + bobMasterPubkey]: bobMasterPubkey,
+                    },
+                },
+                self_signing: bobSSK,
+            },
+            firstUse: 1,
+            unsigned: {},
+        });
+        const bobDevice = {
+            user_id: "@bob:example.com",
+            device_id: "Dynabook",
+            algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
+            keys: {
+                "curve25519:Dynabook": "somePubkey",
+                "ed25519:Dynabook": "someOtherPubkey",
+            },
+        };
+        const bobDeviceString = anotherjson.stringify(bobDevice);
+        const sig = bobSigning.sign(bobDeviceString);
+        bobDevice.signatures = {};
+        bobDevice.signatures["@bob:example.com"] = {};
+        bobDevice.signatures["@bob:example.com"]["ed25519:" + bobPubkey] = sig;
+        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+            Dynabook: bobDevice,
+        });
+        // Alice verifies Bob's SSK
+        alice.uploadKeySignatures = () => {};
+        await alice.setDeviceVerified("@bob:example.com", bobMasterPubkey, true);
+
+        // Bob's device key should be trusted
+        const bobDeviceTrust = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust.isVerified()).toBeTruthy();
+        expect(bobDeviceTrust.isTofu()).toBeTruthy();
+
+        // Alice downloads new SSK for Bob
+        const bobMasterSigning2 = new global.Olm.PkSigning();
+        const bobMasterPrivkey2 = bobMasterSigning2.generate_seed();
+        const bobMasterPubkey2 = bobMasterSigning2.init_with_seed(bobMasterPrivkey2);
+        const bobSigning2 = new global.Olm.PkSigning();
+        const bobPrivkey2 = bobSigning2.generate_seed();
+        const bobPubkey2 = bobSigning2.init_with_seed(bobPrivkey2);
+        const bobSSK2 = {
+            user_id: "@bob:example.com",
+            usage: ["self_signing"],
+            keys: {
+                ["ed25519:" + bobPubkey2]: bobPubkey2,
+            },
+        };
+        const sskSig2 = bobMasterSigning2.sign(anotherjson.stringify(bobSSK2));
+        bobSSK2.signatures = {
+            "@bob:example.com": {
+                ["ed25519:" + bobMasterPubkey2]: sskSig2,
+            },
+        };
+        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+            keys: {
+                master: {
+                    user_id: "@bob:example.com",
+                    usage: ["master"],
+                    keys: {
+                        ["ed25519:" + bobMasterPubkey2]: bobMasterPubkey2,
+                    },
+                },
+                self_signing: bobSSK2,
+            },
+            firstUse: 0,
+            unsigned: {},
+        });
+        // Bob's and his device should be untrusted
+        const bobTrust = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust.isVerified()).toBeFalsy();
+        expect(bobTrust.isTofu()).toBeFalsy();
+
+        const bobDeviceTrust2 = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust2.isVerified()).toBeFalsy();
+        expect(bobDeviceTrust2.isTofu()).toBeFalsy();
+
+        // Alice verifies Bob's SSK
+        alice.uploadKeySignatures = () => {};
+        await alice.setDeviceVerified("@bob:example.com", bobMasterPubkey2, true);
+
+        // Bob should be trusted but not his device
+        const bobTrust2 = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust2.isVerified()).toBeTruthy();
+
+        const bobDeviceTrust3 = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust3.isVerified()).toBeFalsy();
+
+        // Alice gets new signature for device
+        const sig2 = bobSigning2.sign(bobDeviceString);
+        bobDevice.signatures["@bob:example.com"]["ed25519:" + bobPubkey2] = sig2;
+        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+            Dynabook: bobDevice,
+        });
+
+        // Bob's device should be trusted again (but not TOFU)
+        const bobTrust3 = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust3.isVerified()).toBeTruthy();
+
+        const bobDeviceTrust4 = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust4.isCrossSigningVerified()).toBeTruthy();
+    });
+
+    it("should offer to upgrade device verifications to cross-signing", async function() {
+        let upgradeResolveFunc;
+
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            {
+                cryptoCallbacks: {
+                    shouldUpgradeDeviceVerifications: (verifs) => {
+                        expect(verifs.users["@bob:example.com"]).toBeDefined();
+                        upgradeResolveFunc();
+                        return ["@bob:example.com"];
+                    },
+                },
+            },
+        );
+        const bob = await makeTestClient(
+            {userId: "@bob:example.com", deviceId: "Dynabook"},
+        );
+
+        bob.uploadDeviceSigningKeys = async () => {};
+        bob.uploadKeySignatures = async () => {};
+        // set Bob's cross-signing key
+        await resetCrossSigningKeys(bob);
+        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+            Dynabook: {
+                algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
+                keys: {
+                    "curve25519:Dynabook": bob._crypto._olmDevice.deviceCurve25519Key,
+                    "ed25519:Dynabook": bob._crypto._olmDevice.deviceEd25519Key,
+                },
+                verified: 1,
+                known: true,
+            },
+        });
+        alice._crypto._deviceList.storeCrossSigningForUser(
+            "@bob:example.com",
+            bob._crypto._crossSigningInfo.toStorage(),
+        );
+
+        alice.uploadDeviceSigningKeys = async () => {};
+        alice.uploadKeySignatures = async () => {};
+        // when alice sets up cross-signing, she should notice that bob's
+        // cross-signing key is signed by his Dynabook, which alice has
+        // verified, and ask if the device verification should be upgraded to a
+        // cross-signing verification
+        let upgradePromise = new Promise((resolve) => {
+            upgradeResolveFunc = resolve;
+        });
+        await resetCrossSigningKeys(alice);
+        await upgradePromise;
+
+        const bobTrust = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust.isCrossSigningVerified()).toBeTruthy();
+        expect(bobTrust.isTofu()).toBeTruthy();
+
+        // "forget" that Bob is trusted
+        delete alice._crypto._deviceList._crossSigningInfo["@bob:example.com"]
+            .keys.master.signatures["@alice:example.com"];
+
+        const bobTrust2 = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust2.isCrossSigningVerified()).toBeFalsy();
+        expect(bobTrust2.isTofu()).toBeTruthy();
+
+        upgradePromise = new Promise((resolve) => {
+            upgradeResolveFunc = resolve;
+        });
+        alice._crypto._deviceList.emit("userCrossSigningUpdated", "@bob:example.com");
+        await new Promise((resolve) => {
+            alice._crypto.on("userTrustStatusChanged", resolve);
+        });
+        await upgradePromise;
+
+        const bobTrust3 = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust3.isCrossSigningVerified()).toBeTruthy();
+        expect(bobTrust3.isTofu()).toBeTruthy();
+    });
+});

spec/unit/crypto/crypto-utils.js

@@ -0,0 +1,44 @@
+import {IndexedDBCryptoStore} from '../../../src/crypto/store/indexeddb-crypto-store';
+
+
+// needs to be phased out and replaced with bootstrapSecretStorage,
+// but that is doing too much extra stuff for it to be an easy transition.
+export async function resetCrossSigningKeys(client, {
+    level,
+    authUploadDeviceSigningKeys = async func => await func(),
+} = {}) {
+    const crypto = client._crypto;
+
+    const oldKeys = Object.assign({}, crypto._crossSigningInfo.keys);
+    try {
+        await crypto._crossSigningInfo.resetKeys(level);
+        await crypto._signObject(crypto._crossSigningInfo.keys.master);
+        // write a copy locally so we know these are trusted keys
+        await crypto._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+            (txn) => {
+                crypto._cryptoStore.storeCrossSigningKeys(
+                    txn, crypto._crossSigningInfo.keys);
+            },
+        );
+    } catch (e) {
+        // If anything failed here, revert the keys so we know to try again from the start
+        // next time.
+        crypto._crossSigningInfo.keys = oldKeys;
+        throw e;
+    }
+    crypto._baseApis.emit("crossSigning.keysChanged", {});
+    await crypto._afterCrossSigningLocalKeyChange();
+}
+
+export async function createSecretStorageKey() {
+    const decryption = new global.Olm.PkDecryption();
+    const storagePublicKey = decryption.generate_key();
+    const storagePrivateKey = decryption.get_private_key();
+    decryption.free();
+    return {
+        // `pubkey` not used anymore with symmetric 4S
+        keyInfo: { pubkey: storagePublicKey },
+        privateKey: storagePrivateKey,
+    };
+}

spec/unit/crypto/outgoing-room-key-requests.spec.js

@@ -0,0 +1,87 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {
+    IndexedDBCryptoStore,
+} from '../../../src/crypto/store/indexeddb-crypto-store';
+import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
+import 'fake-indexeddb/auto';
+import 'jest-localstorage-mock';
+
+import {
+    ROOM_KEY_REQUEST_STATES,
+} from '../../../src/crypto/OutgoingRoomKeyRequestManager';
+
+const requests = [
+    {
+        requestId: "A",
+        requestBody: { session_id: "A", room_id: "A" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "B",
+        requestBody: { session_id: "B", room_id: "B" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "C",
+        requestBody: { session_id: "C", room_id: "C" },
+        state: ROOM_KEY_REQUEST_STATES.UNSENT,
+    },
+];
+
+describe.each([
+    ["IndexedDBCryptoStore",
+     () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+    ["LocalStorageCryptoStore",
+     () => new IndexedDBCryptoStore(undefined, "tests")],
+    ["MemoryCryptoStore", () => {
+        const store = new IndexedDBCryptoStore(undefined, "tests");
+        store._backend = new MemoryCryptoStore();
+        store._backendPromise = Promise.resolve(store._backend);
+        return store;
+    }],
+])("Outgoing room key requests [%s]", function(name, dbFactory) {
+    let store;
+
+    beforeAll(async () => {
+        store = dbFactory();
+        await store.startup();
+        await Promise.all(requests.map((request) =>
+            store.getOrAddOutgoingRoomKeyRequest(request),
+        ));
+    });
+
+    it("getAllOutgoingRoomKeyRequestsByState retrieves all entries in a given state",
+    async () => {
+        const r = await
+            store.getAllOutgoingRoomKeyRequestsByState(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(r).toHaveLength(2);
+        requests.filter((e) => e.state == ROOM_KEY_REQUEST_STATES.SENT).forEach((e) => {
+            expect(r).toContainEqual(e);
+        });
+    });
+
+    test("getOutgoingRoomKeyRequestByState retrieves any entry in a given state",
+    async () => {
+        const r =
+            await store.getOutgoingRoomKeyRequestByState([ROOM_KEY_REQUEST_STATES.SENT]);
+        expect(r).not.toBeNull();
+        expect(r).not.toBeUndefined();
+        expect(r.state).toEqual(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(requests).toContainEqual(r);
+    });
+});

spec/unit/crypto/secrets.spec.js

@@ -0,0 +1,674 @@
+/*
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import '../../olm-loader';
+import * as olmlib from "../../../src/crypto/olmlib";
+import {SECRET_STORAGE_ALGORITHM_V1_AES} from "../../../src/crypto/SecretStorage";
+import {MatrixEvent} from "../../../src/models/event";
+import {TestClient} from '../../TestClient';
+import {makeTestClients} from './verification/util';
+import {encryptAES} from "../../../src/crypto/aes";
+import {resetCrossSigningKeys, createSecretStorageKey} from "./crypto-utils";
+import {logger} from '../../../src/logger';
+
+import * as utils from "../../../src/utils";
+
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    logger.log('nodejs was compiled without crypto support');
+}
+
+async function makeTestClient(userInfo, options) {
+    const client = (new TestClient(
+        userInfo.userId, userInfo.deviceId, undefined, undefined, options,
+    )).client;
+
+    // Make it seem as if we've synced and thus the store can be trusted to
+    // contain valid account data.
+    client.isInitialSyncComplete = function() {
+        return true;
+    };
+
+    await client.initCrypto();
+
+    // No need to download keys for these tests
+    client._crypto.downloadKeys = async function() {};
+
+    return client;
+}
+
+// Wrapper around pkSign to return a signed object. pkSign returns the
+// signature, rather than the signed object.
+function sign(obj, key, userId) {
+    olmlib.pkSign(obj, key, userId);
+    return obj;
+}
+
+describe("Secrets", function() {
+    if (!global.Olm) {
+        logger.warn('Not running megolm backup unit tests: libolm not present');
+        return;
+    }
+
+    beforeAll(function() {
+        return global.Olm.init();
+    });
+
+    it("should store and retrieve a secret", async function() {
+        const key = new Uint8Array(16);
+        for (let i = 0; i < 16; i++) key[i] = i;
+
+        const signing = new global.Olm.PkSigning();
+        const signingKey = signing.generate_seed();
+        const signingPubKey = signing.init_with_seed(signingKey);
+
+        const signingkeyInfo = {
+            user_id: "@alice:example.com",
+            usage: ['master'],
+            keys: {
+                ['ed25519:' + signingPubKey]: signingPubKey,
+            },
+        };
+
+        const getKey = jest.fn(e => {
+            expect(Object.keys(e.keys)).toEqual(["abc"]);
+            return ['abc', key];
+        });
+
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            {
+                cryptoCallbacks: {
+                    getCrossSigningKey: t => signingKey,
+                    getSecretStorageKey: getKey,
+                },
+            },
+        );
+        alice._crypto._crossSigningInfo.setKeys({
+            master: signingkeyInfo,
+        });
+
+        const secretStorage = alice._crypto._secretStorage;
+
+        alice.setAccountData = async function(eventType, contents, callback) {
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                }),
+            ]);
+            if (callback) {
+                callback();
+            }
+        };
+
+        const keyAccountData = {
+            algorithm: SECRET_STORAGE_ALGORITHM_V1_AES,
+        };
+        await alice._crypto._crossSigningInfo.signObject(keyAccountData, 'master');
+
+        alice.store.storeAccountDataEvents([
+            new MatrixEvent({
+                type: "m.secret_storage.key.abc",
+                content: keyAccountData,
+            }),
+        ]);
+
+        expect(await secretStorage.isStored("foo")).toBeFalsy();
+
+        await secretStorage.store("foo", "bar", ["abc"]);
+
+        expect(await secretStorage.isStored("foo")).toBeTruthy();
+        expect(await secretStorage.get("foo")).toBe("bar");
+
+        expect(getKey).toHaveBeenCalled();
+    });
+
+    it("should throw if given a key that doesn't exist", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+
+        try {
+            await alice.storeSecret("foo", "bar", ["this secret does not exist"]);
+            // should be able to use expect(...).toThrow() but mocha still fails
+            // the test even when it throws for reasons I have no inclination to debug
+            expect(true).toBeFalsy();
+        } catch (e) {
+        }
+    });
+
+    it("should refuse to encrypt with zero keys", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+
+        try {
+            await alice.storeSecret("foo", "bar", []);
+            expect(true).toBeFalsy();
+        } catch (e) {
+        }
+    });
+
+    it("should encrypt with default key if keys is null", async function() {
+        const key = new Uint8Array(16);
+        for (let i = 0; i < 16; i++) key[i] = i;
+        const getKey = jest.fn(e => {
+            expect(Object.keys(e.keys)).toEqual([newKeyId]);
+            return [newKeyId, key];
+        });
+
+        let keys = {};
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            {
+                cryptoCallbacks: {
+                    getCrossSigningKey: t => keys[t],
+                    saveCrossSigningKeys: k => keys = k,
+                    getSecretStorageKey: getKey,
+                },
+            },
+        );
+        alice.setAccountData = async function(eventType, contents, callback) {
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                }),
+            ]);
+        };
+        resetCrossSigningKeys(alice);
+
+        const { keyId: newKeyId } = await alice.addSecretStorageKey(
+            SECRET_STORAGE_ALGORITHM_V1_AES,
+        );
+        // we don't await on this because it waits for the event to come down the sync
+        // which won't happen in the test setup
+        alice.setDefaultSecretStorageKeyId(newKeyId);
+        await alice.storeSecret("foo", "bar");
+
+        const accountData = alice.getAccountData('foo');
+        expect(accountData.getContent().encrypted).toBeTruthy();
+    });
+
+    it("should refuse to encrypt if no keys given and no default key", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+
+        try {
+            await alice.storeSecret("foo", "bar");
+            expect(true).toBeFalsy();
+        } catch (e) {
+        }
+    });
+
+    it("should request secrets from other clients", async function() {
+        const [osborne2, vax] = await makeTestClients(
+            [
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {userId: "@alice:example.com", deviceId: "VAX"},
+            ],
+            {
+                cryptoCallbacks: {
+                    onSecretRequested: (userId, deviceId, requestId, secretName, deviceTrust) => {
+                        expect(secretName).toBe("foo");
+                        return "bar";
+                    },
+                },
+            },
+        );
+
+        const vaxDevice = vax.client._crypto._olmDevice;
+        const osborne2Device = osborne2.client._crypto._olmDevice;
+        const secretStorage = osborne2.client._crypto._secretStorage;
+
+        osborne2.client._crypto._deviceList.storeDevicesForUser("@alice:example.com", {
+            "VAX": {
+                user_id: "@alice:example.com",
+                device_id: "VAX",
+                algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                keys: {
+                    "ed25519:VAX": vaxDevice.deviceEd25519Key,
+                    "curve25519:VAX": vaxDevice.deviceCurve25519Key,
+                },
+            },
+        });
+        vax.client._crypto._deviceList.storeDevicesForUser("@alice:example.com", {
+            "Osborne2": {
+                user_id: "@alice:example.com",
+                device_id: "Osborne2",
+                algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                keys: {
+                    "ed25519:Osborne2": osborne2Device.deviceEd25519Key,
+                    "curve25519:Osborne2": osborne2Device.deviceCurve25519Key,
+                },
+            },
+        });
+
+        await osborne2Device.generateOneTimeKeys(1);
+        const otks = (await osborne2Device.getOneTimeKeys()).curve25519;
+        await osborne2Device.markKeysAsPublished();
+
+        await vax.client._crypto._olmDevice.createOutboundSession(
+            osborne2Device.deviceCurve25519Key,
+            Object.values(otks)[0],
+        );
+
+        const request = await secretStorage.request("foo", ["VAX"]);
+        const secret = await request.promise;
+
+        expect(secret).toBe("bar");
+    });
+
+    describe("bootstrap", function() {
+        // keys used in some of the tests
+        const XSK = new Uint8Array(
+            olmlib.decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="),
+        );
+        const XSPubKey = "DRb8pFVJyEJ9OWvXeUoM0jq/C2Wt+NxzBZVuk2nRb+0";
+        const USK = new Uint8Array(
+            olmlib.decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="),
+        );
+        const USPubKey = "CUpoiTtHiyXpUmd+3ohb7JVxAlUaOG1NYs9Jlx8soQU";
+        const SSK = new Uint8Array(
+            olmlib.decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="),
+        );
+        const SSPubKey = "0DfNsRDzEvkCLA0gD3m7VAGJ5VClhjEsewI35xq873Q";
+        const SSSSKey = new Uint8Array(
+            olmlib.decodeBase64(
+                "XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0=",
+            ),
+        );
+
+        it("bootstraps when no storage or cross-signing keys locally", async function() {
+            const key = new Uint8Array(16);
+            for (let i = 0; i < 16; i++) key[i] = i;
+            const getKey = jest.fn(e => {
+                return [Object.keys(e.keys)[0], key];
+            });
+
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: getKey,
+                    },
+                },
+            );
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
+
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey,
+            });
+
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;
+
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
+        });
+
+        it("bootstraps when cross-signing keys in secret storage", async function() {
+            const decryption = new global.Olm.PkDecryption();
+            const storagePublicKey = decryption.generate_key();
+            const storagePrivateKey = decryption.get_private_key();
+
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: async request => {
+                            const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
+                            expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
+                            return [defaultKeyId, storagePrivateKey];
+                        },
+                    },
+                },
+            );
+
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
+            bob._crypto.checkKeyBackup = async () => {};
+
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;
+
+            // Set up cross-signing keys from scratch with specific storage key
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey: async () => ({
+                    // `pubkey` not used anymore with symmetric 4S
+                    keyInfo: { pubkey: storagePublicKey },
+                    privateKey: storagePrivateKey,
+                }),
+            });
+
+            // Clear local cross-signing keys and read from secret storage
+            bob._crypto._deviceList.storeCrossSigningForUser(
+                "@bob:example.com",
+                crossSigning.toStorage(),
+            );
+            crossSigning.keys = {};
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
+        });
+
+        it("adds passphrase checking if it's lacking", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                // we never use these values, other than checking that they
+                // exist, so just use dummy values
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            expect(alice.getAccountData("m.secret_storage.default_key").getContent())
+                .toEqual({key: "key_id"});
+            const keyInfo = alice.getAccountData("m.secret_storage.key.key_id")
+                .getContent();
+            expect(keyInfo.algorithm)
+                .toEqual("m.secret_storage.v1.aes-hmac-sha2");
+            expect(keyInfo.passphrase).toEqual({
+                algorithm: "m.pbkdf2",
+                iterations: 500000,
+                salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+            });
+            expect(keyInfo).toHaveProperty("iv");
+            expect(keyInfo).toHaveProperty("mac");
+            expect(alice.checkSecretStorageKey(secretStorageKeys.key_id, keyInfo))
+                .toBeTruthy();
+        });
+        it("fixes backup keys in the wrong format", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.megolm_backup.v1",
+                    content: {
+                        encrypted: {
+                            key_id: await encryptAES(
+                                "123,45,6,7,89,1,234,56,78,90,12,34,5,67,8,90",
+                                secretStorageKeys.key_id, "m.megolm_backup.v1",
+                            ),
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            const backupKey = alice.getAccountData("m.megolm_backup.v1")
+                .getContent();
+            expect(backupKey.encrypted).toHaveProperty("key_id");
+            expect(await alice.getSecret("m.megolm_backup.v1"))
+                .toEqual("ey0GB1kB6jhOWgwiBUMIWg==");
+        });
+    });
+});

spec/unit/crypto/verification/InRoomChannel.spec.js

@@ -0,0 +1,98 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import {InRoomChannel} from "../../../../src/crypto/verification/request/InRoomChannel";
+    "../../../../src/crypto/verification/request/ToDeviceChannel";
+import {MatrixEvent} from "../../../../src/models/event";
+
+describe("InRoomChannel tests", function() {
+    const ALICE = "@alice:hs.tld";
+    const BOB = "@bob:hs.tld";
+    const MALORY = "@malory:hs.tld";
+    const client = {
+        getUserId() { return ALICE; },
+    };
+
+    it("getEventType only returns .request for a message with a msgtype", function() {
+        const invalidEvent = new MatrixEvent({
+            type: "m.key.verification.request",
+        });
+        expect(InRoomChannel.getEventType(invalidEvent)).toStrictEqual("");
+        const validEvent = new MatrixEvent({
+            type: "m.room.message",
+            content: { msgtype: "m.key.verification.request" },
+        });
+        expect(InRoomChannel.getEventType(validEvent)).
+            toStrictEqual("m.key.verification.request");
+        const validFooEvent = new MatrixEvent({ type: "m.foo" });
+        expect(InRoomChannel.getEventType(validFooEvent)).
+            toStrictEqual("m.foo");
+    });
+
+    it("getEventType should return m.room.message for messages", function() {
+        const messageEvent = new MatrixEvent({
+            type: "m.room.message",
+            content: { msgtype: "m.text" },
+        });
+        // XXX: The event type doesn't matter too much, just as long as it's not a verification event
+        expect(InRoomChannel.getEventType(messageEvent)).
+            toStrictEqual("m.room.message");
+    });
+
+    it("getEventType should return actual type for non-message events", function() {
+        const event = new MatrixEvent({
+            type: "m.room.member",
+            content: { },
+        });
+        expect(InRoomChannel.getEventType(event)).
+            toStrictEqual("m.room.member");
+    });
+
+    it("getOtherPartyUserId should not return anything for a request not " +
+        "directed at me", function() {
+        const event = new MatrixEvent({
+            sender: BOB,
+            type: "m.room.message",
+            content: { msgtype: "m.key.verification.request", to: MALORY },
+        });
+        expect(InRoomChannel.getOtherPartyUserId(event, client)).toStrictEqual(undefined);
+    });
+
+    it("getOtherPartyUserId should not return anything an event that is not of a valid " +
+        "request type", function() {
+        // invalid because this should be a room message with msgtype
+        const invalidRequest = new MatrixEvent({
+            sender: BOB,
+            type: "m.key.verification.request",
+            content: { to: ALICE },
+        });
+        expect(InRoomChannel.getOtherPartyUserId(invalidRequest, client))
+            .toStrictEqual(undefined);
+        const startEvent = new MatrixEvent({
+            sender: BOB,
+            type: "m.key.verification.start",
+            content: { to: ALICE },
+        });
+        expect(InRoomChannel.getOtherPartyUserId(startEvent, client))
+            .toStrictEqual(undefined);
+        const fooEvent = new MatrixEvent({
+            sender: BOB,
+            type: "m.foo",
+            content: { to: ALICE },
+        });
+        expect(InRoomChannel.getOtherPartyUserId(fooEvent, client))
+            .toStrictEqual(undefined);
+    });
+});

spec/unit/crypto/verification/qr_code.spec.js

@@ -0,0 +1,41 @@
+/*
+Copyright 2018-2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import "../../../olm-loader";
+import {logger} from "../../../../src/logger";
+
+const Olm = global.Olm;
+
+describe("QR code verification", function() {
+    if (!global.Olm) {
+        logger.warn('Not running device verification tests: libolm not present');
+        return;
+    }
+
+    beforeAll(function() {
+        return Olm.init();
+    });
+
+    describe("reciprocate", () => {
+        it("should verify the secret", () => {
+            // TODO: Actually write a test for this.
+            // Tests are hard because we are running before the verification
+            // process actually begins, and are largely UI-driven rather than
+            // logic-driven (compared to something like SAS). In the interest
+            // of time, tests are currently excluded.
+        });
+    });
+});

spec/unit/crypto/verification/request.spec.js

@@ -0,0 +1,82 @@
+/*
+Copyright 2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import "../../../olm-loader";
+import {verificationMethods} from "../../../../src/crypto";
+import {logger} from "../../../../src/logger";
+import {SAS} from "../../../../src/crypto/verification/SAS";
+import {makeTestClients, setupWebcrypto, teardownWebcrypto} from './util';
+
+const Olm = global.Olm;
+
+jest.useFakeTimers();
+
+describe("verification request integration tests with crypto layer", function() {
+    if (!global.Olm) {
+        logger.warn('Not running device verification unit tests: libolm not present');
+        return;
+    }
+
+    beforeAll(function() {
+        setupWebcrypto();
+        return Olm.init();
+    });
+
+    afterAll(() => {
+        teardownWebcrypto();
+    });
+
+    it("should request and accept a verification", async function() {
+        const [alice, bob] = await makeTestClients(
+            [
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {userId: "@bob:example.com", deviceId: "Dynabook"},
+            ],
+            {
+                verificationMethods: [verificationMethods.SAS],
+            },
+        );
+        alice.client._crypto._deviceList.getRawStoredDevicesForUser = function() {
+            return {
+                Dynabook: {
+                    keys: {
+                        "ed25519:Dynabook": "bob+base64+ed25519+key",
+                    },
+                },
+            };
+        };
+        alice.client.downloadKeys = () => {
+            return Promise.resolve();
+        };
+        bob.client.downloadKeys = () => {
+            return Promise.resolve();
+        };
+        bob.client.on("crypto.verification.request", (request) => {
+            const bobVerifier = request.beginKeyVerification(verificationMethods.SAS);
+            bobVerifier.verify();
+
+            // XXX: Private function access (but it's a test, so we're okay)
+            bobVerifier._endTimer();
+        });
+        const aliceRequest = await alice.client.requestVerification("@bob:example.com");
+        await aliceRequest.waitFor(r => r.started);
+        const aliceVerifier = aliceRequest.verifier;
+        expect(aliceVerifier).toBeInstanceOf(SAS);
+
+        // XXX: Private function access (but it's a test, so we're okay)
+        aliceVerifier._endTimer();
+    });
+});

spec/unit/crypto/verification/sas.spec.js

@@ -0,0 +1,506 @@
+/*
+Copyright 2018-2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import "../../../olm-loader";
+import {makeTestClients, setupWebcrypto, teardownWebcrypto} from './util';
+import {MatrixEvent} from "../../../../src/models/event";
+import {SAS} from "../../../../src/crypto/verification/SAS";
+import {DeviceInfo} from "../../../../src/crypto/deviceinfo";
+import {verificationMethods} from "../../../../src/crypto";
+import * as olmlib from "../../../../src/crypto/olmlib";
+import {logger} from "../../../../src/logger";
+import {resetCrossSigningKeys} from "../crypto-utils";
+
+const Olm = global.Olm;
+
+let ALICE_DEVICES;
+let BOB_DEVICES;
+
+describe("SAS verification", function() {
+    if (!global.Olm) {
+        logger.warn('Not running device verification unit tests: libolm not present');
+        return;
+    }
+
+    beforeAll(function() {
+        setupWebcrypto();
+        return Olm.init();
+    });
+
+    afterAll(() => {
+        teardownWebcrypto();
+    });
+
+    it("should error on an unexpected event", async function() {
+        //channel, baseApis, userId, deviceId, startEvent, request
+        const request = {
+            onVerifierCancelled: function() {},
+        };
+        const channel = {
+            send: function() {
+                return Promise.resolve();
+            },
+        };
+        const sas = new SAS(channel, {}, "@alice:example.com", "ABCDEFG", null, request);
+        sas.handleEvent(new MatrixEvent({
+            sender: "@alice:example.com",
+            type: "es.inquisition",
+            content: {},
+        }));
+        const spy = jest.fn();
+        await sas.verify().catch(spy);
+        expect(spy).toHaveBeenCalled();
+
+        // Cancel the SAS for cleanup (we started a verification, so abort)
+        sas.cancel();
+    });
+
+    describe("verification", () => {
+        let alice;
+        let bob;
+        let aliceSasEvent;
+        let bobSasEvent;
+        let aliceVerifier;
+        let bobPromise;
+
+        beforeEach(async () => {
+            [alice, bob] = await makeTestClients(
+                [
+                    {userId: "@alice:example.com", deviceId: "Osborne2"},
+                    {userId: "@bob:example.com", deviceId: "Dynabook"},
+                ],
+                {
+                    verificationMethods: [verificationMethods.SAS],
+                },
+            );
+
+            const aliceDevice = alice.client._crypto._olmDevice;
+            const bobDevice = bob.client._crypto._olmDevice;
+
+            ALICE_DEVICES = {
+                Osborne2: {
+                    user_id: "@alice:example.com",
+                    device_id: "Osborne2",
+                    algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                    keys: {
+                        "ed25519:Osborne2": aliceDevice.deviceEd25519Key,
+                        "curve25519:Osborne2": aliceDevice.deviceCurve25519Key,
+                    },
+                },
+            };
+
+            BOB_DEVICES = {
+                Dynabook: {
+                    user_id: "@bob:example.com",
+                    device_id: "Dynabook",
+                    algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                    keys: {
+                        "ed25519:Dynabook": bobDevice.deviceEd25519Key,
+                        "curve25519:Dynabook": bobDevice.deviceCurve25519Key,
+                    },
+                },
+            };
+
+            alice.client._crypto._deviceList.storeDevicesForUser(
+                "@bob:example.com", BOB_DEVICES,
+            );
+            alice.client.downloadKeys = () => {
+                return Promise.resolve();
+            };
+
+            bob.client._crypto._deviceList.storeDevicesForUser(
+                "@alice:example.com", ALICE_DEVICES,
+            );
+            bob.client.downloadKeys = () => {
+                return Promise.resolve();
+            };
+
+            aliceSasEvent = null;
+            bobSasEvent = null;
+
+            bobPromise = new Promise((resolve, reject) => {
+                bob.client.on("crypto.verification.request", request => {
+                    request.verifier.on("show_sas", (e) => {
+                        if (!e.sas.emoji || !e.sas.decimal) {
+                            e.cancel();
+                        } else if (!aliceSasEvent) {
+                            bobSasEvent = e;
+                        } else {
+                            try {
+                                expect(e.sas).toEqual(aliceSasEvent.sas);
+                                e.confirm();
+                                aliceSasEvent.confirm();
+                            } catch (error) {
+                                e.mismatch();
+                                aliceSasEvent.mismatch();
+                            }
+                        }
+                    });
+                    resolve(request.verifier);
+                });
+            });
+
+            aliceVerifier = alice.client.beginKeyVerification(
+                verificationMethods.SAS, bob.client.getUserId(), bob.deviceId,
+            );
+            aliceVerifier.on("show_sas", (e) => {
+                if (!e.sas.emoji || !e.sas.decimal) {
+                    e.cancel();
+                } else if (!bobSasEvent) {
+                    aliceSasEvent = e;
+                } else {
+                    try {
+                        expect(e.sas).toEqual(bobSasEvent.sas);
+                        e.confirm();
+                        bobSasEvent.confirm();
+                    } catch (error) {
+                        e.mismatch();
+                        bobSasEvent.mismatch();
+                    }
+                }
+            });
+        });
+        afterEach(async () => {
+            await Promise.all([
+                alice.stop(),
+                bob.stop(),
+            ]);
+        });
+
+        it("should verify a key", async () => {
+            let macMethod;
+            let keyAgreement;
+            const origSendToDevice = bob.client.sendToDevice.bind(bob.client);
+            bob.client.sendToDevice = function(type, map) {
+                if (type === "m.key.verification.accept") {
+                    macMethod = map[alice.client.getUserId()][alice.client.deviceId]
+                        .message_authentication_code;
+                    keyAgreement = map[alice.client.getUserId()][alice.client.deviceId]
+                        .key_agreement_protocol;
+                }
+                return origSendToDevice(type, map);
+            };
+
+            alice.httpBackend.when('POST', '/keys/query').respond(200, {
+                failures: {},
+                device_keys: {
+                    "@bob:example.com": BOB_DEVICES,
+                },
+            });
+            bob.httpBackend.when('POST', '/keys/query').respond(200, {
+                failures: {},
+                device_keys: {
+                    "@alice:example.com": ALICE_DEVICES,
+                },
+            });
+
+            await Promise.all([
+                aliceVerifier.verify(),
+                bobPromise.then((verifier) => verifier.verify()),
+                alice.httpBackend.flush(),
+                bob.httpBackend.flush(),
+            ]);
+
+            // make sure that it uses the preferred method
+            expect(macMethod).toBe("hkdf-hmac-sha256");
+            expect(keyAgreement).toBe("curve25519-hkdf-sha256");
+
+            // make sure Alice and Bob verified each other
+            const bobDevice
+                  = await alice.client.getStoredDevice("@bob:example.com", "Dynabook");
+            expect(bobDevice.isVerified()).toBeTruthy();
+            const aliceDevice
+                  = await bob.client.getStoredDevice("@alice:example.com", "Osborne2");
+            expect(aliceDevice.isVerified()).toBeTruthy();
+        });
+
+        it("should be able to verify using the old MAC", async () => {
+            // pretend that Alice can only understand the old (incorrect) MAC,
+            // and make sure that she can still verify with Bob
+            let macMethod;
+            const aliceOrigSendToDevice = alice.client.sendToDevice.bind(alice.client);
+            alice.client.sendToDevice = (type, map) => {
+                if (type === "m.key.verification.start") {
+                    // Note: this modifies not only the message that Bob
+                    // receives, but also the copy of the message that Alice
+                    // has, since it is the same object.  If this does not
+                    // happen, the verification will fail due to a hash
+                    // commitment mismatch.
+                    map[bob.client.getUserId()][bob.client.deviceId]
+                        .message_authentication_codes = ['hmac-sha256'];
+                }
+                return aliceOrigSendToDevice(type, map);
+            };
+            const bobOrigSendToDevice = bob.client.sendToDevice.bind(bob.client);
+            bob.client.sendToDevice = (type, map) => {
+                if (type === "m.key.verification.accept") {
+                    macMethod = map[alice.client.getUserId()][alice.client.deviceId]
+                        .message_authentication_code;
+                }
+                return bobOrigSendToDevice(type, map);
+            };
+
+            alice.httpBackend.when('POST', '/keys/query').respond(200, {
+                failures: {},
+                device_keys: {
+                    "@bob:example.com": BOB_DEVICES,
+                },
+            });
+            bob.httpBackend.when('POST', '/keys/query').respond(200, {
+                failures: {},
+                device_keys: {
+                    "@alice:example.com": ALICE_DEVICES,
+                },
+            });
+
+            await Promise.all([
+                aliceVerifier.verify(),
+                bobPromise.then((verifier) => verifier.verify()),
+                alice.httpBackend.flush(),
+                bob.httpBackend.flush(),
+            ]);
+
+            expect(macMethod).toBe("hmac-sha256");
+
+            const bobDevice
+                  = await alice.client.getStoredDevice("@bob:example.com", "Dynabook");
+            expect(bobDevice.isVerified()).toBeTruthy();
+            const aliceDevice
+                  = await bob.client.getStoredDevice("@alice:example.com", "Osborne2");
+            expect(aliceDevice.isVerified()).toBeTruthy();
+        });
+
+        it("should verify a cross-signing key", async () => {
+            alice.httpBackend.when('POST', '/keys/device_signing/upload').respond(
+                200, {},
+            );
+            alice.httpBackend.when('POST', '/keys/signatures/upload').respond(200, {});
+            alice.httpBackend.flush(undefined, 2);
+            await resetCrossSigningKeys(alice.client);
+            bob.httpBackend.when('POST', '/keys/device_signing/upload').respond(200, {});
+            bob.httpBackend.when('POST', '/keys/signatures/upload').respond(200, {});
+            bob.httpBackend.flush(undefined, 2);
+
+            await resetCrossSigningKeys(bob.client);
+
+            bob.client._crypto._deviceList.storeCrossSigningForUser(
+                "@alice:example.com", {
+                    keys: alice.client._crypto._crossSigningInfo.keys,
+                },
+            );
+
+            const verifyProm = Promise.all([
+                aliceVerifier.verify(),
+                bobPromise.then((verifier) => {
+                    bob.httpBackend.when(
+                        'POST', '/keys/signatures/upload',
+                    ).respond(200, {});
+                    bob.httpBackend.flush(undefined, 1, 2000);
+                    return verifier.verify();
+                }),
+            ]);
+
+            await verifyProm;
+
+            const bobDeviceTrust = alice.client.checkDeviceTrust(
+                "@bob:example.com", "Dynabook",
+            );
+            expect(bobDeviceTrust.isLocallyVerified()).toBeTruthy();
+            expect(bobDeviceTrust.isCrossSigningVerified()).toBeFalsy();
+
+            const aliceTrust = bob.client.checkUserTrust("@alice:example.com");
+            expect(aliceTrust.isCrossSigningVerified()).toBeTruthy();
+            expect(aliceTrust.isTofu()).toBeTruthy();
+
+            const aliceDeviceTrust = bob.client.checkDeviceTrust(
+                "@alice:example.com", "Osborne2",
+            );
+            expect(aliceDeviceTrust.isLocallyVerified()).toBeTruthy();
+            expect(aliceDeviceTrust.isCrossSigningVerified()).toBeFalsy();
+        });
+    });
+
+    it("should send a cancellation message on error", async function() {
+        const [alice, bob] = await makeTestClients(
+            [
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {userId: "@bob:example.com", deviceId: "Dynabook"},
+            ],
+            {
+                verificationMethods: [verificationMethods.SAS],
+            },
+        );
+        alice.client.setDeviceVerified = jest.fn();
+        alice.client.downloadKeys = () => {
+            return Promise.resolve();
+        };
+        bob.client.setDeviceVerified = jest.fn();
+        bob.client.downloadKeys = () => {
+            return Promise.resolve();
+        };
+
+        const bobPromise = new Promise((resolve, reject) => {
+            bob.client.on("crypto.verification.request", request => {
+                request.verifier.on("show_sas", (e) => {
+                    e.mismatch();
+                });
+                resolve(request.verifier);
+            });
+        });
+
+        const aliceVerifier = alice.client.beginKeyVerification(
+            verificationMethods.SAS, bob.client.getUserId(), bob.client.deviceId,
+        );
+
+        const aliceSpy = jest.fn();
+        const bobSpy = jest.fn();
+        await Promise.all([
+            aliceVerifier.verify().catch(aliceSpy),
+            bobPromise.then((verifier) => verifier.verify()).catch(bobSpy),
+        ]);
+        expect(aliceSpy).toHaveBeenCalled();
+        expect(bobSpy).toHaveBeenCalled();
+        expect(alice.client.setDeviceVerified)
+            .not.toHaveBeenCalled();
+        expect(bob.client.setDeviceVerified)
+            .not.toHaveBeenCalled();
+    });
+
+    describe("verification in DM", function() {
+        let alice;
+        let bob;
+        let aliceSasEvent;
+        let bobSasEvent;
+        let aliceVerifier;
+        let bobPromise;
+
+        beforeEach(async function() {
+            [alice, bob] = await makeTestClients(
+                [
+                    {userId: "@alice:example.com", deviceId: "Osborne2"},
+                    {userId: "@bob:example.com", deviceId: "Dynabook"},
+                ],
+                {
+                    verificationMethods: [verificationMethods.SAS],
+                },
+            );
+
+            alice.client.setDeviceVerified = jest.fn();
+            alice.client.getDeviceEd25519Key = () => {
+                return "alice+base64+ed25519+key";
+            };
+            alice.client.getStoredDevice = () => {
+                return DeviceInfo.fromStorage(
+                    {
+                        keys: {
+                            "ed25519:Dynabook": "bob+base64+ed25519+key",
+                        },
+                    },
+                    "Dynabook",
+                );
+            };
+            alice.client.downloadKeys = () => {
+                return Promise.resolve();
+            };
+
+            bob.client.setDeviceVerified = jest.fn();
+            bob.client.getStoredDevice = () => {
+                return DeviceInfo.fromStorage(
+                    {
+                        keys: {
+                            "ed25519:Osborne2": "alice+base64+ed25519+key",
+                        },
+                    },
+                    "Osborne2",
+                );
+            };
+            bob.client.getDeviceEd25519Key = () => {
+                return "bob+base64+ed25519+key";
+            };
+            bob.client.downloadKeys = () => {
+                return Promise.resolve();
+            };
+
+            aliceSasEvent = null;
+            bobSasEvent = null;
+
+            bobPromise = new Promise((resolve, reject) => {
+                bob.client.on("crypto.verification.request", async (request) => {
+                    const verifier = request.beginKeyVerification(SAS.NAME);
+                    verifier.on("show_sas", (e) => {
+                        if (!e.sas.emoji || !e.sas.decimal) {
+                            e.cancel();
+                        } else if (!aliceSasEvent) {
+                            bobSasEvent = e;
+                        } else {
+                            try {
+                                expect(e.sas).toEqual(aliceSasEvent.sas);
+                                e.confirm();
+                                aliceSasEvent.confirm();
+                            } catch (error) {
+                                e.mismatch();
+                                aliceSasEvent.mismatch();
+                            }
+                        }
+                    });
+                    await verifier.verify();
+                    resolve();
+                });
+            });
+
+            const aliceRequest = await alice.client.requestVerificationDM(
+                bob.client.getUserId(), "!room_id",
+            );
+            await aliceRequest.waitFor(r => r.started);
+            aliceVerifier = aliceRequest.verifier;
+            aliceVerifier.on("show_sas", (e) => {
+                if (!e.sas.emoji || !e.sas.decimal) {
+                    e.cancel();
+                } else if (!bobSasEvent) {
+                    aliceSasEvent = e;
+                } else {
+                    try {
+                        expect(e.sas).toEqual(bobSasEvent.sas);
+                        e.confirm();
+                        bobSasEvent.confirm();
+                    } catch (error) {
+                        e.mismatch();
+                        bobSasEvent.mismatch();
+                    }
+                }
+            });
+        });
+        afterEach(async function() {
+            await Promise.all([
+                alice.stop(),
+                bob.stop(),
+            ]);
+        });
+
+        it("should verify a key", async function() {
+            await Promise.all([
+                aliceVerifier.verify(),
+                bobPromise,
+            ]);
+
+            // make sure Alice and Bob verified each other
+            expect(alice.client.setDeviceVerified)
+                .toHaveBeenCalledWith(bob.client.getUserId(), bob.client.deviceId);
+            expect(bob.client.setDeviceVerified)
+                .toHaveBeenCalledWith(alice.client.getUserId(), alice.client.deviceId);
+        });
+    });
+});

spec/unit/crypto/verification/secret_request.spec.js

@@ -0,0 +1,118 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {VerificationBase} from '../../../../src/crypto/verification/Base';
+import {CrossSigningInfo} from '../../../../src/crypto/CrossSigning';
+import {encodeBase64} from "../../../../src/crypto/olmlib";
+import {setupWebcrypto, teardownWebcrypto} from './util';
+
+jest.useFakeTimers();
+
+// Private key for tests only
+const testKey = new Uint8Array([
+    0xda, 0x5a, 0x27, 0x60, 0xe3, 0x3a, 0xc5, 0x82,
+    0x9d, 0x12, 0xc3, 0xbe, 0xe8, 0xaa, 0xc2, 0xef,
+    0xae, 0xb1, 0x05, 0xc1, 0xe7, 0x62, 0x78, 0xa6,
+    0xd7, 0x1f, 0xf8, 0x2c, 0x51, 0x85, 0xf0, 0x1d,
+]);
+const testKeyPub = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk";
+
+describe("self-verifications", () => {
+    beforeAll(function() {
+        setupWebcrypto();
+        return global.Olm.init();
+    });
+
+    afterAll(() => {
+        teardownWebcrypto();
+    });
+
+    it("triggers a request for key sharing upon completion", async () => {
+        const userId = "@test:localhost";
+
+        const cacheCallbacks = {
+            getCrossSigningKeyCache: jest.fn().mockReturnValue(null),
+            storeCrossSigningKeyCache: jest.fn(),
+        };
+
+        const _crossSigningInfo = new CrossSigningInfo(
+            userId,
+            {},
+            cacheCallbacks,
+        );
+        _crossSigningInfo.keys = {
+            master: { keys: { X: testKeyPub } },
+            self_signing: { keys: { X: testKeyPub } },
+            user_signing: { keys: { X: testKeyPub } },
+        };
+
+        const _secretStorage = {
+            request: jest.fn().mockReturnValue({
+                promise: Promise.resolve(encodeBase64(testKey)),
+            }),
+        };
+
+        const storeSessionBackupPrivateKey = jest.fn();
+        const restoreKeyBackupWithCache = jest.fn(() => Promise.resolve());
+
+        const client = {
+            _crypto: {
+                _crossSigningInfo,
+                _secretStorage,
+                storeSessionBackupPrivateKey,
+                getSessionBackupPrivateKey: () => null,
+            },
+            requestSecret: _secretStorage.request.bind(_secretStorage),
+            getUserId: () => userId,
+            getKeyBackupVersion: () => Promise.resolve({}),
+            restoreKeyBackupWithCache,
+        };
+
+        const request = {
+            onVerifierFinished: () => undefined,
+        };
+
+        const verification = new VerificationBase(
+            undefined, // channel
+            client, // baseApis
+            userId,
+            "ABC", // deviceId
+            undefined, // startEvent
+            request,
+        );
+        verification._resolve = () => undefined;
+
+        const result = await verification.done();
+
+        /* We should request, and store, 3 cross signing keys and the key backup key */
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls.length).toBe(3);
+        expect(_secretStorage.request.mock.calls.length).toBe(4);
+
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[0][1])
+          .toEqual(testKey);
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[1][1])
+          .toEqual(testKey);
+
+        expect(storeSessionBackupPrivateKey.mock.calls[0][0])
+          .toEqual(testKey);
+
+        expect(restoreKeyBackupWithCache).toHaveBeenCalled();
+
+        expect(result).toBeInstanceOf(Array);
+        expect(result[0][0]).toBe(testKeyPub);
+        expect(result[1][0]).toBe(testKeyPub);
+    });
+});

spec/unit/crypto/verification/util.js

@@ -0,0 +1,118 @@
+/*
+Copyright 2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {TestClient} from '../../../TestClient';
+import {MatrixEvent} from "../../../../src/models/event";
+import nodeCrypto from "crypto";
+import {logger} from '../../../../src/logger';
+
+export async function makeTestClients(userInfos, options) {
+    const clients = [];
+    const clientMap = {};
+    const sendToDevice = function(type, map) {
+        // logger.log(this.getUserId(), "sends", type, map);
+        for (const [userId, devMap] of Object.entries(map)) {
+            if (userId in clientMap) {
+                for (const [deviceId, msg] of Object.entries(devMap)) {
+                    if (deviceId in clientMap[userId]) {
+                        const event = new MatrixEvent({
+                            sender: this.getUserId(), // eslint-disable-line babel/no-invalid-this
+                            type: type,
+                            content: msg,
+                        });
+                        const client = clientMap[userId][deviceId];
+                        const decryptionPromise = event.isEncrypted() ?
+                            event.attemptDecryption(client._crypto) :
+                            Promise.resolve();
+
+                        decryptionPromise.then(
+                            () => client.emit("toDeviceEvent", event),
+                        );
+                    }
+                }
+            }
+        }
+    };
+    const sendEvent = function(room, type, content) {
+        // make up a unique ID as the event ID
+        const eventId = "$" + this.makeTxnId(); // eslint-disable-line babel/no-invalid-this
+        const rawEvent = {
+            sender: this.getUserId(), // eslint-disable-line babel/no-invalid-this
+            type: type,
+            content: content,
+            room_id: room,
+            event_id: eventId,
+            origin_server_ts: Date.now(),
+        };
+        const event = new MatrixEvent(rawEvent);
+        const remoteEcho = new MatrixEvent(Object.assign({}, rawEvent, {
+            unsigned: {
+                transaction_id: this.makeTxnId(), // eslint-disable-line babel/no-invalid-this
+            },
+        }));
+
+        setImmediate(() => {
+            for (const tc of clients) {
+                if (tc.client === this) { // eslint-disable-line babel/no-invalid-this
+                    logger.log("sending remote echo!!");
+                    tc.client.emit("Room.timeline", remoteEcho);
+                } else {
+                    tc.client.emit("Room.timeline", event);
+                }
+            }
+        });
+
+        return Promise.resolve({event_id: eventId});
+    };
+
+    for (const userInfo of userInfos) {
+        let keys = {};
+        if (!options) options = {};
+        if (!options.cryptoCallbacks) options.cryptoCallbacks = {};
+        if (!options.cryptoCallbacks.saveCrossSigningKeys) {
+            options.cryptoCallbacks.saveCrossSigningKeys = k => { keys = k; };
+            options.cryptoCallbacks.getCrossSigningKey = typ => keys[typ];
+        }
+        const testClient = new TestClient(
+            userInfo.userId, userInfo.deviceId, undefined, undefined,
+            options,
+        );
+        if (!(userInfo.userId in clientMap)) {
+            clientMap[userInfo.userId] = {};
+        }
+        clientMap[userInfo.userId][userInfo.deviceId] = testClient.client;
+        testClient.client.sendToDevice = sendToDevice;
+        testClient.client.sendEvent = sendEvent;
+        clients.push(testClient);
+    }
+
+    await Promise.all(clients.map((testClient) => testClient.client.initCrypto()));
+
+    return clients;
+}
+
+export function setupWebcrypto() {
+    global.crypto = {
+        getRandomValues: (buf) => {
+            return nodeCrypto.randomFillSync(buf);
+        },
+    };
+}
+
+export function teardownWebcrypto() {
+    global.crypto = undefined;
+}

spec/unit/crypto/verification/verification_request.spec.js

@@ -0,0 +1,285 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import {VerificationRequest, READY_TYPE, START_TYPE, DONE_TYPE} from
+    "../../../../src/crypto/verification/request/VerificationRequest";
+import {InRoomChannel} from "../../../../src/crypto/verification/request/InRoomChannel";
+import {ToDeviceChannel} from
+    "../../../../src/crypto/verification/request/ToDeviceChannel";
+import {MatrixEvent} from "../../../../src/models/event";
+import {setupWebcrypto, teardownWebcrypto} from "./util";
+
+function makeMockClient(userId, deviceId) {
+    let counter = 1;
+    let events = [];
+    const deviceEvents = {};
+    return {
+        getUserId() { return userId; },
+        getDeviceId() { return deviceId; },
+
+        sendEvent(roomId, type, content) {
+            counter = counter + 1;
+            const eventId = `$${userId}-${deviceId}-${counter}`;
+            events.push(new MatrixEvent({
+                sender: userId,
+                event_id: eventId,
+                room_id: roomId,
+                type,
+                content,
+                origin_server_ts: Date.now(),
+            }));
+            return Promise.resolve({event_id: eventId});
+        },
+
+        sendToDevice(type, msgMap) {
+            for (const userId of Object.keys(msgMap)) {
+                const deviceMap = msgMap[userId];
+                for (const deviceId of Object.keys(deviceMap)) {
+                    const content = deviceMap[deviceId];
+                    const event = new MatrixEvent({content, type});
+                    deviceEvents[userId] = deviceEvents[userId] || {};
+                    deviceEvents[userId][deviceId] = deviceEvents[userId][deviceId] || [];
+                    deviceEvents[userId][deviceId].push(event);
+                }
+            }
+            return Promise.resolve();
+        },
+
+        popEvents() {
+            const e = events;
+            events = [];
+            return e;
+        },
+
+        popDeviceEvents(userId, deviceId) {
+            const forDevice = deviceEvents[userId];
+            const events = forDevice && forDevice[deviceId];
+            const result = events || [];
+            if (events) {
+                delete forDevice[deviceId];
+            }
+            return result;
+        },
+    };
+}
+
+const MOCK_METHOD = "mock-verify";
+class MockVerifier {
+    constructor(channel, client, userId, deviceId, startEvent) {
+        this._channel = channel;
+        this._startEvent = startEvent;
+    }
+
+    get events() {
+        return [DONE_TYPE];
+    }
+
+    async start() {
+        if (this._startEvent) {
+            await this._channel.send(DONE_TYPE, {});
+        } else {
+            await this._channel.send(START_TYPE, {method: MOCK_METHOD});
+        }
+    }
+
+    async handleEvent(event) {
+        if (event.getType() === DONE_TYPE && !this._startEvent) {
+            await this._channel.send(DONE_TYPE, {});
+        }
+    }
+
+    canSwitchStartEvent() {
+        return false;
+    }
+}
+
+function makeRemoteEcho(event) {
+    return new MatrixEvent(Object.assign({}, event.event, {
+        unsigned: {
+            transaction_id: "abc",
+        },
+    }));
+}
+
+async function distributeEvent(ownRequest, theirRequest, event) {
+    await ownRequest.channel.handleEvent(
+            makeRemoteEcho(event), ownRequest, true);
+    await theirRequest.channel.handleEvent(event, theirRequest, true);
+}
+
+jest.useFakeTimers();
+
+describe("verification request unit tests", function() {
+    beforeAll(function() {
+        setupWebcrypto();
+    });
+
+    afterAll(() => {
+        teardownWebcrypto();
+    });
+
+    it("transition from UNSENT to DONE through happy path", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()),
+            new Map([[MOCK_METHOD, MockVerifier]]), alice);
+        const bobRequest = new VerificationRequest(
+            new InRoomChannel(bob, "!room"),
+            new Map([[MOCK_METHOD, MockVerifier]]), bob);
+        expect(aliceRequest.invalid).toBe(true);
+        expect(bobRequest.invalid).toBe(true);
+
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        expect(requestEvent.getType()).toBe("m.room.message");
+        await distributeEvent(aliceRequest, bobRequest, requestEvent);
+        expect(aliceRequest.requested).toBe(true);
+        expect(bobRequest.requested).toBe(true);
+
+        await bobRequest.accept();
+        const [readyEvent] = bob.popEvents();
+        expect(readyEvent.getType()).toBe(READY_TYPE);
+        await distributeEvent(bobRequest, aliceRequest, readyEvent);
+        expect(bobRequest.ready).toBe(true);
+        expect(aliceRequest.ready).toBe(true);
+
+        const verifier = aliceRequest.beginKeyVerification(MOCK_METHOD);
+        await verifier.start();
+        const [startEvent] = alice.popEvents();
+        expect(startEvent.getType()).toBe(START_TYPE);
+        await distributeEvent(aliceRequest, bobRequest, startEvent);
+        expect(aliceRequest.started).toBe(true);
+        expect(aliceRequest.verifier).toBeInstanceOf(MockVerifier);
+        expect(bobRequest.started).toBe(true);
+        expect(bobRequest.verifier).toBeInstanceOf(MockVerifier);
+
+        await bobRequest.verifier.start();
+        const [bobDoneEvent] = bob.popEvents();
+        expect(bobDoneEvent.getType()).toBe(DONE_TYPE);
+        await distributeEvent(bobRequest, aliceRequest, bobDoneEvent);
+        const [aliceDoneEvent] = alice.popEvents();
+        expect(aliceDoneEvent.getType()).toBe(DONE_TYPE);
+        await distributeEvent(aliceRequest, bobRequest, aliceDoneEvent);
+        expect(aliceRequest.done).toBe(true);
+        expect(bobRequest.done).toBe(true);
+    });
+
+    it("methods only contains common methods", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()),
+            new Map([["c", function() {}], ["a", function() {}]]), alice);
+        const bobRequest = new VerificationRequest(
+            new InRoomChannel(bob, "!room"),
+            new Map([["c", function() {}], ["b", function() {}]]), bob);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        await distributeEvent(aliceRequest, bobRequest, requestEvent);
+        await bobRequest.accept();
+        const [readyEvent] = bob.popEvents();
+        await distributeEvent(bobRequest, aliceRequest, readyEvent);
+        expect(aliceRequest.methods).toStrictEqual(["c"]);
+        expect(bobRequest.methods).toStrictEqual(["c"]);
+    });
+
+    it("other client accepting request puts it in observeOnly mode", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob1 = makeMockClient("@bob:matrix.tld", "device1");
+        const bob2 = makeMockClient("@bob:matrix.tld", "device2");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob1.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        const bob1Request = new VerificationRequest(
+            new InRoomChannel(bob1, "!room"), new Map(), bob1);
+        const bob2Request = new VerificationRequest(
+            new InRoomChannel(bob2, "!room"), new Map(), bob2);
+
+        await bob1Request.channel.handleEvent(requestEvent, bob1Request, true);
+        await bob2Request.channel.handleEvent(requestEvent, bob2Request, true);
+
+        await bob1Request.accept();
+        const [readyEvent] = bob1.popEvents();
+        expect(bob2Request.observeOnly).toBe(false);
+        await bob2Request.channel.handleEvent(readyEvent, bob2Request, true);
+        expect(bob2Request.observeOnly).toBe(true);
+    });
+
+    it("verify own device with to_device messages", async function() {
+        const bob1 = makeMockClient("@bob:matrix.tld", "device1");
+        const bob2 = makeMockClient("@bob:matrix.tld", "device2");
+        const bob1Request = new VerificationRequest(
+            new ToDeviceChannel(bob1, bob1.getUserId(), ["device1", "device2"],
+                ToDeviceChannel.makeTransactionId(), "device2"),
+            new Map([[MOCK_METHOD, MockVerifier]]), bob1);
+        const to = {userId: "@bob:matrix.tld", deviceId: "device2"};
+        const verifier = bob1Request.beginKeyVerification(MOCK_METHOD, to);
+        expect(verifier).toBeInstanceOf(MockVerifier);
+        await verifier.start();
+        const [startEvent] = bob1.popDeviceEvents(to.userId, to.deviceId);
+        expect(startEvent.getType()).toBe(START_TYPE);
+        const bob2Request = new VerificationRequest(
+            new ToDeviceChannel(bob2, bob2.getUserId(), ["device1"]),
+            new Map([[MOCK_METHOD, MockVerifier]]), bob2);
+
+        await bob2Request.channel.handleEvent(startEvent, bob2Request, true);
+        await bob2Request.verifier.start();
+        const [doneEvent1] = bob2.popDeviceEvents("@bob:matrix.tld", "device1");
+        expect(doneEvent1.getType()).toBe(DONE_TYPE);
+        await bob1Request.channel.handleEvent(doneEvent1, bob1Request, true);
+        const [doneEvent2] = bob1.popDeviceEvents("@bob:matrix.tld", "device2");
+        expect(doneEvent2.getType()).toBe(DONE_TYPE);
+        await bob2Request.channel.handleEvent(doneEvent2, bob2Request, true);
+
+        expect(bob1Request.done).toBe(true);
+        expect(bob2Request.done).toBe(true);
+    });
+
+    it("request times out after 10 minutes", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        await aliceRequest.channel.handleEvent(requestEvent, aliceRequest, true,
+            true, true);
+
+        expect(aliceRequest.cancelled).toBe(false);
+        expect(aliceRequest._cancellingUserId).toBe(undefined);
+        jest.advanceTimersByTime(10 * 60 * 1000);
+        expect(aliceRequest._cancellingUserId).toBe(alice.getUserId());
+    });
+
+    it("request times out 2 minutes after receipt", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        const bobRequest = new VerificationRequest(
+            new InRoomChannel(bob, "!room"), new Map(), bob);
+
+        await bobRequest.channel.handleEvent(requestEvent, bobRequest, true);
+
+        expect(bobRequest.cancelled).toBe(false);
+        expect(bobRequest._cancellingUserId).toBe(undefined);
+        jest.advanceTimersByTime(2 * 60 * 1000);
+        expect(bobRequest._cancellingUserId).toBe(bob.getUserId());
+    });
+});

spec/unit/event-timeline.spec.js

@@ -1,16 +1,12 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const EventTimeline = sdk.EventTimeline;
-const utils = require("../test-utils");
+import * as utils from "../test-utils";
+import {EventTimeline} from "../../src/models/event-timeline";
+import {RoomState} from "../../src/models/room-state";
 
 function mockRoomStates(timeline) {
-    timeline._startState = utils.mock(sdk.RoomState, "startState");
-    timeline._endState = utils.mock(sdk.RoomState, "endState");
+    timeline._startState = utils.mock(RoomState, "startState");
+    timeline._endState = utils.mock(RoomState, "endState");
 }
 
-import expect from 'expect';
-
 describe("EventTimeline", function() {
     const roomId = "!foo:bar";
     const userA = "@alice:bar";
@@ -18,8 +14,6 @@ describe("EventTimeline", function() {
     let timeline;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-
         // XXX: this is a horrid hack; should use sinon or something instead to mock
         const timelineSet = { room: { roomId: roomId }};
         timelineSet.room.getUnfilteredTimelineSet = function() {
@@ -78,7 +72,7 @@ describe("EventTimeline", function() {
 
             expect(function() {
                 timeline.initialiseState(state);
-            }).toNotThrow();
+            }).not.toThrow();
             timeline.addEvent(event, false);
             expect(function() {
                 timeline.initialiseState(state);
@@ -121,7 +115,7 @@ describe("EventTimeline", function() {
             const next = {b: "b"};
             expect(function() {
                 timeline.setNeighbouringTimeline(prev, EventTimeline.BACKWARDS);
-            }).toNotThrow();
+            }).not.toThrow();
             expect(timeline.getNeighbouringTimeline(EventTimeline.BACKWARDS))
                 .toBe(prev);
             expect(function() {
@@ -130,7 +124,7 @@ describe("EventTimeline", function() {
 
             expect(function() {
                 timeline.setNeighbouringTimeline(next, EventTimeline.FORWARDS);
-            }).toNotThrow();
+            }).not.toThrow();
             expect(timeline.getNeighbouringTimeline(EventTimeline.FORWARDS))
                 .toBe(next);
             expect(function() {
@@ -187,14 +181,14 @@ describe("EventTimeline", function() {
                 name: "Old Alice",
             };
             timeline.getState(EventTimeline.FORWARDS).getSentinelMember
-                .andCall(function(uid) {
+                .mockImplementation(function(uid) {
                     if (uid === userA) {
                         return sentinel;
                     }
                     return null;
                 });
             timeline.getState(EventTimeline.BACKWARDS).getSentinelMember
-                .andCall(function(uid) {
+                .mockImplementation(function(uid) {
                     if (uid === userA) {
                         return oldSentinel;
                     }
@@ -229,14 +223,14 @@ describe("EventTimeline", function() {
                 name: "Old Alice",
             };
             timeline.getState(EventTimeline.FORWARDS).getSentinelMember
-                .andCall(function(uid) {
+                .mockImplementation(function(uid) {
                     if (uid === userA) {
                         return sentinel;
                     }
                     return null;
                 });
             timeline.getState(EventTimeline.BACKWARDS).getSentinelMember
-                .andCall(function(uid) {
+                .mockImplementation(function(uid) {
                     if (uid === userA) {
                         return oldSentinel;
                     }
@@ -281,7 +275,7 @@ describe("EventTimeline", function() {
             expect(events[1].forwardLooking).toBe(true);
 
             expect(timeline.getState(EventTimeline.BACKWARDS).setStateEvents).
-                toNotHaveBeenCalled();
+                not.toHaveBeenCalled();
         });
 
 
@@ -311,7 +305,7 @@ describe("EventTimeline", function() {
             expect(events[1].forwardLooking).toBe(false);
 
             expect(timeline.getState(EventTimeline.FORWARDS).setStateEvents).
-                toNotHaveBeenCalled();
+                not.toHaveBeenCalled();
         });
     });
 

spec/unit/event.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 New Vector Ltd
+Copyright 2019 The Matrix.org Foundaction C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,19 +15,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import sdk from '../..';
-const MatrixEvent = sdk.MatrixEvent;
-
-import testUtils from '../test-utils';
-
-import expect from 'expect';
-import Promise from 'bluebird';
+import {logger} from "../../src/logger";
+import {MatrixEvent} from "../../src/models/event";
 
 describe("MatrixEvent", () => {
-    beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
-    });
-
     describe(".attemptDecryption", () => {
         let encryptedEvent;
 
@@ -44,21 +36,23 @@ describe("MatrixEvent", () => {
             let callCount = 0;
 
             let prom2;
+            let prom2Fulfilled = false;
 
             const crypto = {
                 decryptEvent: function() {
                     ++callCount;
-                    console.log(`decrypt: ${callCount}`);
+                    logger.log(`decrypt: ${callCount}`);
                     if (callCount == 1) {
                         // schedule a second decryption attempt while
                         // the first one is still running.
                         prom2 = encryptedEvent.attemptDecryption(crypto);
+                        prom2.then(() => prom2Fulfilled = true);
 
                         const error = new Error("nope");
                         error.name = 'DecryptionError';
                         return Promise.reject(error);
                     } else {
-                        expect(prom2.isFulfilled()).toBe(
+                        expect(prom2Fulfilled).toBe(
                             false, 'second attemptDecryption resolved too soon');
 
                         return Promise.resolve({

spec/unit/filter-component.spec.js

@@ -0,0 +1,34 @@
+import {FilterComponent} from "../../src/filter-component";
+import {mkEvent} from '../test-utils';
+
+describe("Filter Component", function() {
+    describe("types", function() {
+        it("should filter out events with other types", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.member',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(false);
+        });
+
+        it("should validate events with the same type", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.message',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(true);
+        });
+    });
+});

spec/unit/filter.spec.js

@@ -1,10 +1,4 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const Filter = sdk.Filter;
-const utils = require("../test-utils");
-
-import expect from 'expect';
+import {Filter} from "../../src/filter";
 
 describe("Filter", function() {
     const filterId = "f1lt3ring15g00d4ursoul";
@@ -12,7 +6,6 @@ describe("Filter", function() {
     let filter;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
         filter = new Filter(userId);
     });
 

spec/unit/interactive-auth.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2016 OpenMarket Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,17 +14,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
 
-import 'source-map-support/register';
-import Promise from 'bluebird';
-const sdk = require("../..");
-const utils = require("../test-utils");
-
-const InteractiveAuth = sdk.InteractiveAuth;
-const MatrixError = sdk.MatrixError;
-
-import expect from 'expect';
+import {logger} from "../../src/logger";
+import {InteractiveAuth} from "../../src/interactive-auth";
+import {MatrixError} from "../../src/http-api";
 
 // Trivial client object to test interactive auth
 // (we do not need TestClient here)
@@ -34,13 +28,9 @@ class FakeClient {
 }
 
 describe("InteractiveAuth", function() {
-    beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-    });
-
-    it("should start an auth stage and complete it", function(done) {
-        const doRequest = expect.createSpy();
-        const stateUpdated = expect.createSpy();
+    it("should start an auth stage and complete it", function() {
+        const doRequest = jest.fn();
+        const stateUpdated = jest.fn();
 
         const ia = new InteractiveAuth({
             matrixClient: new FakeClient(),
@@ -63,8 +53,8 @@ describe("InteractiveAuth", function() {
         });
 
         // first we expect a call here
-        stateUpdated.andCall(function(stage) {
-            console.log('aaaa');
+        stateUpdated.mockImplementation(function(stage) {
+            logger.log('aaaa');
             expect(stage).toEqual("logintype");
             ia.submitAuthDict({
                 type: "logintype",
@@ -74,8 +64,8 @@ describe("InteractiveAuth", function() {
 
         // .. which should trigger a call here
         const requestRes = {"a": "b"};
-        doRequest.andCall(function(authData) {
-            console.log('cccc');
+        doRequest.mockImplementation(function(authData) {
+            logger.log('cccc');
             expect(authData).toEqual({
                 session: "sessionId",
                 type: "logintype",
@@ -84,16 +74,16 @@ describe("InteractiveAuth", function() {
             return Promise.resolve(requestRes);
         });
 
-        ia.attemptAuth().then(function(res) {
+        return ia.attemptAuth().then(function(res) {
             expect(res).toBe(requestRes);
-            expect(doRequest.calls.length).toEqual(1);
-            expect(stateUpdated.calls.length).toEqual(1);
-        }).nodeify(done);
+            expect(doRequest).toBeCalledTimes(1);
+            expect(stateUpdated).toBeCalledTimes(1);
+        });
     });
 
-    it("should make a request if no authdata is provided", function(done) {
-        const doRequest = expect.createSpy();
-        const stateUpdated = expect.createSpy();
+    it("should make a request if no authdata is provided", function() {
+        const doRequest = jest.fn();
+        const stateUpdated = jest.fn();
 
         const ia = new InteractiveAuth({
             matrixClient: new FakeClient(),
@@ -105,9 +95,9 @@ describe("InteractiveAuth", function() {
         expect(ia.getStageParams("logintype")).toBe(undefined);
 
         // first we expect a call to doRequest
-        doRequest.andCall(function(authData) {
-            console.log("request1", authData);
-            expect(authData).toEqual({});
+        doRequest.mockImplementation(function(authData) {
+            logger.log("request1", authData);
+            expect(authData).toEqual(null); // first request should be null
             const err = new MatrixError({
                 session: "sessionId",
                 flows: [
@@ -123,7 +113,7 @@ describe("InteractiveAuth", function() {
 
         // .. which should be followed by a call to stateUpdated
         const requestRes = {"a": "b"};
-        stateUpdated.andCall(function(stage) {
+        stateUpdated.mockImplementation(function(stage) {
             expect(stage).toEqual("logintype");
             expect(ia.getSessionId()).toEqual("sessionId");
             expect(ia.getStageParams("logintype")).toEqual({
@@ -131,8 +121,8 @@ describe("InteractiveAuth", function() {
             });
 
             // submitAuthDict should trigger another call to doRequest
-            doRequest.andCall(function(authData) {
-                console.log("request2", authData);
+            doRequest.mockImplementation(function(authData) {
+                logger.log("request2", authData);
                 expect(authData).toEqual({
                     session: "sessionId",
                     type: "logintype",
@@ -147,10 +137,39 @@ describe("InteractiveAuth", function() {
             });
         });
 
-        ia.attemptAuth().then(function(res) {
+        return ia.attemptAuth().then(function(res) {
             expect(res).toBe(requestRes);
-            expect(doRequest.calls.length).toEqual(2);
-            expect(stateUpdated.calls.length).toEqual(1);
-        }).nodeify(done);
+            expect(doRequest).toBeCalledTimes(2);
+            expect(stateUpdated).toBeCalledTimes(1);
+        });
+    });
+
+    it("should start an auth stage and reject if no auth flow", function() {
+        const doRequest = jest.fn();
+        const stateUpdated = jest.fn();
+
+        const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
+            doRequest: doRequest,
+            stateUpdated: stateUpdated,
+        });
+
+        doRequest.mockImplementation(function(authData) {
+            logger.log("request1", authData);
+            expect(authData).toEqual(null); // first request should be null
+            const err = new MatrixError({
+                session: "sessionId",
+                flows: [],
+                params: {
+                    "logintype": { param: "aa" },
+                },
+            });
+            err.httpStatus = 401;
+            throw err;
+        });
+
+        return ia.attemptAuth().catch(function(error) {
+            expect(error.message).toBe('No appropriate authentication flow found');
+        });
     });
 });

spec/unit/login.spec.js

@@ -0,0 +1,24 @@
+import {TestClient} from '../TestClient';
+
+describe('Login request', function() {
+    let client;
+
+    beforeEach(function() {
+        client = new TestClient();
+    });
+
+    afterEach(function() {
+        client.stop();
+    });
+
+    it('should store "access_token" and "user_id" if in response', async function() {
+        const response = { user_id: 1, access_token: Date.now().toString(16) };
+
+        client.httpBackend.when('POST', '/login').respond(200, response);
+        client.httpBackend.flush('/login', 1, 100);
+        await client.client.login('m.login.any', { user: 'test', password: '12312za' });
+
+        expect(client.client.getAccessToken()).toBe(response.access_token);
+        expect(client.client.getUserId()).toBe(response.user_id);
+    });
+});

spec/unit/matrix-client.spec.js

@@ -1,12 +1,8 @@
-"use strict";
-import 'source-map-support/register';
-import Promise from 'bluebird';
-const sdk = require("../..");
-const MatrixClient = sdk.MatrixClient;
-const utils = require("../test-utils");
+import {logger} from "../../src/logger";
+import {MatrixClient} from "../../src/client";
+import {Filter} from "../../src/filter";
 
-import expect from 'expect';
-import lolex from 'lolex';
+jest.useFakeTimers();
 
 describe("MatrixClient", function() {
     const userId = "@alice:bar";
@@ -15,7 +11,6 @@ describe("MatrixClient", function() {
     let client;
     let store;
     let scheduler;
-    let clock;
 
     const KEEP_ALIVE_PATH = "/_matrix/client/versions";
 
@@ -69,7 +64,7 @@ describe("MatrixClient", function() {
             "MatrixClient[UT] RECV " + method + " " + path + "  " +
             "EXPECT " + (next ? next.method : next) + " " + (next ? next.path : next)
         );
-        console.log(logLine);
+        logger.log(logLine);
 
         if (!next) { // no more things to return
             if (pendingLookup) {
@@ -84,14 +79,14 @@ describe("MatrixClient", function() {
                 );
             }
             pendingLookup = {
-                promise: Promise.defer().promise,
+                promise: new Promise(() => {}),
                 method: method,
                 path: path,
             };
             return pendingLookup.promise;
         }
         if (next.path === path && next.method === method) {
-            console.log(
+            logger.log(
                 "MatrixClient[UT] Matched. Returning " +
                 (next.error ? "BAD" : "GOOD") + " response",
             );
@@ -120,28 +115,26 @@ describe("MatrixClient", function() {
             return Promise.resolve(next.data);
         }
         expect(true).toBe(false, "Expected different request. " + logLine);
-        return Promise.defer().promise;
+        return new Promise(() => {});
     }
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        clock = lolex.install();
         scheduler = [
             "getQueueForEvent", "queueEvent", "removeEventFromQueue",
             "setProcessFunction",
-        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
+        ].reduce((r, k) => { r[k] = jest.fn(); return r; }, {});
         store = [
             "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
             "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom", "storeUser",
             "getFilterIdByName", "setFilterIdByName", "getFilter", "storeFilter",
             "getSyncAccumulator", "startup", "deleteAllData",
-        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
-        store.getSavedSync = expect.createSpy().andReturn(Promise.resolve(null));
-        store.getSavedSyncToken = expect.createSpy().andReturn(Promise.resolve(null));
-        store.setSyncData = expect.createSpy().andReturn(Promise.resolve(null));
-        store.getClientOptions = expect.createSpy().andReturn(Promise.resolve(null));
-        store.storeClientOptions = expect.createSpy().andReturn(Promise.resolve(null));
-        store.isNewlyCreated = expect.createSpy().andReturn(Promise.resolve(true));
+        ].reduce((r, k) => { r[k] = jest.fn(); return r; }, {});
+        store.getSavedSync = jest.fn().mockReturnValue(Promise.resolve(null));
+        store.getSavedSyncToken = jest.fn().mockReturnValue(Promise.resolve(null));
+        store.setSyncData = jest.fn().mockReturnValue(Promise.resolve(null));
+        store.getClientOptions = jest.fn().mockReturnValue(Promise.resolve(null));
+        store.storeClientOptions = jest.fn().mockReturnValue(Promise.resolve(null));
+        store.isNewlyCreated = jest.fn().mockReturnValue(Promise.resolve(true));
         client = new MatrixClient({
             baseUrl: "https://my.home.server",
             idBaseUrl: identityServerUrl,
@@ -153,13 +146,10 @@ describe("MatrixClient", function() {
         });
         // FIXME: We shouldn't be yanking _http like this.
         client._http = [
-            "authedRequest", "authedRequestWithPrefix", "getContentUri",
-            "request", "requestWithPrefix", "uploadContent",
-        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
-        client._http.authedRequest.andCall(httpReq);
-        client._http.authedRequestWithPrefix.andCall(httpReq);
-        client._http.requestWithPrefix.andCall(httpReq);
-        client._http.request.andCall(httpReq);
+            "authedRequest", "getContentUri", "request", "uploadContent",
+        ].reduce((r, k) => { r[k] = jest.fn(); return r; }, {});
+        client._http.authedRequest.mockImplementation(httpReq);
+        client._http.request.mockImplementation(httpReq);
 
         // set reasonable working defaults
         acceptKeepalives = true;
@@ -171,17 +161,13 @@ describe("MatrixClient", function() {
     });
 
     afterEach(function() {
-        clock.uninstall();
         // need to re-stub the requests with NOPs because there are no guarantees
         // clients from previous tests will be GC'd before the next test. This
         // means they may call /events and then fail an expect() which will fail
         // a DIFFERENT test (pollution between tests!) - we return unresolved
         // promises to stop the client from continuing to run.
-        client._http.authedRequest.andCall(function() {
-            return Promise.defer().promise;
-        });
-        client._http.authedRequestWithPrefix.andCall(function() {
-            return Promise.defer().promise;
+        client._http.authedRequest.mockImplementation(function() {
+            return new Promise(() => {});
         });
     });
 
@@ -190,10 +176,10 @@ describe("MatrixClient", function() {
         httpLookups.push(PUSH_RULES_RESPONSE);
         httpLookups.push(SYNC_RESPONSE);
         const filterId = "ehfewf";
-        store.getFilterIdByName.andReturn(filterId);
-        const filter = new sdk.Filter(0, filterId);
+        store.getFilterIdByName.mockReturnValue(filterId);
+        const filter = new Filter(0, filterId);
         filter.setDefinition({"room": {"timeline": {"limit": 8}}});
-        store.getFilter.andReturn(filter);
+        store.getFilter.mockReturnValue(filter);
         const syncPromise = new Promise((resolve, reject) => {
             client.on("sync", function syncListener(state) {
                 if (state === "SYNCING") {
@@ -254,11 +240,11 @@ describe("MatrixClient", function() {
                 },
             });
             httpLookups.push(FILTER_RESPONSE);
-            store.getFilterIdByName.andReturn(invalidFilterId);
+            store.getFilterIdByName.mockReturnValue(invalidFilterId);
 
             const filterName = getFilterName(client.credentials.userId);
             client.store.setFilterIdByName(filterName, invalidFilterId);
-            const filter = new sdk.Filter(client.credentials.userId);
+            const filter = new Filter(client.credentials.userId);
 
             client.getOrCreateFilter(filterName, filter).then(function(filterId) {
                 expect(filterId).toEqual(FILTER_RESPONSE.data.filter_id);
@@ -286,7 +272,7 @@ describe("MatrixClient", function() {
                 if (state === "ERROR" && httpLookups.length > 0) {
                     expect(httpLookups.length).toEqual(2);
                     expect(client.retryImmediately()).toBe(true);
-                    clock.tick(1);
+                    jest.advanceTimersByTime(1);
                 } else if (state === "PREPARED" && httpLookups.length === 0) {
                     client.removeListener("sync", syncListener);
                     done();
@@ -312,9 +298,9 @@ describe("MatrixClient", function() {
                     expect(client.retryImmediately()).toBe(
                         true, "retryImmediately returned false",
                     );
-                    clock.tick(1);
+                    jest.advanceTimersByTime(1);
                 } else if (state === "RECONNECTING" && httpLookups.length > 0) {
-                    clock.tick(10000);
+                    jest.advanceTimersByTime(10000);
                 } else if (state === "SYNCING" && httpLookups.length === 0) {
                     client.removeListener("sync", syncListener);
                     done();
@@ -336,7 +322,7 @@ describe("MatrixClient", function() {
                 if (state === "ERROR" && httpLookups.length > 0) {
                     expect(httpLookups.length).toEqual(3);
                     expect(client.retryImmediately()).toBe(true);
-                    clock.tick(1);
+                    jest.advanceTimersByTime(1);
                 } else if (state === "PREPARED" && httpLookups.length === 0) {
                     client.removeListener("sync", syncListener);
                     done();
@@ -353,7 +339,7 @@ describe("MatrixClient", function() {
         function syncChecker(expectedStates, done) {
             return function syncListener(state, old) {
                 const expected = expectedStates.shift();
-                console.log(
+                logger.log(
                     "'sync' curr=%s old=%s EXPECT=%s", state, old, expected,
                 );
                 if (!expected) {
@@ -367,7 +353,7 @@ describe("MatrixClient", function() {
                     done();
                 }
                 // standard retry time is 5 to 10 seconds
-                clock.tick(10000);
+                jest.advanceTimersByTime(10000);
             };
         }
 
@@ -535,4 +521,19 @@ describe("MatrixClient", function() {
         xit("should be able to peek into a room using peekInRoom", function(done) {
         });
     });
+
+    describe("getPresence", function() {
+        it("should send a presence HTTP GET", function() {
+            httpLookups = [{
+                method: "GET",
+                path: `/presence/${encodeURIComponent(userId)}/status`,
+                data: {
+                    "presence": "unavailable",
+                    "last_active_ago": 420845,
+                },
+            }];
+            client.getPresence(userId);
+            expect(httpLookups.length).toEqual(0);
+        });
+    });
 });

spec/unit/pushprocessor.spec.js

@@ -1,9 +1,5 @@
-"use strict";
-import 'source-map-support/register';
-const PushProcessor = require("../../lib/pushprocessor");
-const utils = require("../test-utils");
-
-import expect from 'expect';
+import * as utils from "../test-utils";
+import {PushProcessor} from "../../src/pushprocessor";
 
 describe('NotificationService', function() {
     const testUserId = "@ali:matrix.org";

spec/unit/realtime-callbacks.spec.js

@@ -1,53 +1,43 @@
-"use strict";
+import * as callbacks from "../../src/realtime-callbacks";
 
-import 'source-map-support/register';
-const callbacks = require("../../lib/realtime-callbacks");
-const testUtils = require("../test-utils.js");
-
-import expect from 'expect';
-import lolex from 'lolex';
+let wallTime = 1234567890;
+jest.useFakeTimers();
 
 describe("realtime-callbacks", function() {
-    let clock;
-
     function tick(millis) {
-        clock.tick(millis);
+        wallTime += millis;
+        jest.advanceTimersByTime(millis);
     }
 
     beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
-        clock = lolex.install();
-        const fakeDate = clock.Date;
-        callbacks.setNow(fakeDate.now.bind(fakeDate));
+        callbacks.setNow(() => wallTime);
     });
 
     afterEach(function() {
         callbacks.setNow();
-        clock.uninstall();
     });
 
     describe("setTimeout", function() {
         it("should call the callback after the timeout", function() {
-            const callback = expect.createSpy();
+            const callback = jest.fn();
             callbacks.setTimeout(callback, 100);
 
-            expect(callback).toNotHaveBeenCalled();
+            expect(callback).not.toHaveBeenCalled();
             tick(100);
             expect(callback).toHaveBeenCalled();
         });
 
-
         it("should default to a zero timeout", function() {
-            const callback = expect.createSpy();
+            const callback = jest.fn();
             callbacks.setTimeout(callback);
 
-            expect(callback).toNotHaveBeenCalled();
+            expect(callback).not.toHaveBeenCalled();
             tick(0);
             expect(callback).toHaveBeenCalled();
         });
 
         it("should pass any parameters to the callback", function() {
-            const callback = expect.createSpy();
+            const callback = jest.fn();
             callbacks.setTimeout(callback, 0, "a", "b", "c");
             tick(0);
             expect(callback).toHaveBeenCalledWith("a", "b", "c");
@@ -56,8 +46,8 @@ describe("realtime-callbacks", function() {
         it("should set 'this' to the global object", function() {
             let passed = false;
             const callback = function() {
-                expect(this).toBe(global); // eslint-disable-line no-invalid-this
-                expect(this.console).toBeTruthy(); // eslint-disable-line no-invalid-this
+                expect(this).toBe(global); // eslint-disable-line babel/no-invalid-this
+                expect(this.console).toBeTruthy(); // eslint-disable-line babel/no-invalid-this
                 passed = true;
             };
             callbacks.setTimeout(callback);
@@ -66,10 +56,10 @@ describe("realtime-callbacks", function() {
         });
 
         it("should handle timeouts of several seconds", function() {
-            const callback = expect.createSpy();
+            const callback = jest.fn();
             callbacks.setTimeout(callback, 2000);
 
-            expect(callback).toNotHaveBeenCalled();
+            expect(callback).not.toHaveBeenCalled();
             for (let i = 0; i < 4; i++) {
                 tick(500);
             }
@@ -77,24 +67,24 @@ describe("realtime-callbacks", function() {
         });
 
         it("should call multiple callbacks in the right order", function() {
-            const callback1 = expect.createSpy();
-            const callback2 = expect.createSpy();
-            const callback3 = expect.createSpy();
+            const callback1 = jest.fn();
+            const callback2 = jest.fn();
+            const callback3 = jest.fn();
             callbacks.setTimeout(callback2, 200);
             callbacks.setTimeout(callback1, 100);
             callbacks.setTimeout(callback3, 300);
 
-            expect(callback1).toNotHaveBeenCalled();
-            expect(callback2).toNotHaveBeenCalled();
-            expect(callback3).toNotHaveBeenCalled();
+            expect(callback1).not.toHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
+            expect(callback3).not.toHaveBeenCalled();
             tick(100);
             expect(callback1).toHaveBeenCalled();
-            expect(callback2).toNotHaveBeenCalled();
-            expect(callback3).toNotHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
+            expect(callback3).not.toHaveBeenCalled();
             tick(100);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
-            expect(callback3).toNotHaveBeenCalled();
+            expect(callback3).not.toHaveBeenCalled();
             tick(100);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
@@ -102,35 +92,34 @@ describe("realtime-callbacks", function() {
         });
 
         it("should treat -ve timeouts the same as a zero timeout", function() {
-            const callback1 = expect.createSpy();
-            const callback2 = expect.createSpy();
+            const callback1 = jest.fn();
+            const callback2 = jest.fn();
 
             // check that cb1 is called before cb2
-            callback1.andCall(function() {
-                expect(callback2).toNotHaveBeenCalled();
+            callback1.mockImplementation(function() {
+                expect(callback2).not.toHaveBeenCalled();
             });
 
             callbacks.setTimeout(callback1);
             callbacks.setTimeout(callback2, -100);
 
-            expect(callback1).toNotHaveBeenCalled();
-            expect(callback2).toNotHaveBeenCalled();
+            expect(callback1).not.toHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
             tick(0);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
         });
 
         it("should not get confused by chained calls", function() {
-            const callback2 = expect.createSpy();
-            const callback1 = expect.createSpy();
-            callback1.andCall(function() {
+            const callback2 = jest.fn();
+            const callback1 = jest.fn(function() {
                 callbacks.setTimeout(callback2, 0);
-                expect(callback2).toNotHaveBeenCalled();
+                expect(callback2).not.toHaveBeenCalled();
             });
 
             callbacks.setTimeout(callback1);
-            expect(callback1).toNotHaveBeenCalled();
-            expect(callback2).toNotHaveBeenCalled();
+            expect(callback1).not.toHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
             tick(0);
             expect(callback1).toHaveBeenCalled();
             // the fake timer won't actually run callbacks registered during
@@ -140,16 +129,15 @@ describe("realtime-callbacks", function() {
         });
 
         it("should be immune to exceptions", function() {
-            const callback1 = expect.createSpy();
-            callback1.andCall(function() {
+            const callback1 = jest.fn(function() {
                 throw new Error("prepare to die");
             });
-            const callback2 = expect.createSpy();
+            const callback2 = jest.fn();
             callbacks.setTimeout(callback1, 0);
             callbacks.setTimeout(callback2, 0);
 
-            expect(callback1).toNotHaveBeenCalled();
-            expect(callback2).toNotHaveBeenCalled();
+            expect(callback1).not.toHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
             tick(0);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
@@ -158,16 +146,16 @@ describe("realtime-callbacks", function() {
 
     describe("cancelTimeout", function() {
         it("should cancel a pending timeout", function() {
-            const callback = expect.createSpy();
+            const callback = jest.fn();
             const k = callbacks.setTimeout(callback);
             callbacks.clearTimeout(k);
             tick(0);
-            expect(callback).toNotHaveBeenCalled();
+            expect(callback).not.toHaveBeenCalled();
         });
 
         it("should not affect sooner timeouts", function() {
-            const callback1 = expect.createSpy();
-            const callback2 = expect.createSpy();
+            const callback1 = jest.fn();
+            const callback2 = jest.fn();
 
             callbacks.setTimeout(callback1, 100);
             const k = callbacks.setTimeout(callback2, 200);
@@ -175,10 +163,10 @@ describe("realtime-callbacks", function() {
 
             tick(100);
             expect(callback1).toHaveBeenCalled();
-            expect(callback2).toNotHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
 
             tick(150);
-            expect(callback2).toNotHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
         });
     });
 });

spec/unit/room-member.spec.js

@@ -1,10 +1,5 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const RoomMember = sdk.RoomMember;
-const utils = require("../test-utils");
-
-import expect from 'expect';
+import * as utils from "../test-utils";
+import {RoomMember} from "../../src/models/room-member";
 
 describe("RoomMember", function() {
     const roomId = "!foo:bar";
@@ -14,7 +9,6 @@ describe("RoomMember", function() {
     let member;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
         member = new RoomMember(roomId, userA);
     });
 
@@ -36,13 +30,7 @@ describe("RoomMember", function() {
             const url = member.getAvatarUrl(hsUrl);
             // we don't care about how the mxc->http conversion is done, other
             // than it contains the mxc body.
-            expect(url.indexOf("flibble/wibble")).toNotEqual(-1);
-        });
-
-        it("should return an identicon HTTP URL if allowDefault was set and there " +
-        "was no m.room.member event", function() {
-            const url = member.getAvatarUrl(hsUrl, 64, 64, "crop", true);
-            expect(url.indexOf("http")).toEqual(0); // don't care about form
+            expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
         });
 
         it("should return nothing if there is no m.room.member and allowDefault=false",
@@ -255,9 +243,9 @@ describe("RoomMember", function() {
             member.setMembershipEvent(joinEvent);
             expect(member.name).toEqual("Alice"); // prefer displayname
             member.setMembershipEvent(joinEvent, roomState);
-            expect(member.name).toNotEqual("Alice"); // it should disambig.
+            expect(member.name).not.toEqual("Alice"); // it should disambig.
             // user_id should be there somewhere
-            expect(member.name.indexOf(userA)).toNotEqual(-1);
+            expect(member.name.indexOf(userA)).not.toEqual(-1);
         });
 
         it("should emit 'RoomMember.membership' if the membership changes", function() {
@@ -285,5 +273,52 @@ describe("RoomMember", function() {
             member.setMembershipEvent(joinEvent); // no-op
             expect(emitCount).toEqual(1);
         });
+
+        it("should set 'name' to user_id if it is just whitespace", function() {
+            const joinEvent = utils.mkMembership({
+                event: true,
+                mship: "join",
+                user: userA,
+                room: roomId,
+                name: " \u200b ",
+            });
+
+            expect(member.name).toEqual(userA); // default = user_id
+            member.setMembershipEvent(joinEvent);
+            expect(member.name).toEqual(userA); // it should fallback because all whitespace
+        });
+
+        it("should disambiguate users on a fuzzy displayname match", function() {
+            const joinEvent = utils.mkMembership({
+                event: true,
+                mship: "join",
+                user: userA,
+                room: roomId,
+                name: "Alíce\u200b", // note diacritic and zero width char
+            });
+
+            const roomState = {
+                getStateEvents: function(type) {
+                    if (type !== "m.room.member") {
+                        return [];
+                    }
+                    return [
+                        utils.mkMembership({
+                            event: true, mship: "join", room: roomId,
+                            user: userC, name: "Alice",
+                        }),
+                        joinEvent,
+                    ];
+                },
+                getUserIdsWithDisplayName: function(displayName) {
+                    return [userA, userC];
+                },
+            };
+            expect(member.name).toEqual(userA); // default = user_id
+            member.setMembershipEvent(joinEvent, roomState);
+            expect(member.name).not.toEqual("Alíce"); // it should disambig.
+            // user_id should be there somewhere
+            expect(member.name.indexOf(userA)).not.toEqual(-1);
+        });
     });
 });

spec/unit/room-state.spec.js

@@ -1,11 +1,6 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const RoomState = sdk.RoomState;
-const RoomMember = sdk.RoomMember;
-const utils = require("../test-utils");
-
-import expect from 'expect';
+import * as utils from "../test-utils";
+import {RoomState} from "../../src/models/room-state";
+import {RoomMember} from "../../src/models/room-member";
 
 describe("RoomState", function() {
     const roomId = "!foo:bar";
@@ -17,7 +12,6 @@ describe("RoomState", function() {
     let state;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
         state = new RoomState(roomId);
         state.setStateEvents([
             utils.mkMembership({  // userA joined
@@ -49,8 +43,8 @@ describe("RoomState", function() {
             const members = state.getMembers();
             expect(members.length).toEqual(2);
             // ordering unimportant
-            expect([userA, userB].indexOf(members[0].userId)).toNotEqual(-1);
-            expect([userA, userB].indexOf(members[1].userId)).toNotEqual(-1);
+            expect([userA, userB].indexOf(members[0].userId)).not.toEqual(-1);
+            expect([userA, userB].indexOf(members[1].userId)).not.toEqual(-1);
         });
     });
 
@@ -120,8 +114,8 @@ describe("RoomState", function() {
             const events = state.getStateEvents("m.room.member");
             expect(events.length).toEqual(2);
             // ordering unimportant
-            expect([userA, userB].indexOf(events[0].getStateKey())).toNotEqual(-1);
-            expect([userA, userB].indexOf(events[1].getStateKey())).toNotEqual(-1);
+            expect([userA, userB].indexOf(events[0].getStateKey())).not.toEqual(-1);
+            expect([userA, userB].indexOf(events[1].getStateKey())).not.toEqual(-1);
         });
 
         it("should return a single MatrixEvent if a state_key was specified",
@@ -258,7 +252,7 @@ describe("RoomState", function() {
             });
             state.setStateEvents([memberEvent]);
 
-            expect(state.members[userA].setMembershipEvent).toNotHaveBeenCalled();
+            expect(state.members[userA].setMembershipEvent).not.toHaveBeenCalled();
             expect(state.members[userB].setMembershipEvent).toHaveBeenCalledWith(
                 memberEvent, state,
             );
@@ -306,7 +300,7 @@ describe("RoomState", function() {
             state.markOutOfBandMembersStarted();
             state.setOutOfBandMembers([oobMemberEvent]);
             const memberA = state.getMember(userA);
-            expect(memberA.events.member.getId()).toNotEqual(oobMemberEvent.getId());
+            expect(memberA.events.member.getId()).not.toEqual(oobMemberEvent.getId());
             expect(memberA.isOutOfBand()).toEqual(false);
         });
 

spec/unit/room.spec.js

@@ -1,14 +1,8 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const Room = sdk.Room;
-const RoomState = sdk.RoomState;
-const MatrixEvent = sdk.MatrixEvent;
-const EventStatus = sdk.EventStatus;
-const EventTimeline = sdk.EventTimeline;
-const utils = require("../test-utils");
-
-import expect from 'expect';
+import * as utils from "../test-utils";
+import {EventStatus, MatrixEvent} from "../../src/models/event";
+import {EventTimeline} from "../../src/models/event-timeline";
+import {RoomState} from "../../src/models/room-state";
+import {Room} from "../../src/models/room";
 
 describe("Room", function() {
     const roomId = "!foo:bar";
@@ -19,20 +13,19 @@ describe("Room", function() {
     let room;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
         room = new Room(roomId);
         // mock RoomStates
         room.oldState = room.getLiveTimeline()._startState =
-            utils.mock(sdk.RoomState, "oldState");
+            utils.mock(RoomState, "oldState");
         room.currentState = room.getLiveTimeline()._endState =
-            utils.mock(sdk.RoomState, "currentState");
+            utils.mock(RoomState, "currentState");
     });
 
     describe("getAvatarUrl", function() {
         const hsUrl = "https://my.home.server";
 
         it("should return the URL from m.room.avatar preferentially", function() {
-            room.currentState.getStateEvents.andCall(function(type, key) {
+            room.currentState.getStateEvents.mockImplementation(function(type, key) {
                 if (type === "m.room.avatar" && key === "") {
                     return utils.mkEvent({
                         event: true,
@@ -49,13 +42,7 @@ describe("Room", function() {
             const url = room.getAvatarUrl(hsUrl);
             // we don't care about how the mxc->http conversion is done, other
             // than it contains the mxc body.
-            expect(url.indexOf("flibble/wibble")).toNotEqual(-1);
-        });
-
-        it("should return an identicon HTTP URL if allowDefault was set and there " +
-        "was no m.room.avatar event", function() {
-            const url = room.getAvatarUrl(hsUrl, 64, 64, "crop", true);
-            expect(url.indexOf("http")).toEqual(0); // don't care about form
+            expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
         });
 
         it("should return nothing if there is no m.room.avatar and allowDefault=false",
@@ -67,13 +54,13 @@ describe("Room", function() {
 
     describe("getMember", function() {
         beforeEach(function() {
-            room.currentState.getMember.andCall(function(userId) {
+            room.currentState.getMember.mockImplementation(function(userId) {
                 return {
                     "@alice:bar": {
                         userId: userA,
                         roomId: roomId,
                     },
-                }[userId];
+                }[userId] || null;
             });
         });
 
@@ -82,7 +69,7 @@ describe("Room", function() {
         });
 
         it("should return the member from current state", function() {
-            expect(room.getMember(userA)).toNotEqual(null);
+            expect(room.getMember(userA)).not.toEqual(null);
         });
     });
 
@@ -104,7 +91,7 @@ describe("Room", function() {
                     user_ids: [userA],
                 },
             });
-            room.addLiveEvents([typing]);
+            room.addEphemeralEvents([typing]);
             expect(room.currentState.setTypingEvent).toHaveBeenCalledWith(typing);
         });
 
@@ -174,7 +161,7 @@ describe("Room", function() {
             );
             expect(events[0].forwardLooking).toBe(true);
             expect(events[1].forwardLooking).toBe(true);
-            expect(room.oldState.setStateEvents).toNotHaveBeenCalled();
+            expect(room.oldState.setStateEvents).not.toHaveBeenCalled();
         });
 
         it("should synthesize read receipts for the senders of events", function() {
@@ -183,7 +170,7 @@ describe("Room", function() {
                 membership: "join",
                 name: "Alice",
             };
-            room.currentState.getSentinelMember.andCall(function(uid) {
+            room.currentState.getSentinelMember.mockImplementation(function(uid) {
                 if (uid === userA) {
                     return sentinel;
                 }
@@ -292,13 +279,13 @@ describe("Room", function() {
                 membership: "join",
                 name: "Old Alice",
             };
-            room.currentState.getSentinelMember.andCall(function(uid) {
+            room.currentState.getSentinelMember.mockImplementation(function(uid) {
                 if (uid === userA) {
                     return sentinel;
                 }
                 return null;
             });
-            room.oldState.getSentinelMember.andCall(function(uid) {
+            room.oldState.getSentinelMember.mockImplementation(function(uid) {
                 if (uid === userA) {
                     return oldSentinel;
                 }
@@ -331,13 +318,13 @@ describe("Room", function() {
                 membership: "join",
                 name: "Old Alice",
             };
-            room.currentState.getSentinelMember.andCall(function(uid) {
+            room.currentState.getSentinelMember.mockImplementation(function(uid) {
                 if (uid === userA) {
                     return sentinel;
                 }
                 return null;
             });
-            room.oldState.getSentinelMember.andCall(function(uid) {
+            room.oldState.getSentinelMember.mockImplementation(function(uid) {
                 if (uid === userA) {
                     return oldSentinel;
                 }
@@ -379,7 +366,7 @@ describe("Room", function() {
             );
             expect(events[0].forwardLooking).toBe(false);
             expect(events[1].forwardLooking).toBe(false);
-            expect(room.currentState.setStateEvents).toNotHaveBeenCalled();
+            expect(room.currentState.setStateEvents).not.toHaveBeenCalled();
         });
     });
 
@@ -545,7 +532,7 @@ describe("Room", function() {
 
     describe("getJoinedMembers", function() {
         it("should return members whose membership is 'join'", function() {
-            room.currentState.getMembers.andCall(function() {
+            room.currentState.getMembers.mockImplementation(function() {
                 return [
                     { userId: "@alice:bar", membership: "join" },
                     { userId: "@bob:bar", membership: "invite" },
@@ -558,7 +545,7 @@ describe("Room", function() {
         });
 
         it("should return an empty list if no membership is 'join'", function() {
-            room.currentState.getMembers.andCall(function() {
+            room.currentState.getMembers.mockImplementation(function() {
                 return [
                     { userId: "@bob:bar", membership: "invite" },
                 ];
@@ -571,7 +558,7 @@ describe("Room", function() {
     describe("hasMembershipState", function() {
         it("should return true for a matching userId and membership",
         function() {
-            room.currentState.getMember.andCall(function(userId) {
+            room.currentState.getMember.mockImplementation(function(userId) {
                 return {
                     "@alice:bar": { userId: "@alice:bar", membership: "join" },
                     "@bob:bar": { userId: "@bob:bar", membership: "invite" },
@@ -582,7 +569,7 @@ describe("Room", function() {
 
         it("should return false if match membership but no match userId",
         function() {
-            room.currentState.getMember.andCall(function(userId) {
+            room.currentState.getMember.mockImplementation(function(userId) {
                 return {
                     "@alice:bar": { userId: "@alice:bar", membership: "join" },
                 }[userId];
@@ -592,7 +579,7 @@ describe("Room", function() {
 
         it("should return false if match userId but no match membership",
         function() {
-            room.currentState.getMember.andCall(function(userId) {
+            room.currentState.getMember.mockImplementation(function(userId) {
                 return {
                     "@alice:bar": { userId: "@alice:bar", membership: "join" },
                 }[userId];
@@ -602,7 +589,7 @@ describe("Room", function() {
 
         it("should return false if no match membership or userId",
         function() {
-            room.currentState.getMember.andCall(function(userId) {
+            room.currentState.getMember.mockImplementation(function(userId) {
                 return {
                     "@alice:bar": { userId: "@alice:bar", membership: "join" },
                 }[userId];
@@ -624,13 +611,10 @@ describe("Room", function() {
                 }, event: true,
             })]);
         };
-        const setAliases = function(aliases, stateKey) {
-            if (!stateKey) {
-                stateKey = "flibble";
-            }
+        const setAltAliases = function(aliases) {
             room.addLiveEvents([utils.mkEvent({
-                type: "m.room.aliases", room: roomId, skey: stateKey, content: {
-                    aliases: aliases,
+                type: "m.room.canonical_alias", room: roomId, skey: "", content: {
+                    alt_aliases: aliases,
                 }, event: true,
             })]);
         };
@@ -814,8 +798,8 @@ describe("Room", function() {
                 addMember(userC);
                 room.recalculate();
                 const name = room.name;
-                expect(name.indexOf(userB)).toNotEqual(-1, name);
-                expect(name.indexOf(userC)).toNotEqual(-1, name);
+                expect(name.indexOf(userB)).not.toEqual(-1, name);
+                expect(name.indexOf(userC)).not.toEqual(-1, name);
             });
 
             it("should return the names of members in a public (public join_rules)" +
@@ -827,8 +811,8 @@ describe("Room", function() {
                 addMember(userC);
                 room.recalculate();
                 const name = room.name;
-                expect(name.indexOf(userB)).toNotEqual(-1, name);
-                expect(name.indexOf(userC)).toNotEqual(-1, name);
+                expect(name.indexOf(userB)).not.toEqual(-1, name);
+                expect(name.indexOf(userC)).not.toEqual(-1, name);
             });
 
             it("should show the other user's name for public (public join_rules)" +
@@ -839,7 +823,7 @@ describe("Room", function() {
                 addMember(userB);
                 room.recalculate();
                 const name = room.name;
-                expect(name.indexOf(userB)).toNotEqual(-1, name);
+                expect(name.indexOf(userB)).not.toEqual(-1, name);
             });
 
             it("should show the other user's name for private " +
@@ -850,7 +834,7 @@ describe("Room", function() {
                 addMember(userB);
                 room.recalculate();
                 const name = room.name;
-                expect(name.indexOf(userB)).toNotEqual(-1, name);
+                expect(name.indexOf(userB)).not.toEqual(-1, name);
             });
 
             it("should show the other user's name for private" +
@@ -860,14 +844,14 @@ describe("Room", function() {
                 addMember(userB);
                 room.recalculate();
                 const name = room.name;
-                expect(name.indexOf(userB)).toNotEqual(-1, name);
+                expect(name.indexOf(userB)).not.toEqual(-1, name);
             });
 
             it("should show the room alias if one exists for private " +
             "(invite join_rules) rooms if a room name doesn't exist.", function() {
                 const alias = "#room_alias:here";
                 setJoinRule("invite");
-                setAliases([alias, "#another:one"]);
+                setAltAliases([alias, "#another:here"]);
                 room.recalculate();
                 const name = room.name;
                 expect(name).toEqual(alias);
@@ -877,7 +861,7 @@ describe("Room", function() {
             "(public join_rules) rooms if a room name doesn't exist.", function() {
                 const alias = "#room_alias:here";
                 setJoinRule("public");
-                setAliases([alias, "#another:one"]);
+                setAltAliases([alias, "#another:here"]);
                 room.recalculate();
                 const name = room.name;
                 expect(name).toEqual(alias);
@@ -1004,7 +988,7 @@ describe("Room", function() {
 
             it("should emit an event when a receipt is added",
             function() {
-                const listener = expect.createSpy();
+                const listener = jest.fn();
                 room.on("Room.receipt", listener);
 
                 const ts = 13787898424;
@@ -1175,7 +1159,7 @@ describe("Room", function() {
             it("should emit Room.tags event when new tags are " +
                "received on the event stream",
             function() {
-                const listener = expect.createSpy();
+                const listener = jest.fn();
                 room.on("Room.tags", listener);
 
                 const tags = { "m.foo": { "order": 0.5 } };
@@ -1318,6 +1302,9 @@ describe("Room", function() {
                     // events should already be MatrixEvents
                     return function(event) {return event;};
                 },
+                isCryptoEnabled() {
+                    return true;
+                },
                 isRoomEncrypted: function() {
                     return false;
                 },
@@ -1386,7 +1373,7 @@ describe("Room", function() {
             let hasThrown = false;
             try {
                 await room.loadMembersIfNeeded();
-            } catch(err) {
+            } catch (err) {
                 hasThrown = true;
             }
             expect(hasThrown).toEqual(true);

spec/unit/scheduler.spec.js

@@ -1,22 +1,18 @@
 // This file had a function whose name is all caps, which displeases eslint
 /* eslint new-cap: "off" */
 
-import 'source-map-support/register';
-import Promise from 'bluebird';
-const sdk = require("../..");
-const MatrixScheduler = sdk.MatrixScheduler;
-const MatrixError = sdk.MatrixError;
-const utils = require("../test-utils");
+import {defer} from '../../src/utils';
+import {MatrixError} from "../../src/http-api";
+import {MatrixScheduler} from "../../src/scheduler";
+import * as utils from "../test-utils";
 
-import expect from 'expect';
-import lolex from 'lolex';
+jest.useFakeTimers();
 
 describe("MatrixScheduler", function() {
-    let clock;
     let scheduler;
     let retryFn;
     let queueFn;
-    let defer;
+    let deferred;
     const roomId = "!foo:bar";
     const eventA = utils.mkMessage({
         user: "@alice:bar", room: roomId, event: true,
@@ -26,8 +22,6 @@ describe("MatrixScheduler", function() {
     });
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        clock = lolex.install();
         scheduler = new MatrixScheduler(function(ev, attempts, err) {
             if (retryFn) {
                 return retryFn(ev, attempts, err);
@@ -41,46 +35,44 @@ describe("MatrixScheduler", function() {
         });
         retryFn = null;
         queueFn = null;
-        defer = Promise.defer();
+        deferred = defer();
     });
 
-    afterEach(function() {
-        clock.uninstall();
-    });
-
-    it("should process events in a queue in a FIFO manner", function(done) {
+    it("should process events in a queue in a FIFO manner", async function() {
         retryFn = function() {
             return 0;
         };
         queueFn = function() {
             return "one_big_queue";
         };
-        const deferA = Promise.defer();
-        const deferB = Promise.defer();
-        let resolvedA = false;
+        const deferA = defer();
+        const deferB = defer();
+        let yieldedA = false;
         scheduler.setProcessFunction(function(event) {
-            if (resolvedA) {
+            if (yieldedA) {
                 expect(event).toEqual(eventB);
                 return deferB.promise;
             } else {
+                yieldedA = true;
                 expect(event).toEqual(eventA);
                 return deferA.promise;
             }
         });
-        scheduler.queueEvent(eventA);
-        scheduler.queueEvent(eventB).done(function() {
-            expect(resolvedA).toBe(true);
-            done();
-        });
-        deferA.resolve({});
-        resolvedA = true;
-        deferB.resolve({});
+        const abPromise = Promise.all([
+            scheduler.queueEvent(eventA),
+            scheduler.queueEvent(eventB),
+        ]);
+        deferB.resolve({b: true});
+        deferA.resolve({a: true});
+        const [a, b] = await abPromise;
+        expect(a.a).toEqual(true);
+        expect(b.b).toEqual(true);
     });
 
     it("should invoke the retryFn on failure and wait the amount of time specified",
-    function(done) {
+    async function() {
         const waitTimeMs = 1500;
-        const retryDefer = Promise.defer();
+        const retryDefer = defer();
         retryFn = function() {
             retryDefer.resolve();
             return waitTimeMs;
@@ -94,27 +86,29 @@ describe("MatrixScheduler", function() {
             procCount += 1;
             if (procCount === 1) {
                 expect(ev).toEqual(eventA);
-                return defer.promise;
+                return deferred.promise;
             } else if (procCount === 2) {
-                // don't care about this defer
-                return Promise.defer().promise;
+                // don't care about this deferred
+                return new Promise();
             }
             expect(procCount).toBeLessThan(3);
         });
 
         scheduler.queueEvent(eventA);
+        // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
+        // wait just long enough before it does
+        await Promise.resolve();
         expect(procCount).toEqual(1);
-        defer.reject({});
-        retryDefer.promise.done(function() {
-            expect(procCount).toEqual(1);
-            clock.tick(waitTimeMs);
-            expect(procCount).toEqual(2);
-            done();
-        });
+        deferred.reject({});
+        await retryDefer.promise;
+        expect(procCount).toEqual(1);
+        jest.advanceTimersByTime(waitTimeMs);
+        await Promise.resolve();
+        expect(procCount).toEqual(2);
     });
 
     it("should give up if the retryFn on failure returns -1 and try the next event",
-    function(done) {
+    async function() {
         // Queue A & B.
         // Reject A and return -1 on retry.
         // Expect B to be tried next and the promise for A to be rejected.
@@ -122,11 +116,11 @@ describe("MatrixScheduler", function() {
             return -1;
         };
         queueFn = function() {
- return "yep";
-};
+            return "yep";
+        };
 
-        const deferA = Promise.defer();
-        const deferB = Promise.defer();
+        const deferA = defer();
+        const deferB = defer();
         let procCount = 0;
         scheduler.setProcessFunction(function(ev) {
             procCount += 1;
@@ -142,13 +136,17 @@ describe("MatrixScheduler", function() {
 
         const globalA = scheduler.queueEvent(eventA);
         scheduler.queueEvent(eventB);
-
+        // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
+        // wait just long enough before it does
+        await Promise.resolve();
         expect(procCount).toEqual(1);
         deferA.reject({});
-        globalA.catch(function() {
+        try {
+            await globalA;
+        } catch (err) {
+            await Promise.resolve();
             expect(procCount).toEqual(2);
-            done();
-        });
+        }
     });
 
     it("should treat each queue separately", function(done) {
@@ -177,14 +175,14 @@ describe("MatrixScheduler", function() {
         const expectOrder = [
             eventA.getId(), eventB.getId(), eventD.getId(),
         ];
-        const deferA = Promise.defer();
+        const deferA = defer();
         scheduler.setProcessFunction(function(event) {
             const id = expectOrder.shift();
             expect(id).toEqual(event.getId());
             if (expectOrder.length === 0) {
                 done();
             }
-            return id === eventA.getId() ? deferA.promise : defer.promise;
+            return id === eventA.getId() ? deferA.promise : deferred.promise;
         });
         scheduler.queueEvent(eventA);
         scheduler.queueEvent(eventB);
@@ -195,7 +193,7 @@ describe("MatrixScheduler", function() {
         setTimeout(function() {
             deferA.resolve({});
         }, 1000);
-        clock.tick(1000);
+        jest.advanceTimersByTime(1000);
     });
 
     describe("queueEvent", function() {
@@ -298,9 +296,13 @@ describe("MatrixScheduler", function() {
             scheduler.setProcessFunction(function(ev) {
                 procCount += 1;
                 expect(ev).toEqual(eventA);
-                return defer.promise;
+                return deferred.promise;
+            });
+            // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
+            // wait just long enough before it does
+            Promise.resolve().then(() => {
+                expect(procCount).toEqual(1);
             });
-            expect(procCount).toEqual(1);
         });
 
         it("should not call the processFn if there are no queued events", function() {
@@ -310,7 +312,7 @@ describe("MatrixScheduler", function() {
             let procCount = 0;
             scheduler.setProcessFunction(function(ev) {
                 procCount += 1;
-                return defer.promise;
+                return deferred.promise;
             });
             expect(procCount).toEqual(0);
         });

spec/unit/sync-accumulator.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,19 +15,45 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-"use strict";
-import 'source-map-support/register';
-import utils from "../test-utils";
-import sdk from "../..";
-import expect from 'expect';
+import {SyncAccumulator} from "../../src/sync-accumulator";
 
-const SyncAccumulator = sdk.SyncAccumulator;
+// The event body & unsigned object get frozen to assert that they don't get altered
+// by the impl
+const RES_WITH_AGE = {
+    next_batch: "abc",
+    rooms: {
+        invite: {},
+        leave: {},
+        join: {
+            "!foo:bar": {
+                account_data: { events: [] },
+                ephemeral: { events: [] },
+                unread_notifications: {},
+                timeline: {
+                    events: [
+                        Object.freeze({
+                            content: {
+                                body: "This thing is happening right now!",
+                            },
+                            origin_server_ts: 123456789,
+                            sender: "@alice:localhost",
+                            type: "m.room.message",
+                            unsigned: Object.freeze({
+                                age: 50,
+                            }),
+                        }),
+                    ],
+                    prev_batch: "something",
+                },
+            },
+        },
+    },
+};
 
 describe("SyncAccumulator", function() {
     let sa;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
         sa = new SyncAccumulator({
             maxTimelineEntries: 10,
         });
@@ -374,6 +401,39 @@ describe("SyncAccumulator", function() {
             expect(summary["m.joined_member_count"]).toEqual(5);
             expect(summary["m.heroes"]).toEqual(["@bob:bar"]);
         });
+
+        it("should return correctly adjusted age attributes", () => {
+            const delta = 1000;
+            const startingTs = 1000;
+
+            const oldDateNow = Date.now;
+            try {
+                Date.now = jest.fn();
+                Date.now.mockReturnValue(startingTs);
+
+                sa.accumulate(RES_WITH_AGE);
+
+                Date.now.mockReturnValue(startingTs + delta);
+
+                const output = sa.getJSON();
+                expect(output.roomsData.join["!foo:bar"].timeline.events[0].unsigned.age).toEqual(
+                    RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0].unsigned.age + delta,
+                );
+                expect(Object.keys(output.roomsData.join["!foo:bar"].timeline.events[0])).toEqual(
+                    Object.keys(RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0]),
+                );
+            } finally {
+                Date.now = oldDateNow;
+            }
+        });
+
+        it("should mangle age without adding extra keys", () => {
+            sa.accumulate(RES_WITH_AGE);
+            const output = sa.getJSON();
+            expect(Object.keys(output.roomsData.join["!foo:bar"].timeline.events[0])).toEqual(
+                Object.keys(RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0]),
+            );
+        });
     });
 });
 

spec/unit/timeline-window.spec.js

@@ -1,13 +1,6 @@
-"use strict";
-import 'source-map-support/register';
-import Promise from 'bluebird';
-const sdk = require("../..");
-const EventTimeline = sdk.EventTimeline;
-const TimelineWindow = sdk.TimelineWindow;
-const TimelineIndex = require("../../lib/timeline-window").TimelineIndex;
-
-const utils = require("../test-utils");
-import expect from 'expect';
+import {EventTimeline} from "../../src/models/event-timeline";
+import {TimelineIndex, TimelineWindow} from "../../src/timeline-window";
+import * as utils from "../test-utils";
 
 const ROOM_ID = "roomId";
 const USER_ID = "userId";
@@ -67,10 +60,6 @@ function createLinkedTimelines() {
 
 
 describe("TimelineIndex", function() {
-    beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-    });
-
     describe("minIndex", function() {
         it("should return the min index relative to BaseIndex", function() {
             const timelineIndex = new TimelineIndex(createTimeline(), 0);
@@ -153,7 +142,7 @@ describe("TimelineWindow", function() {
     let timelineSet;
     let client;
     function createWindow(timeline, opts) {
-        timelineSet = {};
+        timelineSet = {getTimelineForEvent: () => null};
         client = {};
         client.getEventTimeline = function(timelineSet0, eventId0) {
             expect(timelineSet0).toBe(timelineSet);
@@ -163,12 +152,8 @@ describe("TimelineWindow", function() {
         return new TimelineWindow(client, timelineSet, opts);
     }
 
-    beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-    });
-
     describe("load", function() {
-        it("should initialise from the live timeline", function(done) {
+        it("should initialise from the live timeline", function() {
             const liveTimeline = createTimeline();
             const room = {};
             room.getLiveTimeline = function() {
@@ -176,17 +161,17 @@ describe("TimelineWindow", function() {
             };
 
             const timelineWindow = new TimelineWindow(undefined, room);
-            timelineWindow.load(undefined, 2).then(function() {
+            return timelineWindow.load(undefined, 2).then(function() {
                 const expectedEvents = liveTimeline.getEvents().slice(1);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).nodeify(done);
+            });
         });
 
-        it("should initialise from a specific event", function(done) {
+        it("should initialise from a specific event", function() {
             const timeline = createTimeline();
             const eventId = timeline.getEvents()[1].getId();
 
-            const timelineSet = {};
+            const timelineSet = {getTimelineForEvent: () => null};
             const client = {};
             client.getEventTimeline = function(timelineSet0, eventId0) {
                 expect(timelineSet0).toBe(timelineSet);
@@ -195,21 +180,20 @@ describe("TimelineWindow", function() {
             };
 
             const timelineWindow = new TimelineWindow(client, timelineSet);
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).nodeify(done);
+            });
         });
 
-        it("canPaginate should return false until load has returned",
-           function(done) {
+        it("canPaginate should return false until load has returned", function() {
             const timeline = createTimeline();
             timeline.setPaginationToken("toktok1", EventTimeline.BACKWARDS);
             timeline.setPaginationToken("toktok2", EventTimeline.FORWARDS);
 
             const eventId = timeline.getEvents()[1].getId();
 
-            const timelineSet = {};
+            const timelineSet = {getTimelineForEvent: () => null};
             const client = {};
 
             const timelineWindow = new TimelineWindow(client, timelineSet);
@@ -222,25 +206,24 @@ describe("TimelineWindow", function() {
                 return Promise.resolve(timeline);
             };
 
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
                 expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
                     .toBe(true);
                 expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
                     .toBe(true);
-            }).nodeify(done);
+            });
         });
     });
 
     describe("pagination", function() {
-        it("should be able to advance across the initial timeline",
-           function(done) {
+        it("should be able to advance across the initial timeline", function() {
             const timeline = createTimeline();
             const eventId = timeline.getEvents()[1].getId();
             const timelineWindow = createWindow(timeline);
 
-            timelineWindow.load(eventId, 1).then(function() {
+            return timelineWindow.load(eventId, 1).then(function() {
                 const expectedEvents = [timeline.getEvents()[1]];
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -277,15 +260,15 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(false);
-            }).nodeify(done);
+            });
         });
 
-        it("should advance into next timeline", function(done) {
+        it("should advance into next timeline", function() {
             const tls = createLinkedTimelines();
             const eventId = tls[0].getEvents()[1].getId();
             const timelineWindow = createWindow(tls[0], {windowLimit: 5});
 
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = tls[0].getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -322,15 +305,15 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(false);
-            }).nodeify(done);
+            });
         });
 
-        it("should retreat into previous timeline", function(done) {
+        it("should retreat into previous timeline", function() {
             const tls = createLinkedTimelines();
             const eventId = tls[1].getEvents()[1].getId();
             const timelineWindow = createWindow(tls[1], {windowLimit: 5});
 
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = tls[1].getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -367,10 +350,10 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(false);
-            }).nodeify(done);
+            });
         });
 
-        it("should make forward pagination requests", function(done) {
+        it("should make forward pagination requests", function() {
             const timeline = createTimeline();
             timeline.setPaginationToken("toktok", EventTimeline.FORWARDS);
 
@@ -386,7 +369,7 @@ describe("TimelineWindow", function() {
                 return Promise.resolve(true);
             };
 
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -399,11 +382,11 @@ describe("TimelineWindow", function() {
                 expect(success).toBe(true);
                 const expectedEvents = timeline.getEvents().slice(0, 5);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).nodeify(done);
+            });
         });
 
 
-        it("should make backward pagination requests", function(done) {
+        it("should make backward pagination requests", function() {
             const timeline = createTimeline();
             timeline.setPaginationToken("toktok", EventTimeline.BACKWARDS);
 
@@ -419,7 +402,7 @@ describe("TimelineWindow", function() {
                 return Promise.resolve(true);
             };
 
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -432,11 +415,10 @@ describe("TimelineWindow", function() {
                 expect(success).toBe(true);
                 const expectedEvents = timeline.getEvents().slice(1, 6);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).nodeify(done);
+            });
         });
 
-        it("should limit the number of unsuccessful pagination requests",
-        function(done) {
+        it("should limit the number of unsuccessful pagination requests", function() {
             const timeline = createTimeline();
             timeline.setPaginationToken("toktok", EventTimeline.FORWARDS);
 
@@ -452,7 +434,7 @@ describe("TimelineWindow", function() {
                 return Promise.resolve(true);
             };
 
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -471,7 +453,7 @@ describe("TimelineWindow", function() {
                     .toBe(false);
                 expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
                     .toBe(true);
-            }).nodeify(done);
+            });
         });
     });
 });

spec/unit/user.spec.js

@@ -1,17 +1,11 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const User = sdk.User;
-const utils = require("../test-utils");
-
-import expect from 'expect';
+import {User} from "../../src/models/user";
+import * as utils from "../test-utils";
 
 describe("User", function() {
     const userId = "@alice:bar";
     let user;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
         user = new User(userId);
     });
 

spec/unit/utils.spec.js

@@ -1,15 +1,6 @@
-"use strict";
-import 'source-map-support/register';
-const utils = require("../../lib/utils");
-const testUtils = require("../test-utils");
-
-import expect from 'expect';
+import * as utils from "../../src/utils";
 
 describe("utils", function() {
-    beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
-    });
-
     describe("encodeParams", function() {
         it("should url encode and concat with &s", function() {
             const params = {
@@ -135,7 +126,7 @@ describe("utils", function() {
                 utils.checkObjectHasKeys({
                     foo: "bar",
                 }, ["foo"]);
-            }).toNotThrow();
+            }).not.toThrow();
         });
     });
 
@@ -152,7 +143,7 @@ describe("utils", function() {
                 utils.checkObjectHasNoAdditionalKeys({
                             foo: "bar",
                         }, ["foo"]);
-            }).toNotThrow();
+            }).not.toThrow();
         });
     });
 

spec/unit/webrtc/call.spec.ts

@@ -0,0 +1,193 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {TestClient} from '../../TestClient';
+import {MatrixCall, CallErrorCode} from '../../../src/webrtc/call';
+
+const DUMMY_SDP = (
+    "v=0\r\n" +
+    "o=- 5022425983810148698 2 IN IP4 127.0.0.1\r\n" +
+    "s=-\r\nt=0 0\r\na=group:BUNDLE 0\r\n" +
+    "a=msid-semantic: WMS h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA\r\n" +
+    "m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126\r\n" +
+    "c=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:hLDR\r\n" +
+    "a=ice-pwd:bMGD9aOldHWiI+6nAq/IIlRw\r\n" +
+    "a=ice-options:trickle\r\n" +
+    "a=fingerprint:sha-256 E4:94:84:F9:4A:98:8A:56:F5:5F:FD:AF:72:B9:32:89:49:5C:4B:9A:" +
+        "4A:15:8E:41:8A:F3:69:E4:39:52:DC:D6\r\n" +
+    "a=setup:active\r\n" +
+    "a=mid:0\r\n" +
+    "a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n" +
+    "a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n" +
+    "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n" +
+    "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n" +
+    "a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n" +
+    "a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n" +
+    "a=sendrecv\r\n" +
+    "a=msid:h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA 4357098f-3795-4131-bff4-9ba9c0348c49\r\n" +
+    "a=rtcp-mux\r\n" +
+    "a=rtpmap:111 opus/48000/2\r\n" +
+    "a=rtcp-fb:111 transport-cc\r\n" +
+    "a=fmtp:111 minptime=10;useinbandfec=1\r\n" +
+    "a=rtpmap:103 ISAC/16000\r\n" +
+    "a=rtpmap:104 ISAC/32000\r\n" +
+    "a=rtpmap:9 G722/8000\r\n" +
+    "a=rtpmap:0 PCMU/8000\r\n" +
+    "a=rtpmap:8 PCMA/8000\r\n" +
+    "a=rtpmap:106 CN/32000\r\n" +
+    "a=rtpmap:105 CN/16000\r\n" +
+    "a=rtpmap:13 CN/8000\r\n" +
+    "a=rtpmap:110 telephone-event/48000\r\n" +
+    "a=rtpmap:112 telephone-event/32000\r\n" +
+    "a=rtpmap:113 telephone-event/16000\r\n" +
+    "a=rtpmap:126 telephone-event/8000\r\n" +
+    "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n"
+);
+
+class MockRTCPeerConnection {
+    localDescription: RTCSessionDescription;
+
+    constructor() {
+        this.localDescription = {
+            sdp: DUMMY_SDP,
+            type: 'offer',
+            toJSON: function() {},
+        };
+    }
+
+    addEventListener() {}
+    createOffer() {
+        return Promise.resolve({});
+    }
+    setRemoteDescription() {
+        return Promise.resolve();
+    }
+    setLocalDescription() {
+        return Promise.resolve();
+    }
+    close() {}
+}
+
+describe('Call', function() {
+    let client;
+    let call;
+    let prevNavigator;
+    let prevDocument;
+    let prevWindow;
+
+    beforeEach(function() {
+        prevNavigator = global.navigator;
+        prevDocument = global.document;
+        prevWindow = global.window;
+
+        global.navigator = {
+            mediaDevices: {
+                // @ts-ignore Mock
+                getUserMedia: () => {
+                    return {
+                        getTracks: () => [],
+                        getAudioTracks: () => [],
+                        getVideoTracks: () => [],
+                    };
+                },
+            },
+        };
+
+        global.window = {
+            // @ts-ignore Mock
+            RTCPeerConnection: MockRTCPeerConnection,
+            // @ts-ignore Mock
+            RTCSessionDescription: {},
+            // @ts-ignore Mock
+            RTCIceCandidate: {},
+            getUserMedia: {},
+        };
+        // @ts-ignore Mock
+        global.document = {};
+
+        client = new TestClient("@alice:foo", "somedevice", "token", undefined, {});
+        // We just stub out sendEvent: we're not interested in testing the client's
+        // event sending code here
+        client.client.sendEvent = () => {};
+        call = new MatrixCall({
+            client: client.client,
+            roomId: '!foo:bar',
+        });
+        // call checks one of these is wired up
+        call.on('error', () => {});
+    });
+
+    afterEach(function() {
+        client.stop();
+        global.navigator = prevNavigator;
+        global.window = prevWindow;
+        global.document = prevDocument;
+    });
+
+    it('should ignore candidate events from non-matching party ID', async function() {
+        await call.placeVoiceCall();
+        await call.onAnswerReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    answer: {
+                        sdp: DUMMY_SDP,
+                    },
+                };
+            },
+        });
+
+        call.peerConn.addIceCandidate = jest.fn();
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    candidates: [
+                        {
+                            candidate: '',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1);
+
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'some_other_party_id',
+                    candidates: [
+                        {
+                            candidate: '',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1);
+
+        // Hangup to stop timers
+        call.hangup(CallErrorCode.UserHangup, true);
+    });
+});

src/@types/event.ts

@@ -0,0 +1,89 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export enum EventType {
+    // Room state events
+    RoomCanonicalAlias = "m.room.canonical_alias",
+    RoomCreate = "m.room.create",
+    RoomJoinRules = "m.room.join_rules",
+    RoomMember = "m.room.member",
+    RoomThirdPartyInvite = "m.room.third_party_invite",
+    RoomPowerLevels = "m.room.power_levels",
+    RoomName = "m.room.name",
+    RoomTopic = "m.room.topic",
+    RoomAvatar = "m.room.avatar",
+    RoomPinnedEvents = "m.room.pinned_events",
+    RoomEncryption = "m.room.encryption",
+    RoomHistoryVisibility = "m.room.history_visibility",
+    RoomGuestAccess = "m.room.guest_access",
+    RoomServerAcl = "m.room.server_acl",
+    RoomTombstone = "m.room.tombstone",
+    /**
+     * @deprecated Should not be used.
+     */
+    RoomAliases = "m.room.aliases", // deprecated https://matrix.org/docs/spec/client_server/r0.6.1#historical-events
+
+    // Room timeline events
+    RoomRedaction = "m.room.redaction",
+    RoomMessage = "m.room.message",
+    RoomMessageEncrypted = "m.room.encrypted",
+    Sticker = "m.sticker",
+    CallInvite = "m.call.invite",
+    CallCandidates = "m.call.candidates",
+    CallAnswer = "m.call.answer",
+    CallHangup = "m.call.hangup",
+    CallReject = "m.call.reject",
+    CallSelectAnswer = "m.call.select_answer",
+    CallNegotiate = "m.call.negotiate",
+    CallReplaces = "m.call.replaces",
+    KeyVerificationRequest = "m.key.verification.request",
+    KeyVerificationStart = "m.key.verification.start",
+    KeyVerificationCancel = "m.key.verification.cancel",
+    KeyVerificationMac = "m.key.verification.mac",
+    // use of this is discouraged https://matrix.org/docs/spec/client_server/r0.6.1#m-room-message-feedback
+    RoomMessageFeedback = "m.room.message.feedback",
+
+    // Room ephemeral events
+    Typing = "m.typing",
+    Receipt = "m.receipt",
+    Presence = "m.presence",
+
+    // Room account_data events
+    FullyRead = "m.fully_read",
+    Tag = "m.tag",
+
+    // User account_data events
+    PushRules = "m.push_rules",
+    Direct = "m.direct",
+    IgnoredUserList = "m.ignored_user_list",
+
+    // to_device events
+    RoomKey = "m.room_key",
+    RoomKeyRequest = "m.room_key_request",
+    ForwardedRoomKey = "m.forwarded_room_key",
+    Dummy = "m.dummy",
+}
+
+export enum MsgType {
+    Text = "m.text",
+    Emote = "m.emote",
+    Notice = "m.notice",
+    Image = "m.image",
+    File = "m.file",
+    Audio = "m.audio",
+    Location = "m.location",
+    Video = "m.video",
+}

src/@types/global.d.ts

@@ -0,0 +1,87 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// this is needed to tell TS about global.Olm
+import * as Olm from "olm"; // eslint-disable-line @typescript-eslint/no-unused-vars
+
+export {};
+
+declare global {
+    namespace NodeJS {
+        interface Global {
+            localStorage: Storage;
+        }
+    }
+
+    interface Window {
+        electron?: Electron;
+    }
+
+    interface Electron {
+        getDesktopCapturerSources(options: GetSourcesOptions): Promise<Array<DesktopCapturerSource>>;
+    }
+
+    interface MediaDevices {
+        // This is experimental and types don't know about it yet
+        // https://github.com/microsoft/TypeScript/issues/33232
+        getDisplayMedia(constraints: MediaStreamConstraints | DesktopCapturerConstraints): Promise<MediaStream>;
+        getUserMedia(constraints: MediaStreamConstraints | DesktopCapturerConstraints): Promise<MediaStream>;
+    }
+
+    export interface DesktopCapturerConstraints {
+        audio: boolean | {
+            mandatory: {
+                chromeMediaSource: string;
+                chromeMediaSourceId: string;
+            };
+        };
+        video: boolean | {
+            mandatory: {
+                chromeMediaSource: string;
+                chromeMediaSourceId: string;
+            };
+        };
+    }
+
+    export interface DesktopCapturerSource {
+        id: string;
+        name: string;
+        thumbnailURL: string;
+    }
+
+    interface GetSourcesOptions {
+        types: Array<string>;
+        thumbnailSize?: {
+            height: number;
+            width: number;
+        };
+        fetchWindowIcons?: boolean;
+    }
+
+    interface HTMLAudioElement {
+        // sinkId & setSinkId are experimental and typescript doesn't know about them
+        sinkId: string;
+        setSinkId(outputId: string);
+    }
+
+    interface DummyInterfaceWeShouldntBeUsingThis {}
+
+    interface Navigator {
+        // We check for the webkit-prefixed getUserMedia to detect if we're
+        // on webkit: we should check if we still need to do this
+        webkitGetUserMedia: DummyInterfaceWeShouldntBeUsingThis;
+    }
+}

src/ReEmitter.js

@@ -1,46 +0,0 @@
-/*
-Copyright 2015, 2016 OpenMarket Ltd
-Copyright 2017 Vector Creations Ltd
-Copyright 2017 New Vector Ltd
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/**
- * @module
- */
-
-export default class Reemitter {
-    constructor(target) {
-        this.target = target;
-
-        // We keep one bound event handler for each event name so we know
-        // what event is arriving
-        this.boundHandlers = {};
-    }
-
-    _handleEvent(eventName, ...args) {
-        this.target.emit(eventName, ...args);
-    }
-
-    reEmit(source, eventNames) {
-        for (const eventName of eventNames) {
-            if (this.boundHandlers[eventName] === undefined) {
-                this.boundHandlers[eventName] = this._handleEvent.bind(this, eventName);
-            }
-            const boundHandler = this.boundHandlers[eventName];
-
-            source.on(eventName, boundHandler);
-        }
-    }
-}

src/ReEmitter.ts

@@ -0,0 +1,50 @@
+/*
+Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventEmitter } from "events";
+
+export class ReEmitter {
+    private target: EventEmitter;
+
+    constructor(target: EventEmitter) {
+        this.target = target;
+    }
+
+    reEmit(source: EventEmitter, eventNames: string[]) {
+        for (const eventName of eventNames) {
+            // We include the source as the last argument for event handlers which may need it,
+            // such as read receipt listeners on the client class which won't have the context
+            // of the room.
+            const forSource = (...args) => {
+                // EventEmitter special cases 'error' to make the emit function throw if no
+                // handler is attached, which sort of makes sense for making sure that something
+                // handles an error, but for re-emitting, there could be a listener on the original
+                // source object so the test doesn't really work. We *could* try to replicate the
+                // same logic and throw if there is no listener on either the source or the target,
+                // but this behaviour is fairly undesireable for us anyway: the main place we throw
+                // 'error' events is for calls, where error events are usually emitted some time
+                // later by a different part of the code where 'emit' throwing because the app hasn't
+                // added an error handler isn't terribly helpful. (A better fix in retrospect may
+                // have been to just avoid using the event name 'error', but backwards compat...)
+                if (eventName === 'error' && this.target.listenerCount('error') === 0) return;
+                this.target.emit(eventName, ...args, source);
+            };
+            source.on(eventName, forSource);
+        }
+    }
+}

src/autodiscovery.js

@@ -0,0 +1,536 @@
+/*
+Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/** @module auto-discovery */
+
+import {logger} from './logger';
+import {URL as NodeURL} from "url";
+
+// Dev note: Auto discovery is part of the spec.
+// See: https://matrix.org/docs/spec/client_server/r0.4.0.html#server-discovery
+
+/**
+ * Description for what an automatically discovered client configuration
+ * would look like. Although this is a class, it is recommended that it
+ * be treated as an interface definition rather than as a class.
+ *
+ * Additional properties than those defined here may be present, and
+ * should follow the Java package naming convention.
+ */
+class DiscoveredClientConfig { // eslint-disable-line no-unused-vars
+    // Dev note: this is basically a copy/paste of the .well-known response
+    // object as defined in the spec. It does have additional information,
+    // however. Overall, this exists to serve as a place for documentation
+    // and not functionality.
+    // See https://matrix.org/docs/spec/client_server/r0.4.0.html#get-well-known-matrix-client
+
+    constructor() {
+        /**
+         * The homeserver configuration the client should use. This will
+         * always be present on the object.
+         * @type {{state: string, base_url: string}} The configuration.
+         */
+        this["m.homeserver"] = {
+            /**
+             * The lookup result state. If this is anything other than
+             * AutoDiscovery.SUCCESS then base_url may be falsey. Additionally,
+             * if this is not AutoDiscovery.SUCCESS then the client should
+             * assume the other properties in the client config (such as
+             * the identity server configuration) are not valid.
+             */
+            state: AutoDiscovery.PROMPT,
+
+            /**
+             * If the state is AutoDiscovery.FAIL_ERROR or .FAIL_PROMPT
+             * then this will contain a human-readable (English) message
+             * for what went wrong. If the state is none of those previously
+             * mentioned, this will be falsey.
+             */
+            error: "Something went wrong",
+
+            /**
+             * The base URL clients should use to talk to the homeserver,
+             * particularly for the login process. May be falsey if the
+             * state is not AutoDiscovery.SUCCESS.
+             */
+            base_url: "https://matrix.org",
+        };
+
+        /**
+         * The identity server configuration the client should use. This
+         * will always be present on teh object.
+         * @type {{state: string, base_url: string}} The configuration.
+         */
+        this["m.identity_server"] = {
+            /**
+             * The lookup result state. If this is anything other than
+             * AutoDiscovery.SUCCESS then base_url may be falsey.
+             */
+            state: AutoDiscovery.PROMPT,
+
+            /**
+             * The base URL clients should use for interacting with the
+             * identity server. May be falsey if the state is not
+             * AutoDiscovery.SUCCESS.
+             */
+            base_url: "https://vector.im",
+        };
+    }
+}
+
+/**
+ * Utilities for automatically discovery resources, such as homeservers
+ * for users to log in to.
+ */
+export class AutoDiscovery {
+    // Dev note: the constants defined here are related to but not
+    // exactly the same as those in the spec. This is to hopefully
+    // translate the meaning of the states in the spec, but also
+    // support our own if needed.
+
+    static get ERROR_INVALID() {
+        return "Invalid homeserver discovery response";
+    }
+
+    static get ERROR_GENERIC_FAILURE() {
+        return "Failed to get autodiscovery configuration from server";
+    }
+
+    static get ERROR_INVALID_HS_BASE_URL() {
+        return "Invalid base_url for m.homeserver";
+    }
+
+    static get ERROR_INVALID_HOMESERVER() {
+        return "Homeserver URL does not appear to be a valid Matrix homeserver";
+    }
+
+    static get ERROR_INVALID_IS_BASE_URL() {
+        return "Invalid base_url for m.identity_server";
+    }
+
+    static get ERROR_INVALID_IDENTITY_SERVER() {
+        return "Identity server URL does not appear to be a valid identity server";
+    }
+
+    static get ERROR_INVALID_IS() {
+        return "Invalid identity server discovery response";
+    }
+
+    static get ERROR_MISSING_WELLKNOWN() {
+        return "No .well-known JSON file found";
+    }
+
+    static get ERROR_INVALID_JSON() {
+        return "Invalid JSON";
+    }
+
+    static get ALL_ERRORS() {
+        return [
+            AutoDiscovery.ERROR_INVALID,
+            AutoDiscovery.ERROR_GENERIC_FAILURE,
+            AutoDiscovery.ERROR_INVALID_HS_BASE_URL,
+            AutoDiscovery.ERROR_INVALID_HOMESERVER,
+            AutoDiscovery.ERROR_INVALID_IS_BASE_URL,
+            AutoDiscovery.ERROR_INVALID_IDENTITY_SERVER,
+            AutoDiscovery.ERROR_INVALID_IS,
+            AutoDiscovery.ERROR_MISSING_WELLKNOWN,
+            AutoDiscovery.ERROR_INVALID_JSON,
+        ];
+    }
+
+    /**
+     * The auto discovery failed. The client is expected to communicate
+     * the error to the user and refuse logging in.
+     * @return {string}
+     * @constructor
+     */
+    static get FAIL_ERROR() { return "FAIL_ERROR"; }
+
+    /**
+     * The auto discovery failed, however the client may still recover
+     * from the problem. The client is recommended to that the same
+     * action it would for PROMPT while also warning the user about
+     * what went wrong. The client may also treat this the same as
+     * a FAIL_ERROR state.
+     * @return {string}
+     * @constructor
+     */
+    static get FAIL_PROMPT() { return "FAIL_PROMPT"; }
+
+    /**
+     * The auto discovery didn't fail but did not find anything of
+     * interest. The client is expected to prompt the user for more
+     * information, or fail if it prefers.
+     * @return {string}
+     * @constructor
+     */
+    static get PROMPT() { return "PROMPT"; }
+
+    /**
+     * The auto discovery was successful.
+     * @return {string}
+     * @constructor
+     */
+    static get SUCCESS() { return "SUCCESS"; }
+
+    /**
+     * Validates and verifies client configuration information for purposes
+     * of logging in. Such information includes the homeserver URL
+     * and identity server URL the client would want. Additional details
+     * may also be included, and will be transparently brought into the
+     * response object unaltered.
+     * @param {string} wellknown The configuration object itself, as returned
+     * by the .well-known auto-discovery endpoint.
+     * @return {Promise<DiscoveredClientConfig>} Resolves to the verified
+     * configuration, which may include error states. Rejects on unexpected
+     * failure, not when verification fails.
+     */
+    static async fromDiscoveryConfig(wellknown) {
+        // Step 1 is to get the config, which is provided to us here.
+
+        // We default to an error state to make the first few checks easier to
+        // write. We'll update the properties of this object over the duration
+        // of this function.
+        const clientConfig = {
+            "m.homeserver": {
+                state: AutoDiscovery.FAIL_ERROR,
+                error: AutoDiscovery.ERROR_INVALID,
+                base_url: null,
+            },
+            "m.identity_server": {
+                // Technically, we don't have a problem with the identity server
+                // config at this point.
+                state: AutoDiscovery.PROMPT,
+                error: null,
+                base_url: null,
+            },
+        };
+
+        if (!wellknown || !wellknown["m.homeserver"]) {
+            logger.error("No m.homeserver key in config");
+
+            clientConfig["m.homeserver"].state = AutoDiscovery.FAIL_PROMPT;
+            clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_INVALID;
+
+            return Promise.resolve(clientConfig);
+        }
+
+        if (!wellknown["m.homeserver"]["base_url"]) {
+            logger.error("No m.homeserver base_url in config");
+
+            clientConfig["m.homeserver"].state = AutoDiscovery.FAIL_PROMPT;
+            clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_INVALID_HS_BASE_URL;
+
+            return Promise.resolve(clientConfig);
+        }
+
+        // Step 2: Make sure the homeserver URL is valid *looking*. We'll make
+        // sure it points to a homeserver in Step 3.
+        const hsUrl = this._sanitizeWellKnownUrl(
+            wellknown["m.homeserver"]["base_url"],
+        );
+        if (!hsUrl) {
+            logger.error("Invalid base_url for m.homeserver");
+            clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_INVALID_HS_BASE_URL;
+            return Promise.resolve(clientConfig);
+        }
+
+        // Step 3: Make sure the homeserver URL points to a homeserver.
+        const hsVersions = await this._fetchWellKnownObject(
+            `${hsUrl}/_matrix/client/versions`,
+        );
+        if (!hsVersions || !hsVersions.raw["versions"]) {
+            logger.error("Invalid /versions response");
+            clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_INVALID_HOMESERVER;
+
+            // Supply the base_url to the caller because they may be ignoring liveliness
+            // errors, like this one.
+            clientConfig["m.homeserver"].base_url = hsUrl;
+
+            return Promise.resolve(clientConfig);
+        }
+
+        // Step 4: Now that the homeserver looks valid, update our client config.
+        clientConfig["m.homeserver"] = {
+            state: AutoDiscovery.SUCCESS,
+            error: null,
+            base_url: hsUrl,
+        };
+
+        // Step 5: Try to pull out the identity server configuration
+        let isUrl = "";
+        if (wellknown["m.identity_server"]) {
+            // We prepare a failing identity server response to save lines later
+            // in this branch.
+            const failingClientConfig = {
+                "m.homeserver": clientConfig["m.homeserver"],
+                "m.identity_server": {
+                    state: AutoDiscovery.FAIL_PROMPT,
+                    error: AutoDiscovery.ERROR_INVALID_IS,
+                    base_url: null,
+                },
+            };
+
+            // Step 5a: Make sure the URL is valid *looking*. We'll make sure it
+            // points to an identity server in Step 5b.
+            isUrl = this._sanitizeWellKnownUrl(
+                wellknown["m.identity_server"]["base_url"],
+            );
+            if (!isUrl) {
+                logger.error("Invalid base_url for m.identity_server");
+                failingClientConfig["m.identity_server"].error =
+                    AutoDiscovery.ERROR_INVALID_IS_BASE_URL;
+                return Promise.resolve(failingClientConfig);
+            }
+
+            // Step 5b: Verify there is an identity server listening on the provided
+            // URL.
+            const isResponse = await this._fetchWellKnownObject(
+                `${isUrl}/_matrix/identity/api/v1`,
+            );
+            if (!isResponse || !isResponse.raw || isResponse.action !== "SUCCESS") {
+                logger.error("Invalid /api/v1 response");
+                failingClientConfig["m.identity_server"].error =
+                    AutoDiscovery.ERROR_INVALID_IDENTITY_SERVER;
+
+                // Supply the base_url to the caller because they may be ignoring
+                // liveliness errors, like this one.
+                failingClientConfig["m.identity_server"].base_url = isUrl;
+
+                return Promise.resolve(failingClientConfig);
+            }
+        }
+
+        // Step 6: Now that the identity server is valid, or never existed,
+        // populate the IS section.
+        if (isUrl && isUrl.length > 0) {
+            clientConfig["m.identity_server"] = {
+                state: AutoDiscovery.SUCCESS,
+                error: null,
+                base_url: isUrl,
+            };
+        }
+
+        // Step 7: Copy any other keys directly into the clientConfig. This is for
+        // things like custom configuration of services.
+        Object.keys(wellknown)
+            .map((k) => {
+                if (k === "m.homeserver" || k === "m.identity_server") {
+                    // Only copy selected parts of the config to avoid overwriting
+                    // properties computed by the validation logic above.
+                    const notProps = ["error", "state", "base_url"];
+                    for (const prop of Object.keys(wellknown[k])) {
+                        if (notProps.includes(prop)) continue;
+                        clientConfig[k][prop] = wellknown[k][prop];
+                    }
+                } else {
+                    // Just copy the whole thing over otherwise
+                    clientConfig[k] = wellknown[k];
+                }
+            });
+
+        // Step 8: Give the config to the caller (finally)
+        return Promise.resolve(clientConfig);
+    }
+
+    /**
+     * Attempts to automatically discover client configuration information
+     * prior to logging in. Such information includes the homeserver URL
+     * and identity server URL the client would want. Additional details
+     * may also be discovered, and will be transparently included in the
+     * response object unaltered.
+     * @param {string} domain The homeserver domain to perform discovery
+     * on. For example, "matrix.org".
+     * @return {Promise<DiscoveredClientConfig>} Resolves to the discovered
+     * configuration, which may include error states. Rejects on unexpected
+     * failure, not when discovery fails.
+     */
+    static async findClientConfig(domain) {
+        if (!domain || typeof(domain) !== "string" || domain.length === 0) {
+            throw new Error("'domain' must be a string of non-zero length");
+        }
+
+        // We use a .well-known lookup for all cases. According to the spec, we
+        // can do other discovery mechanisms if we want such as custom lookups
+        // however we won't bother with that here (mostly because the spec only
+        // supports .well-known right now).
+        //
+        // By using .well-known, we need to ensure we at least pull out a URL
+        // for the homeserver. We don't really need an identity server configuration
+        // but will return one anyways (with state PROMPT) to make development
+        // easier for clients. If we can't get a homeserver URL, all bets are
+        // off on the rest of the config and we'll assume it is invalid too.
+
+        // We default to an error state to make the first few checks easier to
+        // write. We'll update the properties of this object over the duration
+        // of this function.
+        const clientConfig = {
+            "m.homeserver": {
+                state: AutoDiscovery.FAIL_ERROR,
+                error: AutoDiscovery.ERROR_INVALID,
+                base_url: null,
+            },
+            "m.identity_server": {
+                // Technically, we don't have a problem with the identity server
+                // config at this point.
+                state: AutoDiscovery.PROMPT,
+                error: null,
+                base_url: null,
+            },
+        };
+
+        // Step 1: Actually request the .well-known JSON file and make sure it
+        // at least has a homeserver definition.
+        const wellknown = await this._fetchWellKnownObject(
+            `https://${domain}/.well-known/matrix/client`,
+        );
+        if (!wellknown || wellknown.action !== "SUCCESS") {
+            logger.error("No response or error when parsing .well-known");
+            if (wellknown.reason) logger.error(wellknown.reason);
+            if (wellknown.action === "IGNORE") {
+                clientConfig["m.homeserver"] = {
+                    state: AutoDiscovery.PROMPT,
+                    error: null,
+                    base_url: null,
+                };
+            } else {
+                // this can only ever be FAIL_PROMPT at this point.
+                clientConfig["m.homeserver"].state = AutoDiscovery.FAIL_PROMPT;
+                clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_INVALID;
+            }
+            return Promise.resolve(clientConfig);
+        }
+
+        // Step 2: Validate and parse the config
+        return AutoDiscovery.fromDiscoveryConfig(wellknown.raw);
+    }
+
+    /**
+     * Gets the raw discovery client configuration for the given domain name.
+     * Should only be used if there's no validation to be done on the resulting
+     * object, otherwise use findClientConfig().
+     * @param {string} domain The domain to get the client config for.
+     * @returns {Promise<object>} Resolves to the domain's client config. Can
+     * be an empty object.
+     */
+    static async getRawClientConfig(domain) {
+        if (!domain || typeof(domain) !== "string" || domain.length === 0) {
+            throw new Error("'domain' must be a string of non-zero length");
+        }
+
+        const response = await this._fetchWellKnownObject(
+            `https://${domain}/.well-known/matrix/client`,
+        );
+        if (!response) return {};
+        return response.raw || {};
+    }
+
+    /**
+     * Sanitizes a given URL to ensure it is either an HTTP or HTTP URL and
+     * is suitable for the requirements laid out by .well-known auto discovery.
+     * If valid, the URL will also be stripped of any trailing slashes.
+     * @param {string} url The potentially invalid URL to sanitize.
+     * @return {string|boolean} The sanitized URL or a falsey value if the URL is invalid.
+     * @private
+     */
+    static _sanitizeWellKnownUrl(url) {
+        if (!url) return false;
+
+        try {
+            // We have to try and parse the URL using the NodeJS URL
+            // library if we're on NodeJS and use the browser's URL
+            // library when we're in a browser. To accomplish this, we
+            // try the NodeJS version first and fall back to the browser.
+            let parsed = null;
+            try {
+                if (NodeURL) parsed = new NodeURL(url);
+                else parsed = new URL(url);
+            } catch (e) {
+                parsed = new URL(url);
+            }
+
+            if (!parsed || !parsed.hostname) return false;
+            if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return false;
+
+            const port = parsed.port ? `:${parsed.port}` : "";
+            const path = parsed.pathname ? parsed.pathname : "";
+            let saferUrl = `${parsed.protocol}//${parsed.hostname}${port}${path}`;
+            if (saferUrl.endsWith("/")) {
+                saferUrl = saferUrl.substring(0, saferUrl.length - 1);
+            }
+            return saferUrl;
+        } catch (e) {
+            logger.error(e);
+            return false;
+        }
+    }
+
+    /**
+     * Fetches a JSON object from a given URL, as expected by all .well-known
+     * related lookups. If the server gives a 404 then the `action` will be
+     * IGNORE. If the server returns something that isn't JSON, the `action`
+     * will be FAIL_PROMPT. For any other failure the `action` will be FAIL_PROMPT.
+     *
+     * The returned object will be a result of the call in object form with
+     * the following properties:
+     *   raw: The JSON object returned by the server.
+     *   action: One of SUCCESS, IGNORE, or FAIL_PROMPT.
+     *   reason: Relatively human readable description of what went wrong.
+     *   error: The actual Error, if one exists.
+     * @param {string} url The URL to fetch a JSON object from.
+     * @return {Promise<object>} Resolves to the returned state.
+     * @private
+     */
+    static async _fetchWellKnownObject(url) {
+        return new Promise(function(resolve, reject) {
+            const request = require("./matrix").getRequest();
+            if (!request) throw new Error("No request library available");
+            request(
+                { method: "GET", uri: url, timeout: 5000 },
+                (err, response, body) => {
+                    if (err || response &&
+                        (response.statusCode < 200 || response.statusCode >= 300)
+                    ) {
+                        let action = "FAIL_PROMPT";
+                        let reason = (err ? err.message : null) || "General failure";
+                        if (response && response.statusCode === 404) {
+                            action = "IGNORE";
+                            reason = AutoDiscovery.ERROR_MISSING_WELLKNOWN;
+                        }
+                        resolve({raw: {}, action: action, reason: reason, error: err});
+                        return;
+                    }
+
+                    try {
+                        resolve({raw: JSON.parse(body), action: "SUCCESS"});
+                    } catch (e) {
+                        let reason = AutoDiscovery.ERROR_INVALID;
+                        if (e.name === "SyntaxError") {
+                            reason = AutoDiscovery.ERROR_INVALID_JSON;
+                        }
+                        resolve({
+                            raw: {},
+                            action: "FAIL_PROMPT",
+                            reason: reason,
+                            error: e,
+                        });
+                    }
+                },
+            );
+        });
+    }
+}

src/browser-index.js

@@ -0,0 +1,54 @@
+/*
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import * as matrixcs from "./matrix";
+import request from "browser-request";
+import queryString from "qs";
+
+matrixcs.request(function(opts, fn) {
+    // We manually fix the query string for browser-request because
+    // it doesn't correctly handle cases like ?via=one&via=two. Instead
+    // we mimic `request`'s query string interface to make it all work
+    // as expected.
+    // browser-request will happily take the constructed string as the
+    // query string without trying to modify it further.
+    opts.qs = queryString.stringify(opts.qs || {}, opts.qsStringifyOptions);
+    return request(opts, fn);
+});
+
+// just *accessing* indexedDB throws an exception in firefox with
+// indexeddb disabled.
+let indexedDB;
+try {
+    indexedDB = global.indexedDB;
+} catch (e) {}
+
+// if our browser (appears to) support indexeddb, use an indexeddb crypto store.
+if (indexedDB) {
+    matrixcs.setCryptoStoreFactory(
+        function() {
+            return new matrixcs.IndexedDBCryptoStore(
+                indexedDB, "matrix-js-sdk:crypto",
+            );
+        },
+    );
+}
+
+// We export 3 things to make browserify happy as well as downstream projects.
+// It's awkward, but required.
+export * from "./matrix";
+export default matrixcs; // keep export for browserify package deps
+global.matrixcs = matrixcs;

src/content-helpers.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,88 +14,86 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
 
 /** @module ContentHelpers */
-module.exports = {
-    /**
-     * Generates the content for a HTML Message event
-     * @param {string} body the plaintext body of the message
-     * @param {string} htmlBody the HTML representation of the message
-     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
-     */
-    makeHtmlMessage: function(body, htmlBody) {
-        return {
-            msgtype: "m.text",
-            format: "org.matrix.custom.html",
-            body: body,
-            formatted_body: htmlBody,
-        };
-    },
 
-    /**
-     * Generates the content for a HTML Notice event
-     * @param {string} body the plaintext body of the notice
-     * @param {string} htmlBody the HTML representation of the notice
-     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
-     */
-    makeHtmlNotice: function(body, htmlBody) {
-        return {
-            msgtype: "m.notice",
-            format: "org.matrix.custom.html",
-            body: body,
-            formatted_body: htmlBody,
-        };
-    },
+/**
+ * Generates the content for a HTML Message event
+ * @param {string} body the plaintext body of the message
+ * @param {string} htmlBody the HTML representation of the message
+ * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+ */
+export function makeHtmlMessage(body, htmlBody) {
+    return {
+        msgtype: "m.text",
+        format: "org.matrix.custom.html",
+        body: body,
+        formatted_body: htmlBody,
+    };
+}
 
-    /**
-     * Generates the content for a HTML Emote event
-     * @param {string} body the plaintext body of the emote
-     * @param {string} htmlBody the HTML representation of the emote
-     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
-     */
-    makeHtmlEmote: function(body, htmlBody) {
-        return {
-            msgtype: "m.emote",
-            format: "org.matrix.custom.html",
-            body: body,
-            formatted_body: htmlBody,
-        };
-    },
+/**
+ * Generates the content for a HTML Notice event
+ * @param {string} body the plaintext body of the notice
+ * @param {string} htmlBody the HTML representation of the notice
+ * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+ */
+export function makeHtmlNotice(body, htmlBody) {
+    return {
+        msgtype: "m.notice",
+        format: "org.matrix.custom.html",
+        body: body,
+        formatted_body: htmlBody,
+    };
+}
 
-    /**
-     * Generates the content for a Plaintext Message event
-     * @param {string} body the plaintext body of the emote
-     * @returns {{msgtype: string, body: string}}
-     */
-    makeTextMessage: function(body) {
-        return {
-            msgtype: "m.text",
-            body: body,
-        };
-    },
+/**
+ * Generates the content for a HTML Emote event
+ * @param {string} body the plaintext body of the emote
+ * @param {string} htmlBody the HTML representation of the emote
+ * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+ */
+export function makeHtmlEmote(body, htmlBody) {
+    return {
+        msgtype: "m.emote",
+        format: "org.matrix.custom.html",
+        body: body,
+        formatted_body: htmlBody,
+    };
+}
 
-    /**
-     * Generates the content for a Plaintext Notice event
-     * @param {string} body the plaintext body of the notice
-     * @returns {{msgtype: string, body: string}}
-     */
-    makeNotice: function(body) {
-        return {
-            msgtype: "m.notice",
-            body: body,
-        };
-    },
+/**
+ * Generates the content for a Plaintext Message event
+ * @param {string} body the plaintext body of the emote
+ * @returns {{msgtype: string, body: string}}
+ */
+export function makeTextMessage(body) {
+    return {
+        msgtype: "m.text",
+        body: body,
+    };
+}
 
-    /**
-     * Generates the content for a Plaintext Emote event
-     * @param {string} body the plaintext body of the emote
-     * @returns {{msgtype: string, body: string}}
-     */
-    makeEmoteMessage: function(body) {
-        return {
-            msgtype: "m.emote",
-            body: body,
-        };
-    },
-};
+/**
+ * Generates the content for a Plaintext Notice event
+ * @param {string} body the plaintext body of the notice
+ * @returns {{msgtype: string, body: string}}
+ */
+export function makeNotice(body) {
+    return {
+        msgtype: "m.notice",
+        body: body,
+    };
+}
+
+/**
+ * Generates the content for a Plaintext Emote event
+ * @param {string} body the plaintext body of the emote
+ * @returns {{msgtype: string, body: string}}
+ */
+export function makeEmoteMessage(body) {
+    return {
+        msgtype: "m.emote",
+        body: body,
+    };
+}

src/content-repo.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,94 +17,61 @@ limitations under the License.
 /**
  * @module content-repo
  */
-const utils = require("./utils");
 
-/** Content Repo utility functions */
-module.exports = {
-    /**
-     * Get the HTTP URL for an MXC URI.
-     * @param {string} baseUrl The base homeserver url which has a content repo.
-     * @param {string} mxc The mxc:// URI.
-     * @param {Number} width The desired width of the thumbnail.
-     * @param {Number} height The desired height of the thumbnail.
-     * @param {string} resizeMethod The thumbnail resize method to use, either
-     * "crop" or "scale".
-     * @param {Boolean} allowDirectLinks If true, return any non-mxc URLs
-     * directly. Fetching such URLs will leak information about the user to
-     * anyone they share a room with. If false, will return the emptry string
-     * for such URLs.
-     * @return {string} The complete URL to the content.
-     */
-    getHttpUriForMxc: function(baseUrl, mxc, width, height,
-                               resizeMethod, allowDirectLinks) {
-        if (typeof mxc !== "string" || !mxc) {
+import * as utils from "./utils";
+
+/**
+ * Get the HTTP URL for an MXC URI.
+ * @param {string} baseUrl The base homeserver url which has a content repo.
+ * @param {string} mxc The mxc:// URI.
+ * @param {Number} width The desired width of the thumbnail.
+ * @param {Number} height The desired height of the thumbnail.
+ * @param {string} resizeMethod The thumbnail resize method to use, either
+ * "crop" or "scale".
+ * @param {Boolean} allowDirectLinks If true, return any non-mxc URLs
+ * directly. Fetching such URLs will leak information about the user to
+ * anyone they share a room with. If false, will return the emptry string
+ * for such URLs.
+ * @return {string} The complete URL to the content.
+ */
+export function getHttpUriForMxc(baseUrl, mxc, width, height,
+                           resizeMethod, allowDirectLinks) {
+    if (typeof mxc !== "string" || !mxc) {
+        return '';
+    }
+    if (mxc.indexOf("mxc://") !== 0) {
+        if (allowDirectLinks) {
+            return mxc;
+        } else {
             return '';
         }
-        if (mxc.indexOf("mxc://") !== 0) {
-            if (allowDirectLinks) {
-                return mxc;
-            } else {
-                return '';
-            }
-        }
-        let serverAndMediaId = mxc.slice(6); // strips mxc://
-        let prefix = "/_matrix/media/v1/download/";
-        const params = {};
+    }
+    let serverAndMediaId = mxc.slice(6); // strips mxc://
+    let prefix = "/_matrix/media/r0/download/";
+    const params = {};
 
-        if (width) {
-            params.width = width;
-        }
-        if (height) {
-            params.height = height;
-        }
-        if (resizeMethod) {
-            params.method = resizeMethod;
-        }
-        if (utils.keys(params).length > 0) {
-            // these are thumbnailing params so they probably want the
-            // thumbnailing API...
-            prefix = "/_matrix/media/v1/thumbnail/";
-        }
+    if (width) {
+        params.width = Math.round(width);
+    }
+    if (height) {
+        params.height = Math.round(height);
+    }
+    if (resizeMethod) {
+        params.method = resizeMethod;
+    }
+    if (utils.keys(params).length > 0) {
+        // these are thumbnailing params so they probably want the
+        // thumbnailing API...
+        prefix = "/_matrix/media/r0/thumbnail/";
+    }
 
-        const fragmentOffset = serverAndMediaId.indexOf("#");
-        let fragment = "";
-        if (fragmentOffset >= 0) {
-            fragment = serverAndMediaId.substr(fragmentOffset);
-            serverAndMediaId = serverAndMediaId.substr(0, fragmentOffset);
-        }
-        return baseUrl + prefix + serverAndMediaId +
-            (utils.keys(params).length === 0 ? "" :
-            ("?" + utils.encodeParams(params))) + fragment;
-    },
-
-    /**
-     * Get an identicon URL from an arbitrary string.
-     * @param {string} baseUrl The base homeserver url which has a content repo.
-     * @param {string} identiconString The string to create an identicon for.
-     * @param {Number} width The desired width of the image in pixels. Default: 96.
-     * @param {Number} height The desired height of the image in pixels. Default: 96.
-     * @return {string} The complete URL to the identicon.
-     */
-    getIdenticonUri: function(baseUrl, identiconString, width, height) {
-        if (!identiconString) {
-            return null;
-        }
-        if (!width) {
-            width = 96;
-        }
-        if (!height) {
-            height = 96;
-        }
-        const params = {
-            width: width,
-            height: height,
-        };
-
-        const path = utils.encodeUri("/_matrix/media/v1/identicon/$ident", {
-            $ident: identiconString,
-        });
-        return baseUrl + path +
-            (utils.keys(params).length === 0 ? "" :
-                ("?" + utils.encodeParams(params)));
-    },
-};
+    const fragmentOffset = serverAndMediaId.indexOf("#");
+    let fragment = "";
+    if (fragmentOffset >= 0) {
+        fragment = serverAndMediaId.substr(fragmentOffset);
+        serverAndMediaId = serverAndMediaId.substr(0, fragmentOffset);
+    }
+    return baseUrl + prefix + serverAndMediaId +
+        (utils.keys(params).length === 0 ? "" :
+        ("?" + utils.encodeParams(params))) + fragment;
+}

src/crypto/CrossSigning.js

@@ -0,0 +1,816 @@
+/*
+Copyright 2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Cross signing methods
+ * @module crypto/CrossSigning
+ */
+
+import {decodeBase64, encodeBase64, pkSign, pkVerify} from './olmlib';
+import {EventEmitter} from 'events';
+import {logger} from '../logger';
+import {IndexedDBCryptoStore} from '../crypto/store/indexeddb-crypto-store';
+import {decryptAES, encryptAES} from './aes';
+
+const KEY_REQUEST_TIMEOUT_MS = 1000 * 60;
+
+function publicKeyFromKeyInfo(keyInfo) {
+    // `keys` is an object with { [`ed25519:${pubKey}`]: pubKey }
+    // We assume only a single key, and we want the bare form without type
+    // prefix, so we select the values.
+    return Object.values(keyInfo.keys)[0];
+}
+
+export class CrossSigningInfo extends EventEmitter {
+    /**
+     * Information about a user's cross-signing keys
+     *
+     * @class
+     *
+     * @param {string} userId the user that the information is about
+     * @param {object} callbacks Callbacks used to interact with the app
+     *     Requires getCrossSigningKey and saveCrossSigningKeys
+     * @param {object} cacheCallbacks Callbacks used to interact with the cache
+     */
+    constructor(userId, callbacks, cacheCallbacks) {
+        super();
+
+        // you can't change the userId
+        Object.defineProperty(this, 'userId', {
+            enumerable: true,
+            value: userId,
+        });
+        this._callbacks = callbacks || {};
+        this._cacheCallbacks = cacheCallbacks || {};
+        this.keys = {};
+        this.firstUse = true;
+        // This tracks whether we've ever verified this user with any identity.
+        // When you verify a user, any devices online at the time that receive
+        // the verifying signature via the homeserver will latch this to true
+        // and can use it in the future to detect cases where the user has
+        // become unverifed later for any reason.
+        this.crossSigningVerifiedBefore = false;
+    }
+
+    static fromStorage(obj, userId) {
+        const res = new CrossSigningInfo(userId);
+        for (const prop in obj) {
+            if (obj.hasOwnProperty(prop)) {
+                res[prop] = obj[prop];
+            }
+        }
+        return res;
+    }
+
+    toStorage() {
+        return {
+            keys: this.keys,
+            firstUse: this.firstUse,
+            crossSigningVerifiedBefore: this.crossSigningVerifiedBefore,
+        };
+    }
+
+    /**
+     * Calls the app callback to ask for a private key
+     *
+     * @param {string} type The key type ("master", "self_signing", or "user_signing")
+     * @param {string} expectedPubkey The matching public key or undefined to use
+     *     the stored public key for the given key type.
+     * @returns {Array} An array with [ public key, Olm.PkSigning ]
+     */
+    async getCrossSigningKey(type, expectedPubkey) {
+        const shouldCache = ["master", "self_signing", "user_signing"].indexOf(type) >= 0;
+
+        if (!this._callbacks.getCrossSigningKey) {
+            throw new Error("No getCrossSigningKey callback supplied");
+        }
+
+        if (expectedPubkey === undefined) {
+            expectedPubkey = this.getId(type);
+        }
+
+        function validateKey(key) {
+            if (!key) return;
+            const signing = new global.Olm.PkSigning();
+            const gotPubkey = signing.init_with_seed(key);
+            if (gotPubkey === expectedPubkey) {
+                return [gotPubkey, signing];
+            }
+            signing.free();
+        }
+
+        let privkey;
+        if (this._cacheCallbacks.getCrossSigningKeyCache && shouldCache) {
+            privkey = await this._cacheCallbacks
+              .getCrossSigningKeyCache(type, expectedPubkey);
+        }
+
+        const cacheresult = validateKey(privkey);
+        if (cacheresult) {
+            return cacheresult;
+        }
+
+        privkey = await this._callbacks.getCrossSigningKey(type, expectedPubkey);
+        const result = validateKey(privkey);
+        if (result) {
+            if (this._cacheCallbacks.storeCrossSigningKeyCache && shouldCache) {
+                await this._cacheCallbacks.storeCrossSigningKeyCache(type, privkey);
+            }
+            return result;
+        }
+
+        /* No keysource even returned a key */
+        if (!privkey) {
+            throw new Error(
+                "getCrossSigningKey callback for " + type + " returned falsey",
+            );
+        }
+
+        /* We got some keys from the keysource, but none of them were valid */
+        throw new Error(
+            "Key type " + type + " from getCrossSigningKey callback did not match",
+        );
+    }
+
+    /**
+     * Check whether the private keys exist in secret storage.
+     * XXX: This could be static, be we often seem to have an instance when we
+     * want to know this anyway...
+     *
+     * @param {SecretStorage} secretStorage The secret store using account data
+     * @returns {object} map of key name to key info the secret is encrypted
+     *     with, or null if it is not present or not encrypted with a trusted
+     *     key
+     */
+    async isStoredInSecretStorage(secretStorage) {
+        // check what SSSS keys have encrypted the master key (if any)
+        const stored =
+              await secretStorage.isStored("m.cross_signing.master", false) || {};
+        // then check which of those SSSS keys have also encrypted the SSK and USK
+        function intersect(s) {
+            for (const k of Object.keys(stored)) {
+                if (!s[k]) {
+                    delete stored[k];
+                }
+            }
+        }
+        for (const type of ["self_signing", "user_signing"]) {
+            intersect(
+                await secretStorage.isStored(`m.cross_signing.${type}`, false) || {},
+            );
+        }
+        return Object.keys(stored).length ? stored : null;
+    }
+
+    /**
+     * Store private keys in secret storage for use by other devices. This is
+     * typically called in conjunction with the creation of new cross-signing
+     * keys.
+     *
+     * @param {Map} keys The keys to store
+     * @param {SecretStorage} secretStorage The secret store using account data
+     */
+    static async storeInSecretStorage(keys, secretStorage) {
+        for (const [type, privateKey] of keys) {
+            const encodedKey = encodeBase64(privateKey);
+            await secretStorage.store(`m.cross_signing.${type}`, encodedKey);
+        }
+    }
+
+    /**
+     * Get private keys from secret storage created by some other device. This
+     * also passes the private keys to the app-specific callback.
+     *
+     * @param {string} type The type of key to get.  One of "master",
+     * "self_signing", or "user_signing".
+     * @param {SecretStorage} secretStorage The secret store using account data
+     * @return {Uint8Array} The private key
+     */
+    static async getFromSecretStorage(type, secretStorage) {
+        const encodedKey = await secretStorage.get(`m.cross_signing.${type}`);
+        if (!encodedKey) {
+            return null;
+        }
+        return decodeBase64(encodedKey);
+    }
+
+    /**
+     * Check whether the private keys exist in the local key cache.
+     *
+     * @param {string} [type] The type of key to get. One of "master",
+     * "self_signing", or "user_signing". Optional, will check all by default.
+     * @returns {boolean} True if all keys are stored in the local cache.
+     */
+    async isStoredInKeyCache(type) {
+        const cacheCallbacks = this._cacheCallbacks;
+        if (!cacheCallbacks) return false;
+        const types = type ? [type] : ["master", "self_signing", "user_signing"];
+        for (const t of types) {
+            if (!await cacheCallbacks.getCrossSigningKeyCache(t)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Get cross-signing private keys from the local cache.
+     *
+     * @returns {Map} A map from key type (string) to private key (Uint8Array)
+     */
+    async getCrossSigningKeysFromCache() {
+        const keys = new Map();
+        const cacheCallbacks = this._cacheCallbacks;
+        if (!cacheCallbacks) return keys;
+        for (const type of ["master", "self_signing", "user_signing"]) {
+            const privKey = await cacheCallbacks.getCrossSigningKeyCache(type);
+            if (!privKey) {
+                continue;
+            }
+            keys.set(type, privKey);
+        }
+        return keys;
+    }
+
+    /**
+     * Get the ID used to identify the user. This can also be used to test for
+     * the existence of a given key type.
+     *
+     * @param {string} type The type of key to get the ID of.  One of "master",
+     * "self_signing", or "user_signing".  Defaults to "master".
+     *
+     * @return {string} the ID
+     */
+    getId(type) {
+        type = type || "master";
+        if (!this.keys[type]) return null;
+        const keyInfo = this.keys[type];
+        return publicKeyFromKeyInfo(keyInfo);
+    }
+
+    /**
+     * Create new cross-signing keys for the given key types. The public keys
+     * will be held in this class, while the private keys are passed off to the
+     * `saveCrossSigningKeys` application callback.
+     *
+     * @param {CrossSigningLevel} level The key types to reset
+     */
+    async resetKeys(level) {
+        if (!this._callbacks.saveCrossSigningKeys) {
+            throw new Error("No saveCrossSigningKeys callback supplied");
+        }
+
+        // If we're resetting the master key, we reset all keys
+        if (
+            level === undefined ||
+            level & CrossSigningLevel.MASTER ||
+            !this.keys.master
+        ) {
+            level = (
+                CrossSigningLevel.MASTER |
+                CrossSigningLevel.USER_SIGNING |
+                CrossSigningLevel.SELF_SIGNING
+            );
+        } else if (level === 0) {
+            return;
+        }
+
+        const privateKeys = {};
+        const keys = {};
+        let masterSigning;
+        let masterPub;
+
+        try {
+            if (level & CrossSigningLevel.MASTER) {
+                masterSigning = new global.Olm.PkSigning();
+                privateKeys.master = masterSigning.generate_seed();
+                masterPub = masterSigning.init_with_seed(privateKeys.master);
+                keys.master = {
+                    user_id: this.userId,
+                    usage: ['master'],
+                    keys: {
+                        ['ed25519:' + masterPub]: masterPub,
+                    },
+                };
+            } else {
+                [masterPub, masterSigning] = await this.getCrossSigningKey("master");
+            }
+
+            if (level & CrossSigningLevel.SELF_SIGNING) {
+                const sskSigning = new global.Olm.PkSigning();
+                try {
+                    privateKeys.self_signing = sskSigning.generate_seed();
+                    const sskPub = sskSigning.init_with_seed(privateKeys.self_signing);
+                    keys.self_signing = {
+                        user_id: this.userId,
+                        usage: ['self_signing'],
+                        keys: {
+                            ['ed25519:' + sskPub]: sskPub,
+                        },
+                    };
+                    pkSign(keys.self_signing, masterSigning, this.userId, masterPub);
+                } finally {
+                    sskSigning.free();
+                }
+            }
+
+            if (level & CrossSigningLevel.USER_SIGNING) {
+                const uskSigning = new global.Olm.PkSigning();
+                try {
+                    privateKeys.user_signing = uskSigning.generate_seed();
+                    const uskPub = uskSigning.init_with_seed(privateKeys.user_signing);
+                    keys.user_signing = {
+                        user_id: this.userId,
+                        usage: ['user_signing'],
+                        keys: {
+                            ['ed25519:' + uskPub]: uskPub,
+                        },
+                    };
+                    pkSign(keys.user_signing, masterSigning, this.userId, masterPub);
+                } finally {
+                    uskSigning.free();
+                }
+            }
+
+            Object.assign(this.keys, keys);
+            this._callbacks.saveCrossSigningKeys(privateKeys);
+        } finally {
+            if (masterSigning) {
+                masterSigning.free();
+            }
+        }
+    }
+
+    /**
+     * unsets the keys, used when another session has reset the keys, to disable cross-signing
+     */
+    clearKeys() {
+        this.keys = {};
+    }
+
+    setKeys(keys) {
+        const signingKeys = {};
+        if (keys.master) {
+            if (keys.master.user_id !== this.userId) {
+                const error = "Mismatched user ID " + keys.master.user_id +
+                      " in master key from " + this.userId;
+                logger.error(error);
+                throw new Error(error);
+            }
+            if (!this.keys.master) {
+                // this is the first key we've seen, so first-use is true
+                this.firstUse = true;
+            } else if (publicKeyFromKeyInfo(keys.master) !== this.getId()) {
+                // this is a different key, so first-use is false
+                this.firstUse = false;
+            } // otherwise, same key, so no change
+            signingKeys.master = keys.master;
+        } else if (this.keys.master) {
+            signingKeys.master = this.keys.master;
+        } else {
+            throw new Error("Tried to set cross-signing keys without a master key");
+        }
+        const masterKey = publicKeyFromKeyInfo(signingKeys.master);
+
+        // verify signatures
+        if (keys.user_signing) {
+            if (keys.user_signing.user_id !== this.userId) {
+                const error = "Mismatched user ID " + keys.master.user_id +
+                      " in user_signing key from " + this.userId;
+                logger.error(error);
+                throw new Error(error);
+            }
+            try {
+                pkVerify(keys.user_signing, masterKey, this.userId);
+            } catch (e) {
+                logger.error("invalid signature on user-signing key");
+                // FIXME: what do we want to do here?
+                throw e;
+            }
+        }
+        if (keys.self_signing) {
+            if (keys.self_signing.user_id !== this.userId) {
+                const error = "Mismatched user ID " + keys.master.user_id +
+                      " in self_signing key from " + this.userId;
+                logger.error(error);
+                throw new Error(error);
+            }
+            try {
+                pkVerify(keys.self_signing, masterKey, this.userId);
+            } catch (e) {
+                logger.error("invalid signature on self-signing key");
+                // FIXME: what do we want to do here?
+                throw e;
+            }
+        }
+
+        // if everything checks out, then save the keys
+        if (keys.master) {
+            this.keys.master = keys.master;
+            // if the master key is set, then the old self-signing and
+            // user-signing keys are obsolete
+            this.keys.self_signing = null;
+            this.keys.user_signing = null;
+        }
+        if (keys.self_signing) {
+            this.keys.self_signing = keys.self_signing;
+        }
+        if (keys.user_signing) {
+            this.keys.user_signing = keys.user_signing;
+        }
+    }
+
+    updateCrossSigningVerifiedBefore(isCrossSigningVerified) {
+        // It is critical that this value latches forward from false to true but
+        // never back to false to avoid a downgrade attack.
+        if (!this.crossSigningVerifiedBefore && isCrossSigningVerified) {
+            this.crossSigningVerifiedBefore = true;
+        }
+    }
+
+    async signObject(data, type) {
+        if (!this.keys[type]) {
+            throw new Error(
+                "Attempted to sign with " + type + " key but no such key present",
+            );
+        }
+        const [pubkey, signing] = await this.getCrossSigningKey(type);
+        try {
+            pkSign(data, signing, this.userId, pubkey);
+            return data;
+        } finally {
+            signing.free();
+        }
+    }
+
+    async signUser(key) {
+        if (!this.keys.user_signing) {
+            logger.info("No user signing key: not signing user");
+            return;
+        }
+        return this.signObject(key.keys.master, "user_signing");
+    }
+
+    async signDevice(userId, device) {
+        if (userId !== this.userId) {
+            throw new Error(
+                `Trying to sign ${userId}'s device; can only sign our own device`,
+            );
+        }
+        if (!this.keys.self_signing) {
+            logger.info("No self signing key: not signing device");
+            return;
+        }
+        return this.signObject(
+            {
+                algorithms: device.algorithms,
+                keys: device.keys,
+                device_id: device.deviceId,
+                user_id: userId,
+            }, "self_signing",
+        );
+    }
+
+    /**
+     * Check whether a given user is trusted.
+     *
+     * @param {CrossSigningInfo} userCrossSigning Cross signing info for user
+     *
+     * @returns {UserTrustLevel}
+     */
+    checkUserTrust(userCrossSigning) {
+        // if we're checking our own key, then it's trusted if the master key
+        // and self-signing key match
+        if (this.userId === userCrossSigning.userId
+            && this.getId() && this.getId() === userCrossSigning.getId()
+            && this.getId("self_signing")
+            && this.getId("self_signing") === userCrossSigning.getId("self_signing")
+        ) {
+            return new UserTrustLevel(true, true, this.firstUse);
+        }
+
+        if (!this.keys.user_signing) {
+            // If there's no user signing key, they can't possibly be verified.
+            // They may be TOFU trusted though.
+            return new UserTrustLevel(false, false, userCrossSigning.firstUse);
+        }
+
+        let userTrusted;
+        const userMaster = userCrossSigning.keys.master;
+        const uskId = this.getId('user_signing');
+        try {
+            pkVerify(userMaster, uskId, this.userId);
+            userTrusted = true;
+        } catch (e) {
+            userTrusted = false;
+        }
+        return new UserTrustLevel(
+            userTrusted,
+            userCrossSigning.crossSigningVerifiedBefore,
+            userCrossSigning.firstUse,
+        );
+    }
+
+    /**
+     * Check whether a given device is trusted.
+     *
+     * @param {CrossSigningInfo} userCrossSigning Cross signing info for user
+     * @param {module:crypto/deviceinfo} device The device to check
+     * @param {bool} localTrust Whether the device is trusted locally
+     * @param {bool} trustCrossSignedDevices Whether we trust cross signed devices
+     *
+     * @returns {DeviceTrustLevel}
+     */
+    checkDeviceTrust(userCrossSigning, device, localTrust, trustCrossSignedDevices) {
+        const userTrust = this.checkUserTrust(userCrossSigning);
+
+        const userSSK = userCrossSigning.keys.self_signing;
+        if (!userSSK) {
+            // if the user has no self-signing key then we cannot make any
+            // trust assertions about this device from cross-signing
+            return new DeviceTrustLevel(
+                false, false, localTrust, trustCrossSignedDevices,
+            );
+        }
+
+        const deviceObj = deviceToObject(device, userCrossSigning.userId);
+        try {
+            // if we can verify the user's SSK from their master key...
+            pkVerify(userSSK, userCrossSigning.getId(), userCrossSigning.userId);
+            // ...and this device's key from their SSK...
+            pkVerify(
+                deviceObj, publicKeyFromKeyInfo(userSSK), userCrossSigning.userId,
+            );
+            // ...then we trust this device as much as far as we trust the user
+            return DeviceTrustLevel.fromUserTrustLevel(
+                userTrust, localTrust, trustCrossSignedDevices,
+            );
+        } catch (e) {
+            return new DeviceTrustLevel(
+                false, false, localTrust, trustCrossSignedDevices,
+            );
+        }
+    }
+
+    /**
+     * @returns {object} Cache callbacks
+     */
+    getCacheCallbacks() {
+        return this._cacheCallbacks;
+    }
+}
+
+function deviceToObject(device, userId) {
+    return {
+        algorithms: device.algorithms,
+        keys: device.keys,
+        device_id: device.deviceId,
+        user_id: userId,
+        signatures: device.signatures,
+    };
+}
+
+export const CrossSigningLevel = {
+    MASTER: 4,
+    USER_SIGNING: 2,
+    SELF_SIGNING: 1,
+};
+
+/**
+ * Represents the ways in which we trust a user
+ */
+export class UserTrustLevel {
+    constructor(crossSigningVerified, crossSigningVerifiedBefore, tofu) {
+        this._crossSigningVerified = crossSigningVerified;
+        this._crossSigningVerifiedBefore = crossSigningVerifiedBefore;
+        this._tofu = tofu;
+    }
+
+    /**
+     * @returns {bool} true if this user is verified via any means
+     */
+    isVerified() {
+        return this.isCrossSigningVerified();
+    }
+
+    /**
+     * @returns {bool} true if this user is verified via cross signing
+     */
+    isCrossSigningVerified() {
+        return this._crossSigningVerified;
+    }
+
+    /**
+     * @returns {bool} true if we ever verified this user before (at least for
+     * the history of verifications observed by this device).
+     */
+    wasCrossSigningVerified() {
+        return this._crossSigningVerifiedBefore;
+    }
+
+    /**
+     * @returns {bool} true if this user's key is trusted on first use
+     */
+    isTofu() {
+        return this._tofu;
+    }
+}
+
+/**
+ * Represents the ways in which we trust a device
+ */
+export class DeviceTrustLevel {
+    constructor(crossSigningVerified, tofu, localVerified, trustCrossSignedDevices) {
+        this._crossSigningVerified = crossSigningVerified;
+        this._tofu = tofu;
+        this._localVerified = localVerified;
+        this._trustCrossSignedDevices = trustCrossSignedDevices;
+    }
+
+    static fromUserTrustLevel(userTrustLevel, localVerified, trustCrossSignedDevices) {
+        return new DeviceTrustLevel(
+            userTrustLevel._crossSigningVerified,
+            userTrustLevel._tofu,
+            localVerified,
+            trustCrossSignedDevices,
+        );
+    }
+
+    /**
+     * @returns {bool} true if this device is verified via any means
+     */
+    isVerified() {
+        return Boolean(this.isLocallyVerified() || (
+            this._trustCrossSignedDevices && this.isCrossSigningVerified()
+        ));
+    }
+
+    /**
+     * @returns {bool} true if this device is verified via cross signing
+     */
+    isCrossSigningVerified() {
+        return this._crossSigningVerified;
+    }
+
+    /**
+     * @returns {bool} true if this device is verified locally
+     */
+    isLocallyVerified() {
+        return this._localVerified;
+    }
+
+    /**
+     * @returns {bool} true if this device is trusted from a user's key
+     * that is trusted on first use
+     */
+    isTofu() {
+        return this._tofu;
+    }
+}
+
+export function createCryptoStoreCacheCallbacks(store, olmdevice) {
+    return {
+        getCrossSigningKeyCache: async function(type, _expectedPublicKey) {
+            const key = await new Promise((resolve) => {
+                return store.doTxn(
+                    'readonly',
+                    [IndexedDBCryptoStore.STORE_ACCOUNT],
+                    (txn) => {
+                        store.getSecretStorePrivateKey(txn, resolve, type);
+                    },
+                );
+            });
+
+            if (key && key.ciphertext) {
+                const pickleKey = Buffer.from(olmdevice._pickleKey);
+                const decrypted = await decryptAES(key, pickleKey, type);
+                return decodeBase64(decrypted);
+            } else {
+                return key;
+            }
+        },
+        storeCrossSigningKeyCache: async function(type, key) {
+            if (!(key instanceof Uint8Array)) {
+                throw new Error(
+                    `storeCrossSigningKeyCache expects Uint8Array, got ${key}`,
+                );
+            }
+            const pickleKey = Buffer.from(olmdevice._pickleKey);
+            key = await encryptAES(encodeBase64(key), pickleKey, type);
+            return store.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    store.storeSecretStorePrivateKey(txn, type, key);
+                },
+            );
+        },
+    };
+}
+
+/**
+ * Request cross-signing keys from another device during verification.
+ *
+ * @param {module:base-apis~MatrixBaseApis} baseApis base Matrix API interface
+ * @param {string} userId The user ID being verified
+ * @param {string} deviceId The device ID being verified
+ */
+export async function requestKeysDuringVerification(baseApis, userId, deviceId) {
+    // If this is a self-verification, ask the other party for keys
+    if (baseApis.getUserId() !== userId) {
+        return;
+    }
+    logger.log("Cross-signing: Self-verification done; requesting keys");
+    // This happens asynchronously, and we're not concerned about waiting for
+    // it.  We return here in order to test.
+    return new Promise((resolve, reject) => {
+        const client = baseApis;
+        const original = client._crypto._crossSigningInfo;
+
+        // We already have all of the infrastructure we need to validate and
+        // cache cross-signing keys, so instead of replicating that, here we set
+        // up callbacks that request them from the other device and call
+        // CrossSigningInfo.getCrossSigningKey() to validate/cache
+        const crossSigning = new CrossSigningInfo(
+            original.userId,
+            { getCrossSigningKey: async (type) => {
+                logger.debug("Cross-signing: requesting secret",
+                                type, deviceId);
+                const { promise } = client.requestSecret(
+                    `m.cross_signing.${type}`, [deviceId],
+                );
+                const result = await promise;
+                const decoded = decodeBase64(result);
+                return Uint8Array.from(decoded);
+            } },
+            original._cacheCallbacks,
+        );
+        crossSigning.keys = original.keys;
+
+        // XXX: get all keys out if we get one key out
+        // https://github.com/vector-im/element-web/issues/12604
+        // then change here to reject on the timeout
+        // Requests can be ignored, so don't wait around forever
+        const timeout = new Promise((resolve, reject) => {
+            setTimeout(
+                resolve,
+                KEY_REQUEST_TIMEOUT_MS,
+                new Error("Timeout"),
+            );
+        });
+
+        // also request and cache the key backup key
+        const backupKeyPromise = new Promise(async resolve => {
+            const cachedKey = await client._crypto.getSessionBackupPrivateKey();
+            if (!cachedKey) {
+                logger.info("No cached backup key found. Requesting...");
+                const secretReq = client.requestSecret(
+                    'm.megolm_backup.v1', [deviceId],
+                );
+                const base64Key = await secretReq.promise;
+                logger.info("Got key backup key, decoding...");
+                const decodedKey = decodeBase64(base64Key);
+                logger.info("Decoded backup key, storing...");
+                client._crypto.storeSessionBackupPrivateKey(
+                    Uint8Array.from(decodedKey),
+                );
+                logger.info("Backup key stored. Starting backup restore...");
+                const backupInfo = await client.getKeyBackupVersion();
+                // no need to await for this - just let it go in the bg
+                client.restoreKeyBackupWithCache(
+                    undefined, undefined, backupInfo,
+                ).then(() => {
+                    logger.info("Backup restored.");
+                });
+            }
+            resolve();
+        });
+
+        // We call getCrossSigningKey() for its side-effects
+        return Promise.race([
+            Promise.all([
+                crossSigning.getCrossSigningKey("master"),
+                crossSigning.getCrossSigningKey("self_signing"),
+                crossSigning.getCrossSigningKey("user_signing"),
+                backupKeyPromise,
+            ]),
+            timeout,
+        ]).then(resolve, reject);
+    }).catch((e) => {
+        logger.warn("Cross-signing: failure while requesting keys:", e);
+    });
+}

src/crypto/DeviceList.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2017 Vector Creations Ltd
-Copyright 2018 New Vector Ltd
+Copyright 2018, 2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,7 +15,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
 
 /**
  * @module crypto/DeviceList
@@ -22,12 +22,13 @@ limitations under the License.
  * Manages the list of other users' devices
  */
 
-import Promise from 'bluebird';
-
-import logger from '../logger';
-import DeviceInfo from './deviceinfo';
-import olmlib from './olmlib';
-import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+import {EventEmitter} from 'events';
+import {logger} from '../logger';
+import {DeviceInfo} from './deviceinfo';
+import {CrossSigningInfo} from './CrossSigning';
+import * as olmlib from './olmlib';
+import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
+import {defer, sleep} from '../utils';
 
 
 /* State transition diagram for DeviceList._deviceTrackingStatus
@@ -60,10 +61,11 @@ const TRACKING_STATUS_UP_TO_DATE = 3;
 /**
  * @alias module:crypto/DeviceList
  */
-export default class DeviceList {
-    constructor(baseApis, cryptoStore, sessionStore, olmDevice) {
+export class DeviceList extends EventEmitter {
+    constructor(baseApis, cryptoStore, olmDevice) {
+        super();
+
         this._cryptoStore = cryptoStore;
-        this._sessionStore = sessionStore;
 
         // userId -> {
         //     deviceId -> {
@@ -72,6 +74,11 @@ export default class DeviceList {
         // }
         this._devices = {};
 
+        // userId -> {
+        //     [key info]
+        // }
+        this._crossSigningInfo = {};
+
         // map of identity keys to the user who owns it
         this._userByIdentityKey = {};
 
@@ -102,36 +109,25 @@ export default class DeviceList {
         this._savePromiseTime = null;
         // The timer used to delay the save
         this._saveTimer = null;
+        // True if we have fetched data from the server or loaded a non-empty
+        // set of device data from the store
+        this._hasFetched = null;
     }
 
     /**
      * Load the device tracking state from storage
      */
     async load() {
-        let shouldDeleteSessionStore = false;
         await this._cryptoStore.doTxn(
-            // migrate from session store if there's data there and not here
-            'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
+            'readonly', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
                 this._cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
-                    if (deviceData === null) {
-                        logger.log("Migrating e2e device data...");
-                        this._devices = this._sessionStore.getAllEndToEndDevices() || {};
-                        this._deviceTrackingStatus = (
-                            this._sessionStore.getEndToEndDeviceTrackingStatus() || {}
-                        );
-                        this._syncToken = this._sessionStore.getEndToEndDeviceSyncToken();
-                        this._cryptoStore.storeEndToEndDeviceData({
-                            devices: this._devices,
-                            trackingStatus: this._deviceTrackingStatus,
-                            syncToken: this._syncToken,
-                        }, txn);
-                        shouldDeleteSessionStore = true;
-                    } else {
-                        this._devices = deviceData ? deviceData.devices : {},
-                        this._deviceTrackingStatus = deviceData ?
-                            deviceData.trackingStatus : {};
-                        this._syncToken = deviceData ? deviceData.syncToken : null;
-                    }
+                    this._hasFetched = Boolean(deviceData && deviceData.devices);
+                    this._devices = deviceData ? deviceData.devices : {},
+                    this._crossSigningInfo = deviceData ?
+                        deviceData.crossSigningInfo || {} : {};
+                    this._deviceTrackingStatus = deviceData ?
+                        deviceData.trackingStatus : {};
+                    this._syncToken = deviceData ? deviceData.syncToken : null;
                     this._userByIdentityKey = {};
                     for (const user of Object.keys(this._devices)) {
                         const userDevices = this._devices[user];
@@ -146,11 +142,6 @@ export default class DeviceList {
             },
         );
 
-        if (shouldDeleteSessionStore) {
-            // migrated data is now safely persisted: remove from old store
-            this._sessionStore.removeEndToEndDeviceData();
-        }
-
         for (const u of Object.keys(this._deviceTrackingStatus)) {
             // if a download was in progress when we got shut down, it isn't any more.
             if (this._deviceTrackingStatus[u] == TRACKING_STATUS_DOWNLOAD_IN_PROGRESS) {
@@ -210,26 +201,33 @@ export default class DeviceList {
             const resolveSavePromise = this._resolveSavePromise;
             this._savePromiseTime = targetTime;
             this._saveTimer = setTimeout(() => {
-                logger.log('Saving device tracking data at token ' + this._syncToken);
+                logger.log('Saving device tracking data', this._syncToken);
+
                 // null out savePromise now (after the delay but before the write),
                 // otherwise we could return the existing promise when the save has
-                // actually already happened. Likewise for the dirty flag.
+                // actually already happened.
                 this._savePromiseTime = null;
                 this._saveTimer = null;
                 this._savePromise = null;
                 this._resolveSavePromise = null;
 
-                this._dirty = false;
                 this._cryptoStore.doTxn(
                     'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
                         this._cryptoStore.storeEndToEndDeviceData({
                             devices: this._devices,
+                            crossSigningInfo: this._crossSigningInfo,
                             trackingStatus: this._deviceTrackingStatus,
                             syncToken: this._syncToken,
                         }, txn);
                     },
                 ).then(() => {
+                    // The device list is considered dirty until the write
+                    // completes.
+                    this._dirty = false;
                     resolveSavePromise();
+                }, err => {
+                    logger.error('Failed to save device tracking data', this._syncToken);
+                    logger.error(err);
                 });
             }, delay);
         }
@@ -323,6 +321,15 @@ export default class DeviceList {
         return stored;
     }
 
+    /**
+     * Returns a list of all user IDs the DeviceList knows about
+     *
+     * @return {array} All known user IDs
+     */
+    getKnownUserIds() {
+        return Object.keys(this._devices);
+    }
+
     /**
      * Get the stored device keys for a user id
      *
@@ -357,6 +364,17 @@ export default class DeviceList {
         return this._devices[userId];
     }
 
+    getStoredCrossSigningForUser(userId) {
+        if (!this._crossSigningInfo[userId]) return null;
+
+        return CrossSigningInfo.fromStorage(this._crossSigningInfo[userId], userId);
+    }
+
+    storeCrossSigningForUser(userId, info) {
+        this._crossSigningInfo[userId] = info;
+        this._dirty = true;
+    }
+
     /**
      * Get the stored keys for a single device
      *
@@ -374,6 +392,26 @@ export default class DeviceList {
         return DeviceInfo.fromStorage(devs[deviceId], deviceId);
     }
 
+    /**
+     * Get a user ID by one of their device's curve25519 identity key
+     *
+     * @param {string} algorithm  encryption algorithm
+     * @param {string} senderKey  curve25519 key to match
+     *
+     * @return {string} user ID
+     */
+    getUserByIdentityKey(algorithm, senderKey) {
+        if (
+            algorithm !== olmlib.OLM_ALGORITHM &&
+            algorithm !== olmlib.MEGOLM_ALGORITHM
+        ) {
+            // we only deal in olm keys
+            return null;
+        }
+
+        return this._userByIdentityKey[senderKey];
+    }
+
     /**
      * Find a device by curve25519 identity key
      *
@@ -383,19 +421,11 @@ export default class DeviceList {
      * @return {module:crypto/deviceinfo?}
      */
     getDeviceByIdentityKey(algorithm, senderKey) {
-        const userId = this._userByIdentityKey[senderKey];
+        const userId = this.getUserByIdentityKey(algorithm, senderKey);
         if (!userId) {
             return null;
         }
 
-        if (
-            algorithm !== olmlib.OLM_ALGORITHM &&
-            algorithm !== olmlib.MEGOLM_ALGORITHM
-        ) {
-            // we only deal in olm keys
-            return null;
-        }
-
         const devices = this._devices[userId];
         if (!devices) {
             return null;
@@ -475,10 +505,10 @@ export default class DeviceList {
         if (!this._deviceTrackingStatus[userId]) {
             logger.log('Now tracking device list for ' + userId);
             this._deviceTrackingStatus[userId] = TRACKING_STATUS_PENDING_DOWNLOAD;
+            // we don't yet persist the tracking status, since there may be a lot
+            // of calls; we save all data together once the sync is done
+            this._dirty = true;
         }
-        // we don't yet persist the tracking status, since there may be a lot
-        // of calls; we save all data together once the sync is done
-        this._dirty = true;
     }
 
     /**
@@ -584,6 +614,10 @@ export default class DeviceList {
         }
     }
 
+    setRawStoredCrossSigningForUser(userId, info) {
+        this._crossSigningInfo[userId] = info;
+    }
+
     /**
      * Fire off download update requests for the given users, and update the
      * device list tracking status for them, and the
@@ -591,7 +625,7 @@ export default class DeviceList {
      *
      * @param {String[]} users  list of userIds
      *
-     * @return {module:client.Promise} resolves when all the users listed have
+     * @return {Promise} resolves when all the users listed have
      *     been updated. rejects if there was a problem updating any of the
      *     users.
      */
@@ -622,6 +656,7 @@ export default class DeviceList {
         });
 
         const finished = (success) => {
+            this.emit("crypto.willUpdateDevices", users, !this._hasFetched);
             users.forEach((u) => {
                 this._dirty = true;
 
@@ -647,6 +682,8 @@ export default class DeviceList {
                 }
             });
             this.saveIfDirty();
+            this.emit("crypto.devicesUpdated", users, !this._hasFetched);
+            this._hasFetched = true;
         };
 
         return prom;
@@ -695,7 +732,7 @@ class DeviceListUpdateSerialiser {
      * @param {String} syncToken sync token to pass in the query request, to
      *     help the HS give the most recent results
      *
-     * @return {module:client.Promise} resolves when all the users listed have
+     * @return {Promise} resolves when all the users listed have
      *     been updated. rejects if there was a problem updating any of the
      *     users.
      */
@@ -705,7 +742,7 @@ class DeviceListUpdateSerialiser {
         });
 
         if (!this._queuedQueryDeferred) {
-            this._queuedQueryDeferred = Promise.defer();
+            this._queuedQueryDeferred = defer();
         }
 
         // We always take the new sync token and just use the latest one we've
@@ -745,23 +782,35 @@ class DeviceListUpdateSerialiser {
 
         this._baseApis.downloadKeysForUsers(
             downloadUsers, opts,
-        ).then((res) => {
+        ).then(async (res) => {
             const dk = res.device_keys || {};
+            const masterKeys = res.master_keys || {};
+            const ssks = res.self_signing_keys || {};
+            const usks = res.user_signing_keys || {};
 
-            // do each user in a separate promise, to avoid wedging the CPU
-            // (https://github.com/vector-im/riot-web/issues/3158)
+            // yield to other things that want to execute in between users, to
+            // avoid wedging the CPU
+            // (https://github.com/vector-im/element-web/issues/3158)
             //
             // of course we ought to do this in a web worker or similar, but
             // this serves as an easy solution for now.
-            let prom = Promise.resolve();
             for (const userId of downloadUsers) {
-                prom = prom.delay(5).then(() => {
-                    return this._processQueryResponseForUser(userId, dk[userId]);
-                });
+                await sleep(5);
+                try {
+                    await this._processQueryResponseForUser(
+                        userId, dk[userId], {
+                            master: masterKeys[userId],
+                            self_signing: ssks[userId],
+                            user_signing: usks[userId],
+                        },
+                    );
+                } catch (e) {
+                    // log the error but continue, so that one bad key
+                    // doesn't kill the whole process
+                    logger.error(`Error processing keys for ${userId}:`, e);
+                }
             }
-
-            return prom;
-        }).done(() => {
+        }).then(() => {
             logger.log('Completed key download for ' + downloadUsers);
 
             this._downloadInProgress = false;
@@ -780,36 +829,66 @@ class DeviceListUpdateSerialiser {
         return deferred.promise;
     }
 
-    async _processQueryResponseForUser(userId, response) {
-        logger.log('got keys for ' + userId + ':', response);
+    async _processQueryResponseForUser(
+        userId, dkResponse, crossSigningResponse,
+    ) {
+        logger.log('got device keys for ' + userId + ':', dkResponse);
+        logger.log('got cross-signing keys for ' + userId + ':', crossSigningResponse);
 
-        // map from deviceid -> deviceinfo for this user
-        const userStore = {};
-        const devs = this._deviceList.getRawStoredDevicesForUser(userId);
-        if (devs) {
-            Object.keys(devs).forEach((deviceId) => {
-                const d = DeviceInfo.fromStorage(devs[deviceId], deviceId);
-                userStore[deviceId] = d;
+        {
+            // map from deviceid -> deviceinfo for this user
+            const userStore = {};
+            const devs = this._deviceList.getRawStoredDevicesForUser(userId);
+            if (devs) {
+                Object.keys(devs).forEach((deviceId) => {
+                    const d = DeviceInfo.fromStorage(devs[deviceId], deviceId);
+                    userStore[deviceId] = d;
+                });
+            }
+
+            await _updateStoredDeviceKeysForUser(
+                this._olmDevice, userId, userStore, dkResponse || {},
+                this._baseApis.getUserId(), this._baseApis.deviceId,
+            );
+
+            // put the updates into the object that will be returned as our results
+            const storage = {};
+            Object.keys(userStore).forEach((deviceId) => {
+                storage[deviceId] = userStore[deviceId].toStorage();
             });
+
+            this._deviceList._setRawStoredDevicesForUser(userId, storage);
         }
 
-        await _updateStoredDeviceKeysForUser(
-            this._olmDevice, userId, userStore, response || {},
-        );
+        // now do the same for the cross-signing keys
+        {
+            // FIXME: should we be ignoring empty cross-signing responses, or
+            // should we be dropping the keys?
+            if (crossSigningResponse
+                && (crossSigningResponse.master || crossSigningResponse.self_signing
+                    || crossSigningResponse.user_signing)) {
+                const crossSigning
+                      = this._deviceList.getStoredCrossSigningForUser(userId)
+                      || new CrossSigningInfo(userId);
 
-        // put the updates into thr object that will be returned as our results
-        const storage = {};
-        Object.keys(userStore).forEach((deviceId) => {
-            storage[deviceId] = userStore[deviceId].toStorage();
-        });
+                crossSigning.setKeys(crossSigningResponse);
 
-        this._deviceList._setRawStoredDevicesForUser(userId, storage);
+                this._deviceList.setRawStoredCrossSigningForUser(
+                    userId, crossSigning.toStorage(),
+                );
+
+                // NB. Unlike most events in the js-sdk, this one is internal to the
+                // js-sdk and is not re-emitted
+                this._deviceList.emit('userCrossSigningUpdated', userId);
+            }
+        }
     }
 }
 
 
-async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
-        userResult) {
+async function _updateStoredDeviceKeysForUser(
+    _olmDevice, userId, userStore, userResult, localUserId, localDeviceId,
+) {
     let updated = false;
 
     // remove any devices in the store which aren't in the response
@@ -819,6 +898,13 @@ async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
         }
 
         if (!(deviceId in userResult)) {
+            if (userId === localUserId && deviceId === localDeviceId) {
+                logger.warn(
+                    `Local device ${deviceId} missing from sync, skipping removal`,
+                );
+                continue;
+            }
+
             logger.log("Device " + userId + ":" + deviceId +
                 " has been removed");
             delete userStore[deviceId];
@@ -877,6 +963,7 @@ async function _storeDeviceKeys(_olmDevice, userStore, deviceResult) {
     }
 
     const unsigned = deviceResult.unsigned || {};
+    const signatures = deviceResult.signatures || {};
 
     try {
         await olmlib.verifySignature(_olmDevice, deviceResult, userId, deviceId, signKey);
@@ -909,5 +996,6 @@ async function _storeDeviceKeys(_olmDevice, userStore, deviceResult) {
     deviceStore.keys = deviceResult.keys || {};
     deviceStore.algorithms = deviceResult.algorithms || [];
     deviceStore.unsigned = unsigned;
+    deviceStore.signatures = signatures;
     return true;
 }

src/crypto/EncryptionSetup.js

@@ -0,0 +1,357 @@
+import { logger } from "../logger";
+import {MatrixEvent} from "../models/event";
+import {EventEmitter} from "events";
+import {createCryptoStoreCacheCallbacks} from "./CrossSigning";
+import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
+import {
+    PREFIX_UNSTABLE,
+} from "../http-api";
+
+/**
+ * Builds an EncryptionSetupOperation by calling any of the add.. methods.
+ * Once done, `buildOperation()` can be called which allows to apply to operation.
+ *
+ * This is used as a helper by Crypto to keep track of all the network requests
+ * and other side-effects of bootstrapping, so it can be applied in one go (and retried in the future)
+ * Also keeps track of all the private keys created during bootstrapping, so we don't need to prompt for them
+ * more than once.
+ */
+export class EncryptionSetupBuilder {
+    /**
+     * @param {Object.<String, MatrixEvent>} accountData pre-existing account data, will only be read, not written.
+     * @param {CryptoCallbacks} delegateCryptoCallbacks crypto callbacks to delegate to if the key isn't in cache yet
+     */
+    constructor(accountData, delegateCryptoCallbacks) {
+        this.accountDataClientAdapter = new AccountDataClientAdapter(accountData);
+        this.crossSigningCallbacks = new CrossSigningCallbacks();
+        this.ssssCryptoCallbacks = new SSSSCryptoCallbacks(delegateCryptoCallbacks);
+
+        this._crossSigningKeys = null;
+        this._keySignatures = null;
+        this._keyBackupInfo = null;
+    }
+
+    /**
+     * Adds new cross-signing public keys
+     *
+     * @param {function} authUpload Function called to await an interactive auth
+     * flow when uploading device signing keys.
+     * Args:
+     *     {function} A function that makes the request requiring auth. Receives
+     *     the auth data as an object. Can be called multiple times, first with
+     *     an empty authDict, to obtain the flows.
+     * @param {Object} keys the new keys
+     */
+    addCrossSigningKeys(authUpload, keys) {
+        this._crossSigningKeys = {authUpload, keys};
+    }
+
+    /**
+     * Adds the key backup info to be updated on the server
+     *
+     * Used either to create a new key backup, or add signatures
+     * from the new MSK.
+     *
+     * @param {Object} keyBackupInfo as received from/sent to the server
+     */
+    addSessionBackup(keyBackupInfo) {
+        this._keyBackupInfo = keyBackupInfo;
+    }
+
+    /**
+     * Adds the session backup private key to be updated in the local cache
+     *
+     * Used after fixing the format of the key
+     *
+     * @param {Uint8Array} privateKey
+     */
+    addSessionBackupPrivateKeyToCache(privateKey) {
+        this._sessionBackupPrivateKey = privateKey;
+    }
+
+    /**
+     * Add signatures from a given user and device/x-sign key
+     * Used to sign the new cross-signing key with the device key
+     *
+     * @param {String} userId
+     * @param {String} deviceId
+     * @param {String} signature
+     */
+    addKeySignature(userId, deviceId, signature) {
+        if (!this._keySignatures) {
+            this._keySignatures = {};
+        }
+        const userSignatures = this._keySignatures[userId] || {};
+        this._keySignatures[userId] = userSignatures;
+        userSignatures[deviceId] = signature;
+    }
+
+
+    /**
+     * @param {String} type
+     * @param {Object} content
+     * @return {Promise}
+     */
+    setAccountData(type, content) {
+        return this.accountDataClientAdapter.setAccountData(type, content);
+    }
+
+    /**
+     * builds the operation containing all the parts that have been added to the builder
+     * @return {EncryptionSetupOperation}
+     */
+    buildOperation() {
+        const accountData = this.accountDataClientAdapter._values;
+        return new EncryptionSetupOperation(
+            accountData,
+            this._crossSigningKeys,
+            this._keyBackupInfo,
+            this._keySignatures,
+        );
+    }
+
+    /**
+     * Stores the created keys locally.
+     *
+     * This does not yet store the operation in a way that it can be restored,
+     * but that is the idea in the future.
+     *
+     * @param  {Crypto} crypto
+     * @return {Promise}
+     */
+    async persist(crypto) {
+        // store private keys in cache
+        if (this._crossSigningKeys) {
+            const cacheCallbacks = createCryptoStoreCacheCallbacks(
+                crypto._cryptoStore, crypto._olmDevice);
+            for (const type of ["master", "self_signing", "user_signing"]) {
+                logger.log(`Cache ${type} cross-signing private key locally`);
+                const privateKey = this.crossSigningCallbacks.privateKeys.get(type);
+                await cacheCallbacks.storeCrossSigningKeyCache(type, privateKey);
+            }
+            // store own cross-sign pubkeys as trusted
+            await crypto._cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    crypto._cryptoStore.storeCrossSigningKeys(
+                        txn, this._crossSigningKeys.keys);
+                },
+            );
+        }
+        // store session backup key in cache
+        if (this._sessionBackupPrivateKey) {
+            await crypto.storeSessionBackupPrivateKey(this._sessionBackupPrivateKey);
+        }
+    }
+}
+
+/**
+ * Can be created from EncryptionSetupBuilder, or
+ * (in a follow-up PR, not implemented yet) restored from storage, to retry.
+ *
+ * It does not have knowledge of any private keys, unlike the builder.
+ */
+export class EncryptionSetupOperation {
+    /**
+     * @param  {Map<String, Object>} accountData
+     * @param  {Object} crossSigningKeys
+     * @param  {Object} keyBackupInfo
+     * @param  {Object} keySignatures
+     */
+    constructor(accountData, crossSigningKeys, keyBackupInfo, keySignatures) {
+        this._accountData = accountData;
+        this._crossSigningKeys = crossSigningKeys;
+        this._keyBackupInfo = keyBackupInfo;
+        this._keySignatures = keySignatures;
+    }
+
+    /**
+     * Runs the (remaining part of, in the future) operation by sending requests to the server.
+     * @param  {Crypto} crypto
+     */
+    async apply(crypto) {
+        const baseApis = crypto._baseApis;
+        // upload cross-signing keys
+        if (this._crossSigningKeys) {
+            const keys = {};
+            for (const [name, key] of Object.entries(this._crossSigningKeys.keys)) {
+                keys[name + "_key"] = key;
+            }
+
+            // We must only call `uploadDeviceSigningKeys` from inside this auth
+            // helper to ensure we properly handle auth errors.
+            await this._crossSigningKeys.authUpload(authDict => {
+                return baseApis.uploadDeviceSigningKeys(authDict, keys);
+            });
+
+            // pass the new keys to the main instance of our own CrossSigningInfo.
+            crypto._crossSigningInfo.setKeys(this._crossSigningKeys.keys);
+        }
+        // set account data
+        if (this._accountData) {
+            for (const [type, content] of this._accountData) {
+                await baseApis.setAccountData(type, content);
+            }
+        }
+        // upload first cross-signing signatures with the new key
+        // (e.g. signing our own device)
+        if (this._keySignatures) {
+            await baseApis.uploadKeySignatures(this._keySignatures);
+        }
+        // need to create/update key backup info
+        if (this._keyBackupInfo) {
+            if (this._keyBackupInfo.version) {
+                // session backup signature
+                // The backup is trusted because the user provided the private key.
+                // Sign the backup with the cross signing key so the key backup can
+                // be trusted via cross-signing.
+                await baseApis._http.authedRequest(
+                    undefined, "PUT", "/room_keys/version/" + this._keyBackupInfo.version,
+                    undefined, {
+                        algorithm: this._keyBackupInfo.algorithm,
+                        auth_data: this._keyBackupInfo.auth_data,
+                    },
+                    {prefix: PREFIX_UNSTABLE},
+                );
+            } else {
+                // add new key backup
+                await baseApis._http.authedRequest(
+                    undefined, "POST", "/room_keys/version",
+                    undefined, this._keyBackupInfo,
+                    {prefix: PREFIX_UNSTABLE},
+                );
+            }
+        }
+    }
+}
+
+
+/**
+ * Catches account data set by SecretStorage during bootstrapping by
+ * implementing the methods related to account data in MatrixClient
+ */
+class AccountDataClientAdapter extends EventEmitter {
+    /**
+     * @param  {Object.<String, MatrixEvent>} accountData existing account data
+     */
+    constructor(accountData) {
+        super();
+        this._existingValues = accountData;
+        this._values = new Map();
+    }
+
+    /**
+     * @param  {String} type
+     * @return {Promise<Object>} the content of the account data
+     */
+    getAccountDataFromServer(type) {
+        return Promise.resolve(this.getAccountData(type));
+    }
+
+    /**
+     * @param  {String} type
+     * @return {Object} the content of the account data
+     */
+    getAccountData(type) {
+        const modifiedValue = this._values.get(type);
+        if (modifiedValue) {
+            return modifiedValue;
+        }
+        const existingValue = this._existingValues[type];
+        if (existingValue) {
+            return existingValue.getContent();
+        }
+        return null;
+    }
+
+    /**
+     * @param {String} type
+     * @param {Object} content
+     * @return {Promise}
+     */
+    setAccountData(type, content) {
+        const lastEvent = this._values.get(type);
+        this._values.set(type, content);
+        // ensure accountData is emitted on the next tick,
+        // as SecretStorage listens for it while calling this method
+        // and it seems to rely on this.
+        return Promise.resolve().then(() => {
+            const event = new MatrixEvent({type, content});
+            this.emit("accountData", event, lastEvent);
+        });
+    }
+}
+
+/**
+ * Catches the private cross-signing keys set during bootstrapping
+ * by both cache callbacks (see createCryptoStoreCacheCallbacks) as non-cache callbacks.
+ * See CrossSigningInfo constructor
+ */
+class CrossSigningCallbacks {
+    constructor() {
+        this.privateKeys = new Map();
+    }
+
+    // cache callbacks
+    getCrossSigningKeyCache(type, expectedPublicKey) {
+        return this.getCrossSigningKey(type, expectedPublicKey);
+    }
+
+    storeCrossSigningKeyCache(type, key) {
+        this.privateKeys.set(type, key);
+        return Promise.resolve();
+    }
+
+    // non-cache callbacks
+    getCrossSigningKey(type, _expectedPubkey) {
+        return Promise.resolve(this.privateKeys.get(type));
+    }
+
+    saveCrossSigningKeys(privateKeys) {
+        for (const [type, privateKey] of Object.entries(privateKeys)) {
+            this.privateKeys.set(type, privateKey);
+        }
+    }
+}
+
+/**
+ * Catches the 4S private key set during bootstrapping by implementing
+ * the SecretStorage crypto callbacks
+ */
+class SSSSCryptoCallbacks {
+    constructor(delegateCryptoCallbacks) {
+        this._privateKeys = new Map();
+        this._delegateCryptoCallbacks = delegateCryptoCallbacks;
+    }
+
+    async getSecretStorageKey({ keys }, name) {
+        for (const keyId of Object.keys(keys)) {
+            const privateKey = this._privateKeys.get(keyId);
+            if (privateKey) {
+                return [keyId, privateKey];
+            }
+        }
+        // if we don't have the key cached yet, ask
+        // for it to the general crypto callbacks and cache it
+        if (this._delegateCryptoCallbacks) {
+            const result = await this._delegateCryptoCallbacks.
+                getSecretStorageKey({keys}, name);
+            if (result) {
+                const [keyId, privateKey] = result;
+                this._privateKeys.set(keyId, privateKey);
+            }
+            return result;
+        }
+    }
+
+    addPrivateKey(keyId, keyInfo, privKey) {
+        this._privateKeys.set(keyId, privKey);
+        // Also pass along to application to cache if it wishes
+        if (
+            this._delegateCryptoCallbacks &&
+            this._delegateCryptoCallbacks.cacheSecretStorageKey
+        ) {
+            this._delegateCryptoCallbacks.cacheSecretStorageKey(keyId, keyInfo, privKey);
+        }
+    }
+}

src/crypto/OlmDevice.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2016 OpenMarket Ltd
-Copyright 2017 New Vector Ltd
+Copyright 2017, 2019 New Vector Ltd
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,8 +16,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import logger from '../logger';
-import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+import {logger} from '../logger';
+import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
+import * as algorithms from './algorithms';
 
 // The maximum size of an event is 65K, and we base64 the content, so this is a
 // reasonable approximation to the biggest plaintext we can encrypt.
@@ -34,9 +36,15 @@ function checkPayloadLength(payloadString) {
         // Note that even if we manage to do the encryption, the message send may fail,
         // because by the time we've wrapped the ciphertext in the event object, it may
         // exceed 65K. But at least we won't just fail with "abort()" in that case.
-        throw new Error("Message too long (" + payloadString.length + " bytes). " +
+        const err = new Error("Message too long (" + payloadString.length + " bytes). " +
                         "The maximum for an encrypted message is " +
                         MAX_PLAINTEXT_LENGTH + " bytes.");
+        // TODO: [TypeScript] We should have our own error types
+        err.data = {
+            errcode: "M_TOO_LARGE",
+            error: "Payload too large for encrypted message",
+        };
+        throw err;
     }
 }
 
@@ -59,20 +67,17 @@ function checkPayloadLength(payloadString) {
  * Manages the olm cryptography functions. Each OlmDevice has a single
  * OlmAccount and a number of OlmSessions.
  *
- * Accounts and sessions are kept pickled in a sessionStore.
+ * Accounts and sessions are kept pickled in the cryptoStore.
  *
  * @constructor
  * @alias module:crypto/OlmDevice
  *
- * @param {Object} sessionStore A store to be used for data in end-to-end
- *    crypto. This is deprecated and being replaced by cryptoStore.
  * @param {Object} cryptoStore A store for crypto data
  *
  * @property {string} deviceCurve25519Key   Curve25519 key for the account
  * @property {string} deviceEd25519Key      Ed25519 key for the account
  */
-function OlmDevice(sessionStore, cryptoStore) {
-    this._sessionStore = sessionStore;
+export function OlmDevice(cryptoStore) {
     this._cryptoStore = cryptoStore;
     this._pickleKey = "DEFAULT_KEY";
 
@@ -81,7 +86,7 @@ function OlmDevice(sessionStore, cryptoStore) {
     this.deviceEd25519Key = null;
     this._maxOneTimeKeys = null;
 
-    // we don't bother stashing outboundgroupsessions in the sessionstore -
+    // we don't bother stashing outboundgroupsessions in the cryptoStore -
     // instead we keep them here.
     this._outboundGroupSessionStore = {};
 
@@ -102,26 +107,65 @@ function OlmDevice(sessionStore, cryptoStore) {
     // Keys are strings of form "<senderKey>|<session_id>|<message_index>"
     // Values are objects of the form "{id: <event id>, timestamp: <ts>}"
     this._inboundGroupSessionMessageIndexes = {};
+
+    // Keep track of sessions that we're starting, so that we don't start
+    // multiple sessions for the same device at the same time.
+    this._sessionsInProgress = {};
+
+    // Used by olm to serialise prekey message decryptions
+    this._olmPrekeyPromise = Promise.resolve();
 }
 
 /**
  * Initialise the OlmAccount. This must be called before any other operations
  * on the OlmDevice.
  *
+ * Data from an exported Olm device can be provided
+ * in order to re-create this device.
+ *
  * Attempts to load the OlmAccount from the crypto store, or creates one if none is
  * found.
  *
  * Reads the device keys from the OlmAccount object.
+ *
+ * @param {object} opts
+ * @param {object} opts.fromExportedDevice (Optional) data from exported device
+ *     that must be re-created.
+ *     If present, opts.pickleKey is ignored
+ *     (exported data already provides a pickle key)
+ * @param {object} opts.pickleKey (Optional) pickle key to set instead of default one
  */
-OlmDevice.prototype.init = async function() {
-    await this._migrateFromSessionStore();
-
+OlmDevice.prototype.init = async function(opts = {}) {
     let e2eKeys;
     const account = new global.Olm.Account();
+
+    const { pickleKey, fromExportedDevice } = opts;
+
     try {
-        await _initialiseAccount(
-            this._sessionStore, this._cryptoStore, this._pickleKey, account,
-        );
+        if (fromExportedDevice) {
+            if (pickleKey) {
+                logger.warn(
+                    'ignoring opts.pickleKey'
+                    + ' because opts.fromExportedDevice is present.',
+                );
+            }
+            this._pickleKey = fromExportedDevice.pickleKey;
+            await _initialiseFromExportedDevice(
+                fromExportedDevice,
+                this._cryptoStore,
+                this._pickleKey,
+                account,
+            );
+        } else {
+            if (pickleKey) {
+                this._pickleKey = pickleKey;
+            }
+            await _initialiseAccount(
+                this._cryptoStore,
+                this._pickleKey,
+                account,
+            );
+        }
         e2eKeys = JSON.parse(account.identity_keys());
 
         this._maxOneTimeKeys = account.max_number_of_one_time_keys();
@@ -133,18 +177,67 @@ OlmDevice.prototype.init = async function() {
     this.deviceEd25519Key = e2eKeys.ed25519;
 };
 
-async function _initialiseAccount(sessionStore, cryptoStore, pickleKey, account) {
-    await cryptoStore.doTxn('readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT], (txn) => {
-        cryptoStore.getAccount(txn, (pickledAccount) => {
-            if (pickledAccount !== null) {
-                account.unpickle(pickleKey, pickledAccount);
-            } else {
-                account.create();
-                pickledAccount = account.pickle(pickleKey);
-                cryptoStore.storeAccount(txn, pickledAccount);
-            }
-        });
+/**
+ * Populates the crypto store using data that was exported from an existing device.
+ * Note that for now only the “account” and “sessions” stores are populated;
+ * Other stores will be as with a new device.
+ *
+ * @param {Object} exportedData Data exported from another device
+ *     through the “export” method.
+ * @param {module:crypto/store/base~CryptoStore} cryptoStore storage for the crypto layer
+ * @param {string} pickleKey the key that was used to pickle the exported data
+ * @param {Olm.Account} account an olm account to initialize
+ */
+async function _initialiseFromExportedDevice(
+    exportedData,
+    cryptoStore,
+    pickleKey,
+    account,
+) {
+    await cryptoStore.doTxn(
+        'readwrite',
+        [
+            IndexedDBCryptoStore.STORE_ACCOUNT,
+            IndexedDBCryptoStore.STORE_SESSIONS,
+        ],
+        (txn) => {
+            cryptoStore.storeAccount(txn, exportedData.pickledAccount);
+            exportedData.sessions.forEach((session) => {
+                const {
+                    deviceKey,
+                    sessionId,
+                } = session;
+                const sessionInfo = {
+                    session: session.session,
+                    lastReceivedMessageTs: session.lastReceivedMessageTs,
+                };
+                cryptoStore.storeEndToEndSession(
+                    deviceKey,
+                    sessionId,
+                    sessionInfo,
+                    txn,
+                );
+            });
     });
+    account.unpickle(pickleKey, exportedData.pickledAccount);
+}
+
+async function _initialiseAccount(cryptoStore, pickleKey, account) {
+    await cryptoStore.doTxn(
+        'readwrite',
+        [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            cryptoStore.getAccount(txn, (pickledAccount) => {
+                if (pickledAccount !== null) {
+                    account.unpickle(pickleKey, pickledAccount);
+                } else {
+                    account.create();
+                    pickledAccount = account.pickle(pickleKey);
+                    cryptoStore.storeAccount(txn, pickledAccount);
+                }
+            });
+        },
+    );
 }
 
 /**
@@ -154,95 +247,6 @@ OlmDevice.getOlmVersion = function() {
     return global.Olm.get_library_version();
 };
 
-OlmDevice.prototype._migrateFromSessionStore = async function() {
-    // account
-    await this._cryptoStore.doTxn(
-        'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT], (txn) => {
-            this._cryptoStore.getAccount(txn, (pickledAccount) => {
-                if (pickledAccount === null) {
-                    // Migrate from sessionStore
-                    pickledAccount = this._sessionStore.getEndToEndAccount();
-                    if (pickledAccount !== null) {
-                        logger.log("Migrating account from session store");
-                        this._cryptoStore.storeAccount(txn, pickledAccount);
-                    }
-                }
-            });
-        },
-    );
-
-    // remove the old account now the transaction has completed. Either we've
-    // migrated it or decided not to, either way we want to blow away the old data.
-    this._sessionStore.removeEndToEndAccount();
-
-    // sessions
-    const sessions = this._sessionStore.getAllEndToEndSessions();
-    if (Object.keys(sessions).length > 0) {
-        await this._cryptoStore.doTxn(
-            'readwrite', [IndexedDBCryptoStore.STORE_SESSIONS], (txn) => {
-                // Don't migrate sessions from localstorage if we already have sessions
-                // in indexeddb, since this means we've already migrated and an old version
-                // has run against the same localstorage and created some spurious sessions.
-                this._cryptoStore.countEndToEndSessions(txn, (count) => {
-                    if (count) {
-                        logger.log("Crypto store already has sessions: not migrating");
-                        return;
-                    }
-                    let numSessions = 0;
-                    for (const deviceKey of Object.keys(sessions)) {
-                        for (const sessionId of Object.keys(sessions[deviceKey])) {
-                            numSessions++;
-                            this._cryptoStore.storeEndToEndSession(
-                                deviceKey, sessionId, sessions[deviceKey][sessionId], txn,
-                            );
-                        }
-                    }
-                    logger.log(
-                        "Migrating " + numSessions + " sessions from session store",
-                    );
-                });
-            },
-        );
-
-        this._sessionStore.removeAllEndToEndSessions();
-    }
-
-    // inbound group sessions
-    const ibGroupSessions = this._sessionStore.getAllEndToEndInboundGroupSessionKeys();
-    if (Object.keys(ibGroupSessions).length > 0) {
-        let numIbSessions = 0;
-        await this._cryptoStore.doTxn(
-            'readwrite', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
-                // We always migrate inbound group sessions, even if we already have some
-                // in the new store. They should be be safe to migrate.
-                for (const s of ibGroupSessions) {
-                    try {
-                        this._cryptoStore.addEndToEndInboundGroupSession(
-                            s.senderKey, s.sessionId,
-                            JSON.parse(
-                                this._sessionStore.getEndToEndInboundGroupSession(
-                                    s.senderKey, s.sessionId,
-                                ),
-                            ), txn,
-                        );
-                    } catch (e) {
-                        logger.warn(
-                            "Failed to migrate session " + s.senderKey + "/" +
-                            s.sessionId + ": " + e.stack || e,
-                        );
-                    }
-                    ++numIbSessions;
-                }
-                logger.log(
-                    "Migrated " + numIbSessions +
-                    " inbound group sessions from session store",
-                );
-            },
-        );
-        this._sessionStore.removeAllEndToEndInboundGroupSessions();
-    }
-};
-
 /**
  * extract our OlmAccount from the crypto store and call the given function
  * with the account object
@@ -281,6 +285,38 @@ OlmDevice.prototype._storeAccount = function(txn, account) {
     this._cryptoStore.storeAccount(txn, account.pickle(this._pickleKey));
 };
 
+/**
+ * Export data for re-creating the Olm device later.
+ * TODO export data other than just account and (P2P) sessions.
+ *
+ * @return {Promise<object>} The exported data
+*/
+OlmDevice.prototype.export = async function() {
+    const result = {
+        pickleKey: this._pickleKey,
+    };
+    await this._cryptoStore.doTxn(
+        'readonly',
+        [
+            IndexedDBCryptoStore.STORE_ACCOUNT,
+            IndexedDBCryptoStore.STORE_SESSIONS,
+        ],
+        (txn) => {
+            this._cryptoStore.getAccount(txn, (pickledAccount) => {
+                result.pickledAccount = pickledAccount;
+            });
+            result.sessions = [];
+            // Note that the pickledSession object we get in the callback
+            // is not exactly the same thing you get in method _getSession
+            // see documentation of IndexedDBCryptoStore.getAllEndToEndSessions
+            this._cryptoStore.getAllEndToEndSessions(txn, (pickledSession) => {
+                result.sessions.push(pickledSession);
+            });
+        },
+    );
+    return result;
+};
+
 /**
  * extract an OlmSession from the session store and call the given function
  * The session is useable only within the callback passed to this
@@ -441,6 +477,36 @@ OlmDevice.prototype.generateOneTimeKeys = function(numKeys) {
     );
 };
 
+/**
+ * Generate a new fallback keys
+ *
+ * @return {Promise} Resolved once the account is saved back having generated the key
+ */
+OlmDevice.prototype.generateFallbackKey = async function() {
+    await this._cryptoStore.doTxn(
+        'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                account.generate_fallback_key();
+                this._storeAccount(txn, account);
+            });
+        },
+    );
+};
+
+OlmDevice.prototype.getFallbackKey = async function() {
+    let result;
+    await this._cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                result = JSON.parse(account.fallback_key());
+            });
+        },
+    );
+    return result;
+};
+
 /**
  * Generate a new outbound session
  *
@@ -553,6 +619,15 @@ OlmDevice.prototype.createInboundSession = async function(
  * @return {Promise<string[]>}  a list of known session ids for the device
  */
 OlmDevice.prototype.getSessionIdsForDevice = async function(theirDeviceIdentityKey) {
+    if (this._sessionsInProgress[theirDeviceIdentityKey]) {
+        logger.log("waiting for olm session to be created");
+        try {
+            await this._sessionsInProgress[theirDeviceIdentityKey];
+        } catch (e) {
+            // if the session failed to be created, just fall through and
+            // return an empty result
+        }
+    }
     let sessionIds;
     await this._cryptoStore.doTxn(
         'readonly', [IndexedDBCryptoStore.STORE_SESSIONS],
@@ -573,10 +648,18 @@ OlmDevice.prototype.getSessionIdsForDevice = async function(theirDeviceIdentityK
  *
  * @param {string} theirDeviceIdentityKey Curve25519 identity key for the
  *     remote device
+ * @param {boolean} nowait Don't wait for an in-progress session to complete.
+ *     This should only be set to true of the calling function is the function
+ *     that marked the session as being in-progress.
  * @return {Promise<?string>}  session id, or null if no established session
  */
-OlmDevice.prototype.getSessionIdForDevice = async function(theirDeviceIdentityKey) {
-    const sessionInfos = await this.getSessionInfoForDevice(theirDeviceIdentityKey);
+OlmDevice.prototype.getSessionIdForDevice = async function(
+    theirDeviceIdentityKey, nowait,
+) {
+    const sessionInfos = await this.getSessionInfoForDevice(
+        theirDeviceIdentityKey, nowait,
+    );
+
     if (sessionInfos.length === 0) {
         return null;
     }
@@ -611,9 +694,21 @@ OlmDevice.prototype.getSessionIdForDevice = async function(theirDeviceIdentityKe
  * message and is therefore past the pre-key stage), and 'sessionId'.
  *
  * @param {string} deviceIdentityKey Curve25519 identity key for the device
+ * @param {boolean} nowait Don't wait for an in-progress session to complete.
+ *     This should only be set to true of the calling function is the function
+ *     that marked the session as being in-progress.
  * @return {Array.<{sessionId: string, hasReceivedMessage: Boolean}>}
  */
-OlmDevice.prototype.getSessionInfoForDevice = async function(deviceIdentityKey) {
+OlmDevice.prototype.getSessionInfoForDevice = async function(deviceIdentityKey, nowait) {
+    if (this._sessionsInProgress[deviceIdentityKey] && !nowait) {
+        logger.log("waiting for olm session to be created");
+        try {
+            await this._sessionsInProgress[deviceIdentityKey];
+        } catch (e) {
+            // if the session failed to be created, then just fall through and
+            // return an empty result
+        }
+    }
     const info = [];
 
     await this._cryptoStore.doTxn(
@@ -657,6 +752,11 @@ OlmDevice.prototype.encryptMessage = async function(
         'readwrite', [IndexedDBCryptoStore.STORE_SESSIONS],
         (txn) => {
             this._getSession(theirDeviceIdentityKey, sessionId, txn, (sessionInfo) => {
+                const sessionDesc = sessionInfo.session.describe();
+                logger.log(
+                    "encryptMessage: Olm Session ID " + sessionId + " to " +
+                    theirDeviceIdentityKey + ": " + sessionDesc,
+                );
                 res = sessionInfo.session.encrypt(payloadString);
                 this._saveSession(theirDeviceIdentityKey, sessionInfo, txn);
             });
@@ -684,6 +784,11 @@ OlmDevice.prototype.decryptMessage = async function(
         'readwrite', [IndexedDBCryptoStore.STORE_SESSIONS],
         (txn) => {
             this._getSession(theirDeviceIdentityKey, sessionId, txn, (sessionInfo) => {
+                const sessionDesc = sessionInfo.session.describe();
+                logger.log(
+                    "decryptMessage: Olm Session ID " + sessionId + " from " +
+                    theirDeviceIdentityKey + ": " + sessionDesc,
+                );
                 payloadString = sessionInfo.session.decrypt(messageType, ciphertext);
                 sessionInfo.lastReceivedMessageTs = Date.now();
                 this._saveSession(theirDeviceIdentityKey, sessionInfo, txn);
@@ -724,6 +829,18 @@ OlmDevice.prototype.matchesSession = async function(
     return matches;
 };
 
+OlmDevice.prototype.recordSessionProblem = async function(deviceKey, type, fixed) {
+    await this._cryptoStore.storeEndToEndSessionProblem(deviceKey, type, fixed);
+};
+
+OlmDevice.prototype.sessionMayHaveProblems = async function(deviceKey, timestamp) {
+    return await this._cryptoStore.getEndToEndSessionProblem(deviceKey, timestamp);
+};
+
+OlmDevice.prototype.filterOutNotifiedErrorDevices = async function(devices) {
+    return await this._cryptoStore.filterOutNotifiedErrorDevices(devices);
+};
+
 
 // Outbound group session
 // ======================
@@ -793,6 +910,8 @@ OlmDevice.prototype.createOutboundGroupSession = function() {
 OlmDevice.prototype.encryptGroupMessage = function(sessionId, payloadString) {
     const self = this;
 
+    logger.log(`encrypting msg with megolm session ${sessionId}`);
+
     checkPayloadLength(payloadString);
 
     return this._getOutboundGroupSession(sessionId, function(session) {
@@ -869,9 +988,9 @@ OlmDevice.prototype._getInboundGroupSession = function(
     roomId, senderKey, sessionId, txn, func,
 ) {
     this._cryptoStore.getEndToEndInboundGroupSession(
-        senderKey, sessionId, txn, (sessionData) => {
+        senderKey, sessionId, txn, (sessionData, withheld) => {
             if (sessionData === null) {
-                func(null);
+                func(null, null, withheld);
                 return;
             }
 
@@ -885,7 +1004,7 @@ OlmDevice.prototype._getInboundGroupSession = function(
             }
 
             this._unpickleInboundGroupSession(sessionData, (session) => {
-                func(session, sessionData);
+                func(session, sessionData, withheld);
             });
         },
     );
@@ -903,26 +1022,22 @@ OlmDevice.prototype._getInboundGroupSession = function(
  * @param {Object<string, string>} keysClaimed Other keys the sender claims.
  * @param {boolean} exportFormat true if the megolm keys are in export format
  *    (ie, they lack an ed25519 signature)
+ * @param {Object} [extraSessionData={}] any other data to be include with the session
  */
 OlmDevice.prototype.addInboundGroupSession = async function(
     roomId, senderKey, forwardingCurve25519KeyChain,
     sessionId, sessionKey, keysClaimed,
-    exportFormat,
+    exportFormat, extraSessionData = {},
 ) {
     await this._cryptoStore.doTxn(
-        'readwrite', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+        'readwrite', [
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+        ], (txn) => {
             /* if we already have this session, consider updating it */
             this._getInboundGroupSession(
                 roomId, senderKey, sessionId, txn,
                 (existingSession, existingSessionData) => {
-                    if (existingSession) {
-                        logger.log(
-                            "Update for megolm session " + senderKey + "/" + sessionId,
-                        );
-                        // for now we just ignore updates. TODO: implement something here
-                        return;
-                    }
-
                     // new session.
                     const session = new global.Olm.InboundGroupSession();
                     try {
@@ -938,14 +1053,40 @@ OlmDevice.prototype.addInboundGroupSession = async function(
                             );
                         }
 
-                        const sessionData = {
+                        if (existingSession) {
+                            logger.log(
+                                "Update for megolm session "
+                                    + senderKey + "/" + sessionId,
+                            );
+                            if (existingSession.first_known_index()
+                                <= session.first_known_index()
+                                && !(existingSession.first_known_index() == session.first_known_index()
+                                    && !extraSessionData.untrusted
+                                    && existingSessionData.untrusted)) {
+                                // existing session has lower index (i.e. can
+                                // decrypt more), or they have the same index and
+                                // the new sessions trust does not win over the old
+                                // sessions trust, so keep it
+                                logger.log(
+                                    `Keeping existing megolm session ${sessionId}`,
+                                );
+                                return;
+                            }
+                        }
+
+                        logger.info(
+                            "Storing megolm session " + senderKey + "/" + sessionId +
+                            " with first index " + session.first_known_index(),
+                        );
+
+                        const sessionData = Object.assign({}, extraSessionData, {
                             room_id: roomId,
                             session: session.pickle(this._pickleKey),
                             keysClaimed: keysClaimed,
                             forwardingCurve25519KeyChain: forwardingCurve25519KeyChain,
-                        };
+                        });
 
-                        this._cryptoStore.addEndToEndInboundGroupSession(
+                        this._cryptoStore.storeEndToEndInboundGroupSession(
                             senderKey, sessionId, sessionData, txn,
                         );
                     } finally {
@@ -957,6 +1098,60 @@ OlmDevice.prototype.addInboundGroupSession = async function(
     );
 };
 
+/**
+ * Record in the data store why an inbound group session was withheld.
+ *
+ * @param {string} roomId     room that the session belongs to
+ * @param {string} senderKey  base64-encoded curve25519 key of the sender
+ * @param {string} sessionId  session identifier
+ * @param {string} code       reason code
+ * @param {string} reason     human-readable version of `code`
+ */
+OlmDevice.prototype.addInboundGroupSessionWithheld = async function(
+    roomId, senderKey, sessionId, code, reason,
+) {
+    await this._cryptoStore.doTxn(
+        'readwrite', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD],
+        (txn) => {
+            this._cryptoStore.storeEndToEndInboundGroupSessionWithheld(
+                senderKey, sessionId,
+                {
+                    room_id: roomId,
+                    code: code,
+                    reason: reason,
+                },
+                txn,
+            );
+        },
+    );
+};
+
+export const WITHHELD_MESSAGES = {
+    "m.unverified": "The sender has disabled encrypting to unverified devices.",
+    "m.blacklisted": "The sender has blocked you.",
+    "m.unauthorised": "You are not authorised to read the message.",
+    "m.no_olm": "Unable to establish a secure channel.",
+};
+
+/**
+ * Calculate the message to use for the exception when a session key is withheld.
+ *
+ * @param {object} withheld  An object that describes why the key was withheld.
+ *
+ * @return {string} the message
+ *
+ * @private
+ */
+function _calculateWithheldMessage(withheld) {
+    if (withheld.code && withheld.code in WITHHELD_MESSAGES) {
+        return WITHHELD_MESSAGES[withheld.code];
+    } else if (withheld.reason) {
+        return withheld.reason;
+    } else {
+        return "decryption key withheld";
+    }
+}
+
 /**
  * Decrypt a received message with an inbound group session
  *
@@ -977,16 +1172,49 @@ OlmDevice.prototype.decryptGroupMessage = async function(
     roomId, senderKey, sessionId, body, eventId, timestamp,
 ) {
     let result;
+    // when the localstorage crypto store is used as an indexeddb backend,
+    // exceptions thrown from within the inner function are not passed through
+    // to the top level, so we store exceptions in a variable and raise them at
+    // the end
+    let error;
 
     await this._cryptoStore.doTxn(
-        'readwrite', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+        'readwrite', [
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+        ], (txn) => {
             this._getInboundGroupSession(
-                roomId, senderKey, sessionId, txn, (session, sessionData) => {
+                roomId, senderKey, sessionId, txn, (session, sessionData, withheld) => {
                     if (session === null) {
+                        if (withheld) {
+                            error = new algorithms.DecryptionError(
+                                "MEGOLM_UNKNOWN_INBOUND_SESSION_ID",
+                                _calculateWithheldMessage(withheld),
+                                {
+                                    session: senderKey + '|' + sessionId,
+                                },
+                            );
+                        }
                         result = null;
                         return;
                     }
-                    const res = session.decrypt(body);
+                    let res;
+                    try {
+                        res = session.decrypt(body);
+                    } catch (e) {
+                        if (e && e.message === 'OLM.UNKNOWN_MESSAGE_INDEX' && withheld) {
+                            error = new algorithms.DecryptionError(
+                                "MEGOLM_UNKNOWN_INBOUND_SESSION_ID",
+                                _calculateWithheldMessage(withheld),
+                                {
+                                    session: senderKey + '|' + sessionId,
+                                },
+                            );
+                        } else {
+                            error = e;
+                        }
+                        return;
+                    }
 
                     let plaintext = res.plaintext;
                     if (plaintext === undefined) {
@@ -1008,10 +1236,11 @@ OlmDevice.prototype.decryptGroupMessage = async function(
                                 msgInfo.id !== eventId ||
                                 msgInfo.timestamp !== timestamp
                             ) {
-                                throw new Error(
+                                error = new Error(
                                     "Duplicate message index, possible replay attack: " +
                                     messageIndexKey,
                                 );
+                                return;
                             }
                         }
                         this._inboundGroupSessionMessageIndexes[messageIndexKey] = {
@@ -1031,12 +1260,16 @@ OlmDevice.prototype.decryptGroupMessage = async function(
                         forwardingCurve25519KeyChain: (
                             sessionData.forwardingCurve25519KeyChain || []
                         ),
+                        untrusted: sessionData.untrusted,
                     };
                 },
             );
         },
     );
 
+    if (error) {
+        throw error;
+    }
     return result;
 };
 
@@ -1045,14 +1278,17 @@ OlmDevice.prototype.decryptGroupMessage = async function(
  *
  * @param {string} roomId    room in which the message was received
  * @param {string} senderKey base64-encoded curve25519 key of the sender
- * @param {sring} sessionId session identifier
+ * @param {string} sessionId session identifier
  *
  * @returns {Promise<boolean>} true if we have the keys to this session
  */
 OlmDevice.prototype.hasInboundSessionKeys = async function(roomId, senderKey, sessionId) {
     let result;
     await this._cryptoStore.doTxn(
-        'readonly', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+        'readonly', [
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+        ], (txn) => {
             this._cryptoStore.getEndToEndInboundGroupSession(
                 senderKey, sessionId, txn, (sessionData) => {
                     if (sessionData === null) {
@@ -1103,7 +1339,10 @@ OlmDevice.prototype.getInboundGroupSessionKey = async function(
 ) {
     let result;
     await this._cryptoStore.doTxn(
-        'readonly', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+        'readonly', [
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+        ], (txn) => {
             this._getInboundGroupSession(
                 roomId, senderKey, sessionId, txn, (session, sessionData) => {
                     if (session === null) {
@@ -1182,6 +1421,3 @@ OlmDevice.prototype.verifySignature = function(
         util.ed25519_verify(key, message, signature);
     });
 };
-
-/** */
-module.exports = OlmDevice;

src/crypto/OutgoingRoomKeyRequestManager.js

@@ -14,10 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import Promise from 'bluebird';
-
-import logger from '../logger';
-import utils from '../utils';
+import {logger} from '../logger';
+import * as utils from '../utils';
 
 /**
  * Internal module. Management of outgoing room key requests.
@@ -60,7 +58,7 @@ const SEND_KEY_REQUESTS_DELAY_MS = 500;
  *
  * @enum {number}
  */
-const ROOM_KEY_REQUEST_STATES = {
+export const ROOM_KEY_REQUEST_STATES = {
     /** request not yet sent */
     UNSENT: 0,
 
@@ -77,7 +75,7 @@ const ROOM_KEY_REQUEST_STATES = {
     CANCELLATION_PENDING_AND_WILL_RESEND: 3,
 };
 
-export default class OutgoingRoomKeyRequestManager {
+export class OutgoingRoomKeyRequestManager {
     constructor(baseApis, deviceId, cryptoStore) {
         this._baseApis = baseApis;
         this._deviceId = deviceId;
@@ -99,10 +97,6 @@ export default class OutgoingRoomKeyRequestManager {
      */
     start() {
         this._clientRunning = true;
-
-        // set the timer going, to handle any requests which didn't get sent
-        // on the previous run of the client.
-        this._startTimer();
     }
 
     /**
@@ -115,7 +109,14 @@ export default class OutgoingRoomKeyRequestManager {
     }
 
     /**
-     * Send off a room key request, if we haven't already done so.
+     * Send any requests that have been queued
+     */
+    sendQueuedRequests() {
+        this._startTimer();
+    }
+
+    /**
+     * Queue up a room key request, if we haven't already queued or sent one.
      *
      * The `requestBody` is compared (with a deep-equality check) against
      * previous queued or sent requests and if it matches, no change is made.
@@ -124,35 +125,115 @@ export default class OutgoingRoomKeyRequestManager {
      *
      * @param {module:crypto~RoomKeyRequestBody} requestBody
      * @param {Array<{userId: string, deviceId: string}>} recipients
+     * @param {boolean} resend whether to resend the key request if there is
+     *    already one
      *
      * @returns {Promise} resolves when the request has been added to the
      *    pending list (or we have established that a similar request already
      *    exists)
      */
-    sendRoomKeyRequest(requestBody, recipients) {
-        return this._cryptoStore.getOrAddOutgoingRoomKeyRequest({
-            requestBody: requestBody,
-            recipients: recipients,
-            requestId: this._baseApis.makeTxnId(),
-            state: ROOM_KEY_REQUEST_STATES.UNSENT,
-        }).then((req) => {
-            if (req.state === ROOM_KEY_REQUEST_STATES.UNSENT) {
-                this._startTimer();
+    async queueRoomKeyRequest(requestBody, recipients, resend=false) {
+        const req = await this._cryptoStore.getOutgoingRoomKeyRequest(
+            requestBody,
+        );
+        if (!req) {
+            await this._cryptoStore.getOrAddOutgoingRoomKeyRequest({
+                requestBody: requestBody,
+                recipients: recipients,
+                requestId: this._baseApis.makeTxnId(),
+                state: ROOM_KEY_REQUEST_STATES.UNSENT,
+            });
+        } else {
+            switch (req.state) {
+            case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND:
+            case ROOM_KEY_REQUEST_STATES.UNSENT:
+                // nothing to do here, since we're going to send a request anyways
+                return;
+
+            case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING: {
+                // existing request is about to be cancelled.  If we want to
+                // resend, then change the state so that it resends after
+                // cancelling.  Otherwise, just cancel the cancellation.
+                const state = resend ?
+                    ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND :
+                    ROOM_KEY_REQUEST_STATES.SENT;
+                await this._cryptoStore.updateOutgoingRoomKeyRequest(
+                    req.requestId, ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING, {
+                        state,
+                        cancellationTxnId: this._baseApis.makeTxnId(),
+                    },
+                );
+                break;
             }
-        });
+            case ROOM_KEY_REQUEST_STATES.SENT: {
+                // a request has already been sent.  If we don't want to
+                // resend, then do nothing.  If we do want to, then cancel the
+                // existing request and send a new one.
+                if (resend) {
+                    const state =
+                          ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND;
+                    const updatedReq =
+                          await this._cryptoStore.updateOutgoingRoomKeyRequest(
+                              req.requestId, ROOM_KEY_REQUEST_STATES.SENT, {
+                                  state,
+                                  cancellationTxnId: this._baseApis.makeTxnId(),
+                                  // need to use a new transaction ID so that
+                                  // the request gets sent
+                                  requestTxnId: this._baseApis.makeTxnId(),
+                              },
+                          );
+                    if (!updatedReq) {
+                        // updateOutgoingRoomKeyRequest couldn't find the request
+                        // in state ROOM_KEY_REQUEST_STATES.SENT, so we must have
+                        // raced with another tab to mark the request cancelled.
+                        // Try again, to make sure the request is resent.
+                        return await this.queueRoomKeyRequest(
+                            requestBody, recipients, resend,
+                        );
+                    }
+
+                    // We don't want to wait for the timer, so we send it
+                    // immediately. (We might actually end up racing with the timer,
+                    // but that's ok: even if we make the request twice, we'll do it
+                    // with the same transaction_id, so only one message will get
+                    // sent).
+                    //
+                    // (We also don't want to wait for the response from the server
+                    // here, as it will slow down processing of received keys if we
+                    // do.)
+                    try {
+                        await this._sendOutgoingRoomKeyRequestCancellation(
+                            updatedReq,
+                            true,
+                        );
+                    } catch (e) {
+                        logger.error(
+                            "Error sending room key request cancellation;"
+                                + " will retry later.", e,
+                        );
+                    }
+                    // The request has transitioned from
+                    // CANCELLATION_PENDING_AND_WILL_RESEND to UNSENT. We
+                    // still need to resend the request which is now UNSENT, so
+                    // start the timer if it isn't already started.
+                }
+                break;
+            }
+            default:
+                throw new Error('unhandled state: ' + req.state);
+            }
+        }
     }
 
     /**
      * Cancel room key requests, if any match the given requestBody
      *
      * @param {module:crypto~RoomKeyRequestBody} requestBody
-     * @param {boolean} andResend if true, transition to UNSENT instead of
-     *                            deleting after sending cancellation.
      *
      * @returns {Promise} resolves when the request has been updated in our
      *    pending list.
      */
-    cancelRoomKeyRequest(requestBody, andResend=false) {
+    cancelRoomKeyRequest(requestBody) {
         return this._cryptoStore.getOutgoingRoomKeyRequest(
             requestBody,
         ).then((req) => {
@@ -183,15 +264,10 @@ export default class OutgoingRoomKeyRequestManager {
                     );
 
                 case ROOM_KEY_REQUEST_STATES.SENT: {
-                    // If `andResend` is set, transition to UNSENT once the
-                    // cancellation has successfully been sent.
-                    const state = andResend ?
-                        ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND :
-                        ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING;
                     // send a cancellation.
                     return this._cryptoStore.updateOutgoingRoomKeyRequest(
                         req.requestId, ROOM_KEY_REQUEST_STATES.SENT, {
-                            state,
+                            state: ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
                             cancellationTxnId: this._baseApis.makeTxnId(),
                         },
                     ).then((updatedReq) => {
@@ -221,20 +297,12 @@ export default class OutgoingRoomKeyRequestManager {
                         // do.)
                         this._sendOutgoingRoomKeyRequestCancellation(
                             updatedReq,
-                            andResend,
                         ).catch((e) => {
                             logger.error(
                                 "Error sending room key request cancellation;"
                                 + " will retry later.", e,
                             );
                             this._startTimer();
-                        }).then(() => {
-                            if (!andResend) return;
-                            // The request has transitioned from
-                            // CANCELLATION_PENDING_AND_WILL_RESEND to UNSENT. We
-                            // still need to resend the request which is now UNSENT, so
-                            // start the timer if it isn't already started.
-                            this._startTimer();
                         });
                     });
                 }
@@ -259,6 +327,21 @@ export default class OutgoingRoomKeyRequestManager {
         );
     }
 
+    /**
+     * Find anything in `sent` state, and kick it around the loop again.
+     * This is intended for situations where something substantial has changed, and we
+     * don't really expect the other end to even care about the cancellation.
+     * For example, after initialization or self-verification.
+     * @return {Promise} An array of `queueRoomKeyRequest` outputs.
+     */
+    async cancelAndResendAllOutgoingRequests() {
+        const outgoings = await this._cryptoStore.getAllOutgoingRoomKeyRequestsByState(
+            ROOM_KEY_REQUEST_STATES.SENT,
+        );
+        return Promise.all(outgoings.map(({ requestBody, recipients }) =>
+            this.queueRoomKeyRequest(requestBody, recipients, true)));
+    }
+
     // start the background timer to send queued requests, if the timer isn't
     // already running
     _startTimer() {
@@ -280,7 +363,7 @@ export default class OutgoingRoomKeyRequestManager {
                 logger.warn(
                     `error in OutgoingRoomKeyRequestManager: ${e}`,
                 );
-            }).done();
+            });
         };
 
         this._sendOutgoingRoomKeyRequestsTimer = global.setTimeout(
@@ -298,15 +381,12 @@ export default class OutgoingRoomKeyRequestManager {
             return Promise.resolve();
         }
 
-        logger.log("Looking for queued outgoing room key requests");
-
         return this._cryptoStore.getOutgoingRoomKeyRequestByState([
             ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
             ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
             ROOM_KEY_REQUEST_STATES.UNSENT,
         ]).then((req) => {
             if (!req) {
-                logger.log("No more outgoing room key requests");
                 this._sendOutgoingRoomKeyRequestsTimer = null;
                 return;
             }
@@ -330,8 +410,7 @@ export default class OutgoingRoomKeyRequestManager {
             }).catch((e) => {
                 logger.error("Error sending room key request; will retry later.", e);
                 this._sendOutgoingRoomKeyRequestsTimer = null;
-                this._startTimer();
-            }).done();
+            });
         });
     }
 
@@ -351,7 +430,7 @@ export default class OutgoingRoomKeyRequestManager {
         };
 
         return this._sendMessageToDevices(
-            requestMessage, req.recipients, req.requestId,
+            requestMessage, req.recipients, req.requestTxnId || req.requestId,
         ).then(() => {
             return this._cryptoStore.updateOutgoingRoomKeyRequest(
                 req.requestId, ROOM_KEY_REQUEST_STATES.UNSENT,

src/crypto/RoomList.js

@@ -1,5 +1,5 @@
 /*
-Copyright 2018 New Vector Ltd
+Copyright 2018, 2019 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -20,46 +20,27 @@ limitations under the License.
  * Manages the list of encrypted rooms
  */
 
-import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
 
 /**
  * @alias module:crypto/RoomList
  */
-export default class RoomList {
-    constructor(cryptoStore, sessionStore) {
+export class RoomList {
+    constructor(cryptoStore) {
         this._cryptoStore = cryptoStore;
-        this._sessionStore = sessionStore;
 
         // Object of roomId -> room e2e info object (body of the m.room.encryption event)
         this._roomEncryption = {};
     }
 
     async init() {
-        let removeSessionStoreRooms = false;
         await this._cryptoStore.doTxn(
             'readwrite', [IndexedDBCryptoStore.STORE_ROOMS], (txn) => {
                 this._cryptoStore.getEndToEndRooms(txn, (result) => {
-                    if (result === null || Object.keys(result).length === 0) {
-                        // migrate from session store, if there's data there
-                        const sessStoreRooms = this._sessionStore.getAllEndToEndRooms();
-                        if (sessStoreRooms !== null) {
-                            for (const roomId of Object.keys(sessStoreRooms)) {
-                                this._cryptoStore.storeEndToEndRoom(
-                                    roomId, sessStoreRooms[roomId], txn,
-                                );
-                            }
-                        }
-                        this._roomEncryption = sessStoreRooms;
-                        removeSessionStoreRooms = true;
-                    } else {
-                        this._roomEncryption = result;
-                    }
+                    this._roomEncryption = result;
                 });
             },
         );
-        if (removeSessionStoreRooms) {
-            this._sessionStore.removeAllEndToEndRooms();
-        }
     }
 
     getRoomEncryption(roomId) {

src/crypto/SecretStorage.js

@@ -0,0 +1,586 @@
+/*
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {EventEmitter} from 'events';
+import {logger} from '../logger';
+import * as olmlib from './olmlib';
+import {randomString} from '../randomstring';
+import {encryptAES, decryptAES} from './aes';
+import {encodeBase64} from "./olmlib";
+
+export const SECRET_STORAGE_ALGORITHM_V1_AES
+    = "m.secret_storage.v1.aes-hmac-sha2";
+
+const ZERO_STR = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
+
+/**
+ * Implements Secure Secret Storage and Sharing (MSC1946)
+ * @module crypto/SecretStorage
+ */
+export class SecretStorage extends EventEmitter {
+    constructor(baseApis, cryptoCallbacks) {
+        super();
+        this._baseApis = baseApis;
+        this._cryptoCallbacks = cryptoCallbacks;
+        this._requests = {};
+        this._incomingRequests = {};
+    }
+
+    async getDefaultKeyId() {
+        const defaultKey = await this._baseApis.getAccountDataFromServer(
+            'm.secret_storage.default_key',
+        );
+        if (!defaultKey) return null;
+        return defaultKey.key;
+    }
+
+    setDefaultKeyId(keyId) {
+        return new Promise(async (resolve, reject) => {
+            const listener = (ev) => {
+                if (
+                    ev.getType() === 'm.secret_storage.default_key' &&
+                    ev.getContent().key === keyId
+                ) {
+                    this._baseApis.removeListener('accountData', listener);
+                    resolve();
+                }
+            };
+            this._baseApis.on('accountData', listener);
+
+            try {
+                await this._baseApis.setAccountData(
+                    'm.secret_storage.default_key',
+                    { key: keyId },
+                );
+            } catch (e) {
+                this._baseApis.removeListener('accountData', listener);
+                reject(e);
+            }
+        });
+    }
+
+    /**
+     * Add a key for encrypting secrets.
+     *
+     * @param {string} algorithm the algorithm used by the key.
+     * @param {object} opts the options for the algorithm.  The properties used
+     *     depend on the algorithm given.
+     * @param {string} [keyId] the ID of the key.  If not given, a random
+     *     ID will be generated.
+     *
+     * @return {object} An object with:
+     *     keyId: {string} the ID of the key
+     *     keyInfo: {object} details about the key (iv, mac, passphrase)
+     */
+    async addKey(algorithm, opts, keyId) {
+        const keyInfo = {algorithm};
+
+        if (!opts) opts = {};
+
+        if (opts.name) {
+            keyInfo.name = opts.name;
+        }
+
+        if (algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            if (opts.passphrase) {
+                keyInfo.passphrase = opts.passphrase;
+            }
+            if (opts.key) {
+                const {iv, mac} = await SecretStorage._calculateKeyCheck(opts.key);
+                keyInfo.iv = iv;
+                keyInfo.mac = mac;
+            }
+        } else {
+            throw new Error(`Unknown key algorithm ${opts.algorithm}`);
+        }
+
+        if (!keyId) {
+            do {
+                keyId = randomString(32);
+            } while (
+                await this._baseApis.getAccountDataFromServer(
+                    `m.secret_storage.key.${keyId}`,
+                )
+            );
+        }
+
+        await this._baseApis.setAccountData(
+            `m.secret_storage.key.${keyId}`, keyInfo,
+        );
+
+        return {
+            keyId,
+            keyInfo,
+        };
+    }
+
+    /**
+     * Get the key information for a given ID.
+     *
+     * @param {string} [keyId = default key's ID] The ID of the key to check
+     *     for. Defaults to the default key ID if not provided.
+     * @returns {Array?} If the key was found, the return value is an array of
+     *     the form [keyId, keyInfo].  Otherwise, null is returned.
+     */
+    async getKey(keyId) {
+        if (!keyId) {
+            keyId = await this.getDefaultKeyId();
+        }
+        if (!keyId) {
+            return null;
+        }
+
+        const keyInfo = await this._baseApis.getAccountDataFromServer(
+            "m.secret_storage.key." + keyId,
+        );
+        return keyInfo ? [keyId, keyInfo] : null;
+    }
+
+    /**
+     * Check whether we have a key with a given ID.
+     *
+     * @param {string} [keyId = default key's ID] The ID of the key to check
+     *     for. Defaults to the default key ID if not provided.
+     * @return {boolean} Whether we have the key.
+     */
+    async hasKey(keyId) {
+        return !!(await this.getKey(keyId));
+    }
+
+    /**
+     * Check whether a key matches what we expect based on the key info
+     *
+     * @param {Uint8Array} key the key to check
+     * @param {object} info the key info
+     *
+     * @return {boolean} whether or not the key matches
+     */
+    async checkKey(key, info) {
+        if (info.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            if (info.mac) {
+                const {mac} = await SecretStorage._calculateKeyCheck(key, info.iv);
+                return info.mac.replace(/=+$/g, '') === mac.replace(/=+$/g, '');
+            } else {
+                // if we have no information, we have to assume the key is right
+                return true;
+            }
+        } else {
+            throw new Error("Unknown algorithm");
+        }
+    }
+
+    static async _calculateKeyCheck(key, iv) {
+        return await encryptAES(ZERO_STR, key, "", iv);
+    }
+
+    /**
+     * Store an encrypted secret on the server
+     *
+     * @param {string} name The name of the secret
+     * @param {string} secret The secret contents.
+     * @param {Array} keys The IDs of the keys to use to encrypt the secret
+     *     or null/undefined to use the default key.
+     */
+    async store(name, secret, keys) {
+        const encrypted = {};
+
+        if (!keys) {
+            const defaultKeyId = await this.getDefaultKeyId();
+            if (!defaultKeyId) {
+                throw new Error("No keys specified and no default key present");
+            }
+            keys = [defaultKeyId];
+        }
+
+        if (keys.length === 0) {
+            throw new Error("Zero keys given to encrypt with!");
+        }
+
+        for (const keyId of keys) {
+            // get key information from key storage
+            const keyInfo = await this._baseApis.getAccountDataFromServer(
+                "m.secret_storage.key." + keyId,
+            );
+            if (!keyInfo) {
+                throw new Error("Unknown key: " + keyId);
+            }
+
+            // encrypt secret, based on the algorithm
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                const keys = {[keyId]: keyInfo};
+                const [, encryption] = await this._getSecretStorageKey(keys, name);
+                encrypted[keyId] = await encryption.encrypt(secret);
+            } else {
+                logger.warn("unknown algorithm for secret storage key " + keyId
+                            + ": " + keyInfo.algorithm);
+                // do nothing if we don't understand the encryption algorithm
+            }
+        }
+
+        // save encrypted secret
+        await this._baseApis.setAccountData(name, {encrypted});
+    }
+
+    /**
+     * Temporary method to fix up existing accounts where secrets
+     * are incorrectly stored without the 'encrypted' level
+     *
+     * @param {string} name The name of the secret
+     * @param {object} secretInfo The account data object
+     * @returns {object} The fixed object or null if no fix was performed
+     */
+    async _fixupStoredSecret(name, secretInfo) {
+        // We assume the secret was only stored passthrough for 1
+        // key - this was all the broken code supported.
+        const keys = Object.keys(secretInfo);
+        if (
+            keys.length === 1 && keys[0] !== 'encrypted' &&
+            secretInfo[keys[0]].passthrough
+        ) {
+            const hasKey = await this.hasKey(keys[0]);
+            if (hasKey) {
+                logger.log("Fixing up passthrough secret: " + name);
+                await this.storePassthrough(name, keys[0]);
+                const newData = await this._baseApis.getAccountDataFromServer(name);
+                return newData;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Get a secret from storage.
+     *
+     * @param {string} name the name of the secret
+     *
+     * @return {string} the contents of the secret
+     */
+    async get(name) {
+        let secretInfo = await this._baseApis.getAccountDataFromServer(name);
+        if (!secretInfo) {
+            return;
+        }
+        if (!secretInfo.encrypted) {
+            // try to fix it up
+            secretInfo = await this._fixupStoredSecret(name, secretInfo);
+            if (!secretInfo || !secretInfo.encrypted) {
+                throw new Error("Content is not encrypted!");
+            }
+        }
+
+        // get possible keys to decrypt
+        const keys = {};
+        for (const keyId of Object.keys(secretInfo.encrypted)) {
+            // get key information from key storage
+            const keyInfo = await this._baseApis.getAccountDataFromServer(
+                "m.secret_storage.key." + keyId,
+            );
+            const encInfo = secretInfo.encrypted[keyId];
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
+                    keys[keyId] = keyInfo;
+                }
+            }
+        }
+
+        if (Object.keys(keys).length === 0) {
+            throw new Error(`Could not decrypt ${name} because none of ` +
+                `the keys it is encrypted with are for a supported algorithm`);
+        }
+
+        let keyId;
+        let decryption;
+        try {
+            // fetch private key from app
+            [keyId, decryption] = await this._getSecretStorageKey(keys, name);
+
+            const encInfo = secretInfo.encrypted[keyId];
+
+            // We don't actually need the decryption object if it's a passthrough
+            // since we just want to return the key itself. It must be base64
+            // encoded, since this is how a key would normally be stored.
+            if (encInfo.passthrough) return encodeBase64(decryption.get_private_key());
+
+            return await decryption.decrypt(encInfo);
+        } finally {
+            if (decryption && decryption.free) decryption.free();
+        }
+    }
+
+    /**
+     * Check if a secret is stored on the server.
+     *
+     * @param {string} name the name of the secret
+     * @param {boolean} checkKey check if the secret is encrypted by a trusted key
+     *
+     * @return {object?} map of key name to key info the secret is encrypted
+     *     with, or null if it is not present or not encrypted with a trusted
+     *     key
+     */
+    async isStored(name, checkKey) {
+        // check if secret exists
+        let secretInfo = await this._baseApis.getAccountDataFromServer(name);
+        if (!secretInfo) return null;
+        if (!secretInfo.encrypted) {
+            // try to fix it up
+            secretInfo = await this._fixupStoredSecret(name, secretInfo);
+            if (!secretInfo || !secretInfo.encrypted) {
+                return null;
+            }
+        }
+
+        if (checkKey === undefined) checkKey = true;
+
+        const ret = {};
+
+        // filter secret encryption keys with supported algorithm
+        for (const keyId of Object.keys(secretInfo.encrypted)) {
+            // get key information from key storage
+            const keyInfo = await this._baseApis.getAccountDataFromServer(
+                "m.secret_storage.key." + keyId,
+            );
+            if (!keyInfo) continue;
+            const encInfo = secretInfo.encrypted[keyId];
+
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
+                    ret[keyId] = keyInfo;
+                }
+            }
+        }
+        return Object.keys(ret).length ? ret : null;
+    }
+
+    /**
+     * Request a secret from another device
+     *
+     * @param {string} name the name of the secret to request
+     * @param {string[]} devices the devices to request the secret from
+     *
+     * @return {string} the contents of the secret
+     */
+    request(name, devices) {
+        const requestId = this._baseApis.makeTxnId();
+
+        const requestControl = this._requests[requestId] = {
+            name,
+            devices,
+        };
+        const promise = new Promise((resolve, reject) => {
+            requestControl.resolve = resolve;
+            requestControl.reject = reject;
+        });
+        const cancel = (reason) => {
+            // send cancellation event
+            const cancelData = {
+                action: "request_cancellation",
+                requesting_device_id: this._baseApis.deviceId,
+                request_id: requestId,
+            };
+            const toDevice = {};
+            for (const device of devices) {
+                toDevice[device] = cancelData;
+            }
+            this._baseApis.sendToDevice("m.secret.request", {
+                [this._baseApis.getUserId()]: toDevice,
+            });
+
+            // and reject the promise so that anyone waiting on it will be
+            // notified
+            requestControl.reject(new Error(reason || "Cancelled"));
+        };
+
+        // send request to devices
+        const requestData = {
+            name,
+            action: "request",
+            requesting_device_id: this._baseApis.deviceId,
+            request_id: requestId,
+        };
+        const toDevice = {};
+        for (const device of devices) {
+            toDevice[device] = requestData;
+        }
+        logger.info(`Request secret ${name} from ${devices}, id ${requestId}`);
+        this._baseApis.sendToDevice("m.secret.request", {
+            [this._baseApis.getUserId()]: toDevice,
+        });
+
+        return {
+            request_id: requestId,
+            promise,
+            cancel,
+        };
+    }
+
+    async _onRequestReceived(event) {
+        const sender = event.getSender();
+        const content = event.getContent();
+        if (sender !== this._baseApis.getUserId()
+            || !(content.name && content.action
+                 && content.requesting_device_id && content.request_id)) {
+            // ignore requests from anyone else, for now
+            return;
+        }
+        const deviceId = content.requesting_device_id;
+        // check if it's a cancel
+        if (content.action === "request_cancellation") {
+            if (this._incomingRequests[deviceId]
+                && this._incomingRequests[deviceId][content.request_id]) {
+                logger.info("received request cancellation for secret (" + sender
+                            + ", " + deviceId + ", " + content.request_id + ")");
+                this.baseApis.emit("crypto.secrets.requestCancelled", {
+                    user_id: sender,
+                    device_id: deviceId,
+                    request_id: content.request_id,
+                });
+            }
+        } else if (content.action === "request") {
+            if (deviceId === this._baseApis.deviceId) {
+                // no point in trying to send ourself the secret
+                return;
+            }
+
+            // check if we have the secret
+            logger.info("received request for secret (" + sender
+                        + ", " + deviceId + ", " + content.request_id + ")");
+            if (!this._cryptoCallbacks.onSecretRequested) {
+                return;
+            }
+            const secret = await this._cryptoCallbacks.onSecretRequested(
+                sender,
+                deviceId,
+                content.request_id,
+                content.name,
+                this._baseApis.checkDeviceTrust(sender, deviceId),
+            );
+            if (secret) {
+                logger.info(`Preparing ${content.name} secret for ${deviceId}`);
+                const payload = {
+                    type: "m.secret.send",
+                    content: {
+                        request_id: content.request_id,
+                        secret: secret,
+                    },
+                };
+                const encryptedContent = {
+                    algorithm: olmlib.OLM_ALGORITHM,
+                    sender_key: this._baseApis._crypto._olmDevice.deviceCurve25519Key,
+                    ciphertext: {},
+                };
+                await olmlib.ensureOlmSessionsForDevices(
+                    this._baseApis._crypto._olmDevice,
+                    this._baseApis,
+                    {
+                        [sender]: [
+                            this._baseApis.getStoredDevice(sender, deviceId),
+                        ],
+                    },
+                );
+                await olmlib.encryptMessageForDevice(
+                    encryptedContent.ciphertext,
+                    this._baseApis.getUserId(),
+                    this._baseApis.deviceId,
+                    this._baseApis._crypto._olmDevice,
+                    sender,
+                    this._baseApis.getStoredDevice(sender, deviceId),
+                    payload,
+                );
+                const contentMap = {
+                    [sender]: {
+                        [deviceId]: encryptedContent,
+                    },
+                };
+
+                logger.info(`Sending ${content.name} secret for ${deviceId}`);
+                this._baseApis.sendToDevice("m.room.encrypted", contentMap);
+            } else {
+                logger.info(`Request denied for ${content.name} secret for ${deviceId}`);
+            }
+        }
+    }
+
+    _onSecretReceived(event) {
+        if (event.getSender() !== this._baseApis.getUserId()) {
+            // we shouldn't be receiving secrets from anyone else, so ignore
+            // because someone could be trying to send us bogus data
+            return;
+        }
+        const content = event.getContent();
+        logger.log("got secret share for request", content.request_id);
+        const requestControl = this._requests[content.request_id];
+        if (requestControl) {
+            // make sure that the device that sent it is one of the devices that
+            // we requested from
+            const deviceInfo = this._baseApis._crypto._deviceList.getDeviceByIdentityKey(
+                olmlib.OLM_ALGORITHM,
+                event.getSenderKey(),
+            );
+            if (!deviceInfo) {
+                logger.log(
+                    "secret share from unknown device with key", event.getSenderKey(),
+                );
+                return;
+            }
+            if (!requestControl.devices.includes(deviceInfo.deviceId)) {
+                logger.log("unsolicited secret share from device", deviceInfo.deviceId);
+                return;
+            }
+
+            logger.log(
+                `Successfully received secret ${requestControl.name} ` +
+                `from ${deviceInfo.deviceId}`,
+            );
+            requestControl.resolve(content.secret);
+        }
+    }
+
+    async _getSecretStorageKey(keys, name) {
+        if (!this._cryptoCallbacks.getSecretStorageKey) {
+            throw new Error("No getSecretStorageKey callback supplied");
+        }
+
+        const returned = await this._cryptoCallbacks.getSecretStorageKey({ keys }, name);
+
+        if (!returned) {
+            throw new Error("getSecretStorageKey callback returned falsey");
+        }
+        if (returned.length < 2) {
+            throw new Error("getSecretStorageKey callback returned invalid data");
+        }
+
+        const [keyId, privateKey] = returned;
+        if (!keys[keyId]) {
+            throw new Error("App returned unknown key from getSecretStorageKey!");
+        }
+
+        if (keys[keyId].algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            const decryption = {
+                encrypt: async function(secret) {
+                    return await encryptAES(secret, privateKey, name);
+                },
+                decrypt: async function(encInfo) {
+                    return await decryptAES(encInfo, privateKey, name);
+                },
+            };
+            return [keyId, decryption];
+        } else {
+            throw new Error("Unknown key type: " + keys[keyId].algorithm);
+        }
+    }
+}

src/crypto/aes.js

@@ -0,0 +1,251 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {getCrypto} from '../utils';
+import {decodeBase64, encodeBase64} from './olmlib';
+
+const subtleCrypto = (typeof window !== "undefined" && window.crypto) ?
+    (window.crypto.subtle || window.crypto.webkitSubtle) : null;
+
+// salt for HKDF, with 8 bytes of zeros
+const zerosalt = new Uint8Array(8);
+
+/**
+ * encrypt a string in Node.js
+ *
+ * @param {string} data the plaintext to encrypt
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
+ */
+async function encryptNode(data, key, name, ivStr) {
+    const crypto = getCrypto();
+    if (!crypto) {
+        throw new Error("No usable crypto implementation");
+    }
+
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = crypto.randomBytes(16);
+    }
+
+    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
+    // (which would mean we wouldn't be able to decrypt on Android). The loss
+    // of a single bit of iv is a price we have to pay.
+    iv[8] &= 0x7f;
+
+    const [aesKey, hmacKey] = deriveKeysNode(key, name);
+
+    const cipher = crypto.createCipheriv("aes-256-ctr", aesKey, iv);
+    const ciphertext = cipher.update(data, "utf-8", "base64")
+          + cipher.final("base64");
+
+    const hmac = crypto.createHmac("sha256", hmacKey)
+        .update(ciphertext, "base64").digest("base64");
+
+    return {
+        iv: encodeBase64(iv),
+        ciphertext: ciphertext,
+        mac: hmac,
+    };
+}
+
+/**
+ * decrypt a string in Node.js
+ *
+ * @param {object} data the encrypted data
+ * @param {string} data.ciphertext the ciphertext in base64
+ * @param {string} data.iv the initialization vector in base64
+ * @param {string} data.mac the HMAC in base64
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function decryptNode(data, key, name) {
+    const crypto = getCrypto();
+    if (!crypto) {
+        throw new Error("No usable crypto implementation");
+    }
+
+    const [aesKey, hmacKey] = deriveKeysNode(key, name);
+
+    const hmac = crypto.createHmac("sha256", hmacKey)
+        .update(data.ciphertext, "base64").digest("base64").replace(/=+$/g, '');
+
+    if (hmac !== data.mac.replace(/=+$/g, '')) {
+        throw new Error(`Error decrypting secret ${name}: bad MAC`);
+    }
+
+    const decipher = crypto.createDecipheriv(
+        "aes-256-ctr", aesKey, decodeBase64(data.iv),
+    );
+    return decipher.update(data.ciphertext, "base64", "utf-8")
+          + decipher.final("utf-8");
+}
+
+function deriveKeysNode(key, name) {
+    const crypto = getCrypto();
+    const prk = crypto.createHmac("sha256", zerosalt)
+        .update(key).digest();
+
+    const b = Buffer.alloc(1, 1);
+    const aesKey = crypto.createHmac("sha256", prk)
+        .update(name, "utf-8").update(b).digest();
+    b[0] = 2;
+    const hmacKey = crypto.createHmac("sha256", prk)
+        .update(aesKey).update(name, "utf-8").update(b).digest();
+
+    return [aesKey, hmacKey];
+}
+
+/**
+ * encrypt a string in Node.js
+ *
+ * @param {string} data the plaintext to encrypt
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
+ */
+async function encryptBrowser(data, key, name, ivStr) {
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = new Uint8Array(16);
+        window.crypto.getRandomValues(iv);
+    }
+
+    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
+    // (which would mean we wouldn't be able to decrypt on Android). The loss
+    // of a single bit of iv is a price we have to pay.
+    iv[8] &= 0x7f;
+
+    const [aesKey, hmacKey] = await deriveKeysBrowser(key, name);
+    const encodedData = new TextEncoder().encode(data);
+
+    const ciphertext = await subtleCrypto.encrypt(
+        {
+            name: "AES-CTR",
+            counter: iv,
+            length: 64,
+        },
+        aesKey,
+        encodedData,
+    );
+
+    const hmac = await subtleCrypto.sign(
+        {name: 'HMAC'},
+        hmacKey,
+        ciphertext,
+    );
+
+    return {
+        iv: encodeBase64(iv),
+        ciphertext: encodeBase64(ciphertext),
+        mac: encodeBase64(hmac),
+    };
+}
+
+/**
+ * decrypt a string in the browser
+ *
+ * @param {object} data the encrypted data
+ * @param {string} data.ciphertext the ciphertext in base64
+ * @param {string} data.iv the initialization vector in base64
+ * @param {string} data.mac the HMAC in base64
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function decryptBrowser(data, key, name) {
+    const [aesKey, hmacKey] = await deriveKeysBrowser(key, name);
+
+    const ciphertext = decodeBase64(data.ciphertext);
+
+    if (!await subtleCrypto.verify(
+        {name: "HMAC"},
+        hmacKey,
+        decodeBase64(data.mac),
+        ciphertext,
+    )) {
+        throw new Error(`Error decrypting secret ${name}: bad MAC`);
+    }
+
+    const plaintext = await subtleCrypto.decrypt(
+        {
+            name: "AES-CTR",
+            counter: decodeBase64(data.iv),
+            length: 64,
+        },
+        aesKey,
+        ciphertext,
+    );
+
+    return new TextDecoder().decode(new Uint8Array(plaintext));
+}
+
+async function deriveKeysBrowser(key, name) {
+    const hkdfkey = await subtleCrypto.importKey(
+        'raw',
+        key,
+        {name: "HKDF"},
+        false,
+        ["deriveBits"],
+    );
+    const keybits = await subtleCrypto.deriveBits(
+        {
+            name: "HKDF",
+            salt: zerosalt,
+            info: (new TextEncoder().encode(name)),
+            hash: "SHA-256",
+        },
+        hkdfkey,
+        512,
+    );
+
+    const aesKey = keybits.slice(0, 32);
+    const hmacKey = keybits.slice(32);
+
+    const aesProm = subtleCrypto.importKey(
+        'raw',
+        aesKey,
+        {name: 'AES-CTR'},
+        false,
+        ['encrypt', 'decrypt'],
+    );
+
+    const hmacProm = subtleCrypto.importKey(
+        'raw',
+        hmacKey,
+        {
+            name: 'HMAC',
+            hash: {name: 'SHA-256'},
+        },
+        false,
+        ['sign', 'verify'],
+    );
+
+    return await Promise.all([aesProm, hmacProm]);
+}
+
+export function encryptAES(...args) {
+    return subtleCrypto ? encryptBrowser(...args) : encryptNode(...args);
+}
+
+export function decryptAES(...args) {
+    return subtleCrypto ? decryptBrowser(...args) : decryptNode(...args);
+}
+

src/crypto/algorithms/base.js

@@ -20,8 +20,6 @@ limitations under the License.
  * @module
  */
 
-import Promise from 'bluebird';
-
 /**
  * map of registered encryption algorithm classes. A map from string to {@link
  * module:crypto/algorithms/base.EncryptionAlgorithm|EncryptionAlgorithm} class
@@ -52,7 +50,7 @@ export const DECRYPTION_CLASSES = {};
  * @param {string} params.roomId  The ID of the room we will be sending to
  * @param {object} params.config  The body of the m.room.encryption event
  */
-class EncryptionAlgorithm {
+export class EncryptionAlgorithm {
     constructor(params) {
         this._userId = params.userId;
         this._deviceId = params.deviceId;
@@ -62,6 +60,15 @@ class EncryptionAlgorithm {
         this._roomId = params.roomId;
     }
 
+    /**
+     * Perform any background tasks that can be done before a message is ready to
+     * send, in order to speed up sending of the message.
+     *
+     * @param {module:models/room} room the room the event is in
+     */
+    prepareToEncrypt(room) {
+    }
+
     /**
      * Encrypt a message event
      *
@@ -72,7 +79,7 @@ class EncryptionAlgorithm {
      * @param {string} eventType
      * @param {object} plaintext event content
      *
-     * @return {module:client.Promise} Promise which resolves to the new event body
+     * @return {Promise} Promise which resolves to the new event body
      */
 
     /**
@@ -86,7 +93,6 @@ class EncryptionAlgorithm {
      onRoomMembership(event, member, oldMembership) {
      }
 }
-export {EncryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 /**
  * base type for decryption implementations
@@ -100,7 +106,7 @@ export {EncryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
  * @param {string=} params.roomId The ID of the room we will be receiving
  *     from. Null for to-device events.
  */
-class DecryptionAlgorithm {
+export class DecryptionAlgorithm {
     constructor(params) {
         this._userId = params.userId;
         this._crypto = params.crypto;
@@ -161,8 +167,17 @@ class DecryptionAlgorithm {
     shareKeysWithDevice(keyRequest) {
         throw new Error("shareKeysWithDevice not supported for this DecryptionAlgorithm");
     }
+
+    /**
+     * Retry decrypting all the events from a sender that haven't been
+     * decrypted yet.
+     *
+     * @param {string} senderKey the sender's key
+     */
+    async retryDecryptionFromSender(senderKey) {
+        // ignore by default
+    }
 }
-export {DecryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 /**
  * Exception thrown when decryption fails
@@ -175,7 +190,7 @@ export {DecryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
  *
  * @extends Error
  */
-class DecryptionError extends Error {
+export class DecryptionError extends Error {
     constructor(code, msg, details) {
         super(msg);
         this.code = code;
@@ -183,7 +198,6 @@ class DecryptionError extends Error {
         this.detailedString = _detailedStringForDecryptionError(this, details);
     }
 }
-export {DecryptionError}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 function _detailedStringForDecryptionError(err, details) {
     let result = err.name + '[msg: ' + err.message;

src/crypto/algorithms/index.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2016 OpenMarket Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,28 +14,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
 
 /**
  * @module crypto/algorithms
  */
 
-const base = require("./base");
+import "./olm";
+import "./megolm";
 
-require("./olm");
-require("./megolm");
-
-/**
- * @see module:crypto/algorithms/base.ENCRYPTION_CLASSES
- */
-module.exports.ENCRYPTION_CLASSES = base.ENCRYPTION_CLASSES;
-
-/**
- * @see module:crypto/algorithms/base.DECRYPTION_CLASSES
- */
-module.exports.DECRYPTION_CLASSES = base.DECRYPTION_CLASSES;
-
-/**
- * @see module:crypto/algorithms/base.DecryptionError
- */
-module.exports.DecryptionError = base.DecryptionError;
+export * from "./base";