v12.0.0

Rework how disambiguation is handled

.eslintrc.js

@@ -1,71 +1,16 @@
 module.exports = {
-    parser: "babel-eslint", // now needed for class properties
-    parserOptions: {
-        sourceType: "module",
-        ecmaFeatures: {
-        }
-    },
+    plugins: [
+        "matrix-org",
+    ],
+    extends: [
+        "plugin:matrix-org/babel",
+    ],
     env: {
         browser: true,
         node: true,
-
-        // babel's transform-runtime converts references to ES6 globals such as
-        // Promise and Map to core-js polyfills, so we can use ES6 globals.
-        es6: true,
-        jest: true,
     },
-    extends: ["eslint:recommended", "google"],
-    plugins: [
-        "babel",
-        "jest",
-    ],
     rules: {
-        // rules we've always adhered to or now do
-        "max-len": ["error", {
-            code: 90,
-            ignoreComments: true,
-        }],
-        curly: ["error", "multi-line"],
-        "prefer-const": ["error"],
-        "comma-dangle": ["error", {
-            arrays: "always-multiline",
-            objects: "always-multiline",
-            imports: "always-multiline",
-            exports: "always-multiline",
-            functions: "always-multiline",
-        }],
-
-        // loosen jsdoc requirements a little
-        "require-jsdoc": ["error", {
-            require: {
-                FunctionDeclaration: false,
-            }
-        }],
-        "valid-jsdoc": ["error", {
-            requireParamDescription: false,
-            requireReturn: false,
-            requireReturnDescription: false,
-        }],
-
-        // rules we do not want from eslint-recommended
-        "no-console": ["off"],
-        "no-constant-condition": ["off"],
-        "no-empty": ["error", { "allowEmptyCatch": true }],
-
-        // rules we do not want from the google styleguide
-        "object-curly-spacing": ["off"],
-        "spaced-comment": ["off"],
-        "guard-for-in": ["off"],
-
-        // in principle we prefer single quotes, but life is too short
-        quotes: ["off"],
-
-        // rules we'd ideally like to adhere to, but the current
-        // code does not (in most cases because it's still ES5)
-        // we set these to warnings, and assert that the number
-        // of warnings doesn't exceed a given threshold
         "no-var": ["warn"],
-        "brace-style": ["warn", "1tbs", {"allowSingleLine": true}],
         "prefer-rest-params": ["warn"],
         "prefer-spread": ["warn"],
         "one-var": ["warn"],
@@ -79,10 +24,35 @@ module.exports = {
             "asyncArrow": "always",
         }],
         "arrow-parens": "off",
+        "prefer-promise-reject-errors": "off",
+        "quotes": "off",
+        "indent": "off",
+        "no-constant-condition": "off",
+        "no-async-promise-executor": "off",
+        // We use a `logger` intermediary module
+        "no-console": "error",
+    },
+    overrides: [{
+        files: [
+            "**/*.ts",
+        ],
+        extends: [
+            "plugin:matrix-org/typescript",
+        ],
+        rules: {
+            // TypeScript has its own version of this
+            "@babel/no-invalid-this": "off",
 
-        // eslint's built in no-invalid-this rule breaks with class properties
-        "no-invalid-this": "off",
-        // so we replace it with a version that is class property aware
-        "babel/no-invalid-this": "error",
-    }
-}
+            // We're okay being explicit at the moment
+            "@typescript-eslint/no-empty-interface": "off",
+            // We disable this while we're transitioning
+            "@typescript-eslint/no-explicit-any": "off",
+            // We'd rather not do this but we do
+            "@typescript-eslint/ban-ts-comment": "off",
+
+            "quotes": "off",
+            // We use a `logger` intermediary module
+            "no-console": "error",
+        },
+    }],
+};

.gitignore

@@ -12,7 +12,6 @@ lib-cov
 out
 /dist
 /lib
-/specbuild
 
 # version file and tarball created by `npm pack` / `yarn pack`
 /git-revision.txt

CONTRIBUTING.rst

@@ -28,10 +28,9 @@ use GitHub's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use Travis for continuous integration, and all pull requests get
-automatically tested by Travis: if your change breaks the build, then the PR
-will show that there are failed checks, so please check back after a few
-minutes.
+We use continuous integration, and all pull requests get automatically tested:
+if your change breaks the build, then the PR will show that there are failed
+checks, so please check back after a few minutes.
 
 Code style
 ~~~~~~~~~~

README.md

@@ -30,7 +30,7 @@ This SDK targets Node 10 for compatibility, which translates to ES6. If you're u
 a bundler like webpack you'll likely have to transpile dependencies, including this
 SDK, to match your target browsers.
 
-Using `yarn` instead of `npm` is recommended. Please see the Yarn [install guide](https://yarnpkg.com/docs/install/) 
+Using `yarn` instead of `npm` is recommended. Please see the Yarn [install guide](https://classic.yarnpkg.com/en/docs/install) 
 if you do not have it already.
 
 ``yarn add matrix-js-sdk``
@@ -182,10 +182,8 @@ you can pass the result of the promise into it with something like:
   matrixClient.someMethod(arg1, arg2).nodeify(callback);
 ```
 
-The main thing to note is that it is an error to discard the result of a
-promise-returning function, as that will cause exceptions to go unobserved. If
-you have nothing better to do with the result, just call ``.done()`` on it. See
-http://documentup.com/kriskowal/q/#the-end for more information.
+The main thing to note is that it is problematic to discard the result of a
+promise-returning function, as that will cause exceptions to go unobserved.
 
 Methods which return a promise show this in their documentation.
 
@@ -309,7 +307,7 @@ The SDK supports end-to-end encryption via the Olm and Megolm protocols, using
 [libolm](https://gitlab.matrix.org/matrix-org/olm). It is left up to the
 application to make libolm available, via the ``Olm`` global.
 
-It is also necessry to call ``matrixClient.initCrypto()`` after creating a new
+It is also necessary to call ``matrixClient.initCrypto()`` after creating a new
 ``MatrixClient`` (but **before** calling ``matrixClient.startClient()``) to
 initialise the crypto layer.
 

examples/node/app.js

@@ -288,7 +288,7 @@ function printMemberList(room) {
 }
 
 function printRoomInfo(room) {
-    var eventDict = room.currentState.events;
+    var eventMap = room.currentState.events;
     var eTypeHeader = "    Event Type(state_key)    ";
     var sendHeader = "        Sender        ";
     // pad content to 100
@@ -300,14 +300,15 @@ function printRoomInfo(room) {
     var contentHeader = padSide + "Content" + padSide;
     print(eTypeHeader+sendHeader+contentHeader);
     print(new Array(100).join("-"));
-    Object.keys(eventDict).forEach(function(eventType) {
+    eventMap.keys().forEach(function(eventType) {
         if (eventType === "m.room.member") { return; } // use /members instead.
-        Object.keys(eventDict[eventType]).forEach(function(stateKey) {
+        var eventEventMap = eventMap.get(eventType);
+        eventEventMap.keys().forEach(function(stateKey) {
             var typeAndKey = eventType + (
                 stateKey.length > 0 ? "("+stateKey+")" : ""
             );
             var typeStr = fixWidth(typeAndKey, eTypeHeader.length);
-            var event = eventDict[eventType][stateKey];
+            var event = eventEventMap.get(stateKey);
             var sendStr = fixWidth(event.getSender(), sendHeader.length);
             var contentStr = fixWidth(
                 JSON.stringify(event.getContent()), contentHeader.length

examples/voip/browserTest.js

@@ -31,6 +31,23 @@ function addListeners(call) {
         call.hangup();
         disableButtons(false, true, true);
     });
+    call.on("feeds_changed", function(feeds) {
+        const localFeed = feeds.find((feed) => feed.isLocal());
+        const remoteFeed = feeds.find((feed) => !feed.isLocal());
+
+        const remoteElement = document.getElementById("remote");
+        const localElement = document.getElementById("local");
+
+        if (remoteFeed) {
+            remoteElement.srcObject = remoteFeed.stream;
+            remoteElement.play();
+        }
+        if (localFeed) {
+            localElement.muted = true;
+            localElement.srcObject = localFeed.stream;
+            localElement.play();
+        }
+    });
 }
 
 window.onload = function() {
@@ -62,10 +79,7 @@ function syncComplete() {
         );
         console.log("Call => %s", call);
         addListeners(call);
-        call.placeVideoCall(
-            document.getElementById("remote"),
-            document.getElementById("local")
-        );
+        call.placeVideoCall();
         document.getElementById("result").innerHTML = "<p>Placed call.</p>";
         disableButtons(true, true, false);
     };

examples/voip/index.html

@@ -1,26 +1,34 @@
 <html>
+
 <head>
-<title>VoIP Test</title>
-<script src="lib/matrix.js"></script>
-<script src="browserTest.js"></script>
+    <title>VoIP Test</title>
+    <script src="lib/matrix.js"></script>
+    <script src="browserTest.js"></script>
 </head>
+
 <body>
-	You can place and receive calls with this example. Make sure to edit the
+    You can place and receive calls with this example. Make sure to edit the
     constants in <code>browserTest.js</code> first.
     <div id="config"></div>
     <div id="result"></div>
     <button id="call">Place Call</button>
     <button id="answer">Answer Call</button>
     <button id="hangup">Hangup Call</button>
-    <div id="videoBackground">
-        <div id="videoContainer">
-            <video id="remote"></video>
-        </div>
-    </div>
-    <div id="videoBackground">
-        <div id="videoContainer">
-            <video id="local"></video>
-        </div>
+    <div id="videoBackground" class="video-background">
+        <video class="video-element" id="local"></video>
+        <video class="video-element" id="remote"></video>
     </div>
 </body>
+
 </html>
+
+<style>
+    .video-background {
+        height: 500px;
+        margin: 10px;
+    }
+
+    .video-element {
+        height: 100%;
+    }
+</style>

jsdoc.json

@@ -18,6 +18,13 @@
     "readme": "README.md",
     "recurse": true,
     "verbose": true,
-    "template": "node_modules/better-docs"
+    "template": "node_modules/docdash"
+  },
+  "docdash": {
+    "static": true,
+    "private": false,
+    "search": true,
+    "collapse": true,
+    "typedefs": true
   }
 }

package.json

@@ -1,24 +1,25 @@
 {
   "name": "matrix-js-sdk",
-  "version": "5.1.1",
+  "version": "12.0.0",
   "description": "Matrix Client-Server SDK for Javascript",
   "scripts": {
-    "prepare": "yarn build",
+    "prepublishOnly": "yarn build",
     "start": "echo THIS IS FOR LEGACY PURPOSES ONLY. && babel src -w -s -d lib --verbose --extensions \".ts,.js\"",
     "dist": "echo 'This is for the release script so it can make assets (browser bundle).' && yarn build",
     "clean": "rimraf lib dist",
-    "build": "yarn clean && git rev-parse HEAD > git-revision.txt && yarn build:compile && yarn build:compile-browser && yarn build:minify-browser && yarn build:types",
+    "build": "yarn build:dev && yarn build:compile-browser && yarn build:minify-browser",
+    "build:dev": "yarn clean && git rev-parse HEAD > git-revision.txt && yarn build:compile && yarn build:types",
     "build:types": "tsc --emitDeclarationOnly",
     "build:compile": "babel -d lib --verbose --extensions \".ts,.js\" src",
     "build:compile-browser": "mkdirp dist && browserify -d src/browser-index.js -p [ tsify -p ./tsconfig.json ] -t [ babelify --sourceMaps=inline --presets [ @babel/preset-env @babel/preset-typescript ] ] | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js",
     "build:minify-browser": "terser dist/browser-matrix.js --compress --mangle --source-map --output dist/browser-matrix.min.js",
     "gendoc": "jsdoc -c jsdoc.json -P package.json",
-    "lint": "yarn lint:types && yarn lint:ts && yarn lint:js",
-    "lint:js": "eslint --max-warnings 93 src spec",
+    "lint": "yarn lint:types && yarn lint:js",
+    "lint:js": "eslint --max-warnings 57 src spec",
     "lint:types": "tsc --noEmit",
-    "lint:ts": "tslint --project ./tsconfig.json -t stylish",
-    "test": "jest spec/ --coverage --testEnvironment node",
-    "test:watch": "jest spec/ --coverage --testEnvironment node --watch"
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "coverage": "yarn test --coverage"
   },
   "repository": {
     "type": "git",
@@ -28,13 +29,15 @@
     "matrix-org"
   ],
   "main": "./lib/index.js",
-  "typings": "./lib/index.d.ts",
   "browser": "./lib/browser-index.js",
   "matrix_src_main": "./src/index.ts",
   "matrix_src_browser": "./src/browser-index.js",
+  "matrix_lib_main": "./lib/index.js",
+  "matrix_lib_typings": "./lib/index.d.ts",
   "author": "matrix.org",
   "license": "Apache-2.0",
   "files": [
+    "dist",
     "lib",
     "src",
     "git-revision.txt",
@@ -46,51 +49,65 @@
     "release.sh"
   ],
   "dependencies": {
-    "@babel/runtime": "^7.8.3",
+    "@babel/runtime": "^7.12.5",
     "another-json": "^0.2.0",
     "browser-request": "^0.3.3",
     "bs58": "^4.0.1",
-    "content-type": "^1.0.2",
-    "loglevel": "^1.6.4",
-    "qs": "^6.5.2",
-    "request": "^2.88.0",
-    "unhomoglyph": "^1.0.2"
+    "content-type": "^1.0.4",
+    "loglevel": "^1.7.1",
+    "qs": "^6.9.6",
+    "request": "^2.88.2",
+    "unhomoglyph": "^1.0.6"
   },
   "devDependencies": {
-    "@babel/cli": "^7.7.5",
-    "@babel/core": "^7.7.5",
-    "@babel/plugin-proposal-class-properties": "^7.7.4",
-    "@babel/plugin-proposal-numeric-separator": "^7.7.4",
-    "@babel/plugin-proposal-object-rest-spread": "^7.7.4",
-    "@babel/plugin-syntax-dynamic-import": "^7.7.4",
-    "@babel/plugin-transform-runtime": "^7.8.3",
-    "@babel/preset-env": "^7.7.6",
-    "@babel/preset-typescript": "^7.7.4",
-    "@babel/register": "^7.7.4",
+    "@babel/cli": "^7.12.10",
+    "@babel/core": "^7.12.10",
+    "@babel/eslint-parser": "^7.12.10",
+    "@babel/eslint-plugin": "^7.12.10",
+    "@babel/plugin-proposal-class-properties": "^7.12.1",
+    "@babel/plugin-proposal-numeric-separator": "^7.12.7",
+    "@babel/plugin-proposal-object-rest-spread": "^7.12.1",
+    "@babel/plugin-syntax-dynamic-import": "^7.8.3",
+    "@babel/plugin-transform-runtime": "^7.12.10",
+    "@babel/preset-env": "^7.12.11",
+    "@babel/preset-typescript": "^7.12.7",
+    "@babel/register": "^7.12.10",
+    "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.3.tgz",
+    "@types/jest": "^26.0.20",
     "@types/node": "12",
-    "babel-eslint": "^10.0.3",
-    "babel-jest": "^24.9.0",
+    "@types/request": "^2.48.5",
+    "@typescript-eslint/eslint-plugin": "^4.17.0",
+    "@typescript-eslint/parser": "^4.17.0",
+    "babel-jest": "^26.6.3",
     "babelify": "^10.0.0",
-    "better-docs": "^1.4.7",
-    "browserify": "^16.5.0",
-    "eslint": "^5.12.0",
-    "eslint-config-google": "^0.7.1",
-    "eslint-plugin-babel": "^5.3.0",
-    "eslint-plugin-jest": "^23.0.4",
+    "better-docs": "^2.3.2",
+    "browserify": "^17.0.0",
+    "docdash": "^1.2.0",
+    "eslint": "7.18.0",
+    "eslint-config-google": "^0.14.0",
+    "eslint-plugin-matrix-org": "github:matrix-org/eslint-plugin-matrix-org#main",
     "exorcist": "^1.0.1",
-    "fake-indexeddb": "^3.0.0",
-    "jest": "^24.9.0",
-    "jest-localstorage-mock": "^2.4.0",
-    "jsdoc": "^3.5.5",
+    "fake-indexeddb": "^3.1.2",
+    "jest": "^26.6.3",
+    "jest-localstorage-mock": "^2.4.6",
+    "jsdoc": "^3.6.6",
     "matrix-mock-request": "^1.2.3",
-    "olm": "https://packages.matrix.org/npm/olm/olm-3.1.4.tgz",
-    "rimraf": "^3.0.0",
-    "terser": "^4.4.3",
-    "tsify": "^4.0.1",
-    "tslint": "^5.20.1",
-    "typescript": "^3.7.3"
+    "rimraf": "^3.0.2",
+    "terser": "^5.5.1",
+    "tsify": "^5.0.2",
+    "typescript": "^4.1.3"
   },
   "jest": {
-    "testEnvironment": "node"
-  }
+    "testEnvironment": "node",
+    "testMatch": [
+      "<rootDir>/spec/**/*.spec.{js,ts}"
+    ],
+    "collectCoverageFrom": [
+      "<rootDir>/src/**/*.{js,ts}"
+    ],
+    "coverageReporters": [
+      "text"
+    ]
+  },
+  "typings": "./lib/index.d.ts"
 }

release.sh

@@ -10,7 +10,7 @@
 #   npm; typically installed by Node.js
 #   yarn; install via brew (macOS) or similar (https://yarnpkg.com/docs/install/)
 #
-# Note: this script is also used to release matrix-react-sdk and riot-web.
+# Note: this script is also used to release matrix-react-sdk and element-web.
 
 set -e
 
@@ -38,6 +38,7 @@ $USAGE
     -c changelog_file:  specify name of file containing changelog
     -x:                 skip updating the changelog
     -z:                 skip generating the jsdoc
+    -n:                 skip publish to NPM
 EOF
 }
 
@@ -60,9 +61,10 @@ fi
 
 skip_changelog=
 skip_jsdoc=
+skip_npm=
 changelog_file="CHANGELOG.md"
 expected_npm_user="matrixdotorg"
-while getopts hc:u:xz f; do
+while getopts hc:u:xzn f; do
     case $f in
         h)
             help
@@ -77,6 +79,9 @@ while getopts hc:u:xz f; do
         z)
             skip_jsdoc=1
             ;;
+        n)
+            skip_npm=1
+            ;;
         u)
             expected_npm_user="$OPTARG"
             ;;
@@ -89,6 +94,14 @@ if [ $# -ne 1 ]; then
     exit 1
 fi
 
+# We use Git branch / commit dependencies for some packages, and Yarn seems
+# to have a hard time getting that right. See also
+# https://github.com/yarnpkg/yarn/issues/4734. As a workaround, we clean the
+# global cache here to ensure we get the right thing.
+yarn cache clean
+# Ensure all dependencies are updated
+yarn install --ignore-scripts
+
 if [ -z "$skip_changelog" ]; then
     # update_changelog doesn't have a --version flag
     update_changelog -h > /dev/null || (echo "github-changelog-generator is required: please install it"; exit)
@@ -96,10 +109,12 @@ fi
 
 # Login and publish continues to use `npm`, as it seems to have more clearly
 # defined options and semantics than `yarn` for writing to the registry.
-actual_npm_user=`npm whoami`;
-if [ $expected_npm_user != $actual_npm_user ]; then
-    echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
-    exit 1
+if [ -z "$skip_npm" ]; then
+    actual_npm_user=`npm whoami`;
+    if [ $expected_npm_user != $actual_npm_user ]; then
+        echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
+        exit 1
+    fi
 fi
 
 # ignore leading v on release
@@ -163,6 +178,19 @@ echo "yarn version"
 # manually commit the result.
 yarn version --no-git-tag-version --new-version "$release"
 
+# For the published and dist versions of the package, we copy the
+# `matrix_lib_main` and `matrix_lib_typings` fields to `main` and `typings` (if
+# they exist). This small bit of gymnastics allows us to use the TypeScript
+# source directly for development without needing to build before linting or
+# testing.
+for i in main typings
+do
+    lib_value=$(jq -r ".matrix_lib_$i" package.json)
+    if [ "$lib_value" != "null" ]; then
+        jq ".$i = .matrix_lib_$i" package.json > package.json.new && mv package.json.new package.json
+    fi
+done
+
 # commit yarn.lock if it exists, is versioned, and is modified
 if [[ -f yarn.lock && `git status --porcelain yarn.lock | grep '^ M'` ]];
 then
@@ -197,11 +225,6 @@ if [ $dodist -eq 0 ]; then
     pushd "$builddir"
     git clone "$projdir" .
     git checkout "$rel_branch"
-    # We use Git branch / commit dependencies for some packages, and Yarn seems
-    # to have a hard time getting that right. See also
-    # https://github.com/yarnpkg/yarn/issues/4734. As a workaround, we clean the
-    # global cache here to ensure we get the right thing.
-    yarn cache clean
     yarn install
     # We haven't tagged yet, so tell the dist script what version
     # it's building
@@ -298,11 +321,13 @@ rm "${latest_changes}"
 # defined options and semantics than `yarn` for writing to the registry.
 # Tag both releases and prereleases as `next` so the last stable release remains
 # the default.
-npm publish --tag next
-if [ $prerelease -eq 0 ]; then
-    # For a release, also add the default `latest` tag.
-    package=$(cat package.json | jq -er .name)
-    npm dist-tag add "$package@$release" latest
+if [ -z "$skip_npm" ]; then
+    npm publish --tag next
+    if [ $prerelease -eq 0 ]; then
+        # For a release, also add the default `latest` tag.
+        package=$(cat package.json | jq -er .name)
+        npm dist-tag add "$package@$release" latest
+    fi
 fi
 
 if [ -z "$skip_jsdoc" ]; then
@@ -318,6 +343,7 @@ if [ -z "$skip_jsdoc" ]; then
         $release index.html
     git add "$release"
     git commit --no-verify -m "Add jsdoc for $release" index.html "$release"
+    git push origin gh-pages
 fi
 
 # if it is a pre-release, leave it on the release branch for now.
@@ -330,16 +356,40 @@ fi
 echo "updating master branch"
 git checkout master
 git pull
-git merge "$rel_branch"
+git merge "$rel_branch" --no-edit
 
-# push master and docs (if generated) to github
+# push master to github
 git push origin master
-if [ -z "$skip_jsdoc" ]; then
-    git push origin gh-pages
-fi
 
-# finally, merge master back onto develop
-git checkout develop
-git pull
-git merge master
-git push origin develop
+# finally, merge master back onto develop (if it exists)
+if [ $(git branch -lr | grep origin/develop -c) -ge 1 ]; then
+    git checkout develop
+    git pull
+    git merge master --no-edit
+
+    # When merging to develop, we need revert the `main` and `typings` fields if
+    # we adjusted them previously.
+    for i in main typings
+    do
+        # If a `lib` prefixed value is present, it means we adjusted the field
+        # earlier at publish time, so we should revert it now.
+        if [ "$(jq -r ".matrix_lib_$i" package.json)" != "null" ]; then
+            # If there's a `src` prefixed value, use that, otherwise delete.
+            # This is used to delete the `typings` field and reset `main` back
+            # to the TypeScript source.
+            src_value=$(jq -r ".matrix_src_$i" package.json)
+            if [ "$src_value" != "null" ]; then
+                jq ".$i = .matrix_src_$i" package.json > package.json.new && mv package.json.new package.json
+            else
+                jq "del(.$i)" package.json > package.json.new && mv package.json.new package.json
+            fi
+        fi
+    done
+
+    if [ -n "$(git ls-files --modified package.json)" ]; then
+        echo "Committing develop package.json"
+        git commit package.json -m "Resetting package fields for development"
+    fi
+
+    git push origin develop
+fi

spec/MockBlob.ts

@@ -0,0 +1,27 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export class MockBlob {
+    private contents: number[] = [];
+
+    public constructor(private parts: ArrayLike<number>[]) {
+        parts.forEach(p => Array.from(p).forEach(e => this.contents.push(e)));
+    }
+
+    public get size(): number {
+        return this.contents.length;
+    }
+}

spec/TestClient.js

@@ -20,12 +20,12 @@ limitations under the License.
 import './olm-loader';
 
 import MockHttpBackend from 'matrix-mock-request';
-import {LocalStorageCryptoStore} from '../src/crypto/store/localStorage-crypto-store';
-import {logger} from '../src/logger';
-import {WebStorageSessionStore} from "../src/store/session/webstorage";
-import {syncPromise} from "./test-utils";
-import {createClient} from "../src/matrix";
-import {MockStorageApi} from "./MockStorageApi";
+import { LocalStorageCryptoStore } from '../src/crypto/store/localStorage-crypto-store';
+import { logger } from '../src/logger';
+import { WebStorageSessionStore } from "../src/store/session/webstorage";
+import { syncPromise } from "./test-utils";
+import { createClient } from "../src/matrix";
+import { MockStorageApi } from "./MockStorageApi";
 
 /**
  * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
@@ -69,6 +69,9 @@ export function TestClient(
 
     this.deviceKeys = null;
     this.oneTimeKeys = {};
+    this.callEventHandler = {
+        calls: new Map(),
+    };
 }
 
 TestClient.prototype.toString = function() {
@@ -126,11 +129,10 @@ TestClient.prototype.expectDeviceKeyUpload = function() {
         expect(Object.keys(self.oneTimeKeys).length).toEqual(0);
 
         self.deviceKeys = content.device_keys;
-        return {one_time_key_counts: {signed_curve25519: 0}};
+        return { one_time_key_counts: { signed_curve25519: 0 } };
     });
 };
 
-
 /**
  * If one-time keys have already been uploaded, return them. Otherwise,
  * set up an expectation that the keys will be uploaded, and wait for
@@ -148,9 +150,9 @@ TestClient.prototype.awaitOneTimeKeyUpload = function() {
         .respond(200, (path, content) => {
             expect(content.device_keys).toBe(undefined);
             expect(content.one_time_keys).toBe(undefined);
-            return {one_time_key_counts: {
+            return { one_time_key_counts: {
                 signed_curve25519: Object.keys(this.oneTimeKeys).length,
-            }};
+            } };
         });
 
     this.httpBackend.when("POST", "/keys/upload")
@@ -161,9 +163,9 @@ TestClient.prototype.awaitOneTimeKeyUpload = function() {
               logger.log('%s: received %i one-time keys', this,
                           Object.keys(content.one_time_keys).length);
               this.oneTimeKeys = content.one_time_keys;
-              return {one_time_key_counts: {
+              return { one_time_key_counts: {
                   signed_curve25519: Object.keys(this.oneTimeKeys).length,
-              }};
+              } };
           });
 
     // this can take ages
@@ -185,7 +187,7 @@ TestClient.prototype.expectKeyQuery = function(response) {
         200, (path, content) => {
             Object.keys(response.device_keys).forEach((userId) => {
                 expect(content.device_keys[userId]).toEqual(
-                    {},
+                    [],
                     "Expected key query for " + userId + ", got " +
                     Object.keys(content.device_keys),
                 );
@@ -194,7 +196,6 @@ TestClient.prototype.expectKeyQuery = function(response) {
         });
 };
 
-
 /**
  * get the uploaded curve25519 device key
  *
@@ -205,7 +206,6 @@ TestClient.prototype.getDeviceKey = function() {
     return this.deviceKeys.keys[keyId];
 };
 
-
 /**
  * get the uploaded ed25519 device key
  *
@@ -230,3 +230,7 @@ TestClient.prototype.flushSync = function() {
         logger.log(`${this}: flushSync completed`);
     });
 };
+
+TestClient.prototype.isFallbackICEServerAllowed = function() {
+    return true;
+};

spec/browserify/setupTests.js

@@ -0,0 +1,23 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// stub for browser-matrix browserify tests
+global.XMLHttpRequest = jest.fn();
+
+afterAll(() => {
+    // clean up XMLHttpRequest mock
+    global.XMLHttpRequest = undefined;
+});

spec/browserify/sync-browserify.spec.js

@@ -0,0 +1,103 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// load XmlHttpRequest mock
+import "./setupTests";
+import "../../dist/browser-matrix"; // uses browser-matrix instead of the src
+import { MockStorageApi } from "../MockStorageApi";
+import { WebStorageSessionStore } from "../../src/store/session/webstorage";
+import MockHttpBackend from "matrix-mock-request";
+import { LocalStorageCryptoStore } from "../../src/crypto/store/localStorage-crypto-store";
+import * as utils from "../test-utils";
+
+const USER_ID = "@user:test.server";
+const DEVICE_ID = "device_id";
+const ACCESS_TOKEN = "access_token";
+const ROOM_ID = "!room_id:server.test";
+
+/* global matrixcs */
+
+describe("Browserify Test", function() {
+    let client;
+    let httpBackend;
+
+    async function createTestClient() {
+        const sessionStoreBackend = new MockStorageApi();
+        const sessionStore = new WebStorageSessionStore(sessionStoreBackend);
+        const httpBackend = new MockHttpBackend();
+
+        const options = {
+            baseUrl: "http://" + USER_ID + ".test.server",
+            userId: USER_ID,
+            accessToken: ACCESS_TOKEN,
+            deviceId: DEVICE_ID,
+            sessionStore: sessionStore,
+            request: httpBackend.requestFn,
+            cryptoStore: new LocalStorageCryptoStore(sessionStoreBackend),
+        };
+
+        const client = matrixcs.createClient(options);
+
+        httpBackend.when("GET", "/pushrules").respond(200, {});
+        httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+
+        return { client, httpBackend };
+    }
+
+    beforeEach(async () => {
+        ({ client, httpBackend } = await createTestClient());
+        await client.startClient();
+    });
+
+    afterEach(async () => {
+        client.stopClient();
+        await httpBackend.stop();
+    });
+
+    it("Sync", async function() {
+        const event = utils.mkMembership({
+            room: ROOM_ID,
+            mship: "join",
+            user: "@other_user:server.test",
+            name: "Displayname",
+        });
+
+        const syncData = {
+            next_batch: "batch1",
+            rooms: {
+                join: {},
+            },
+        };
+        syncData.rooms.join[ROOM_ID] = {
+            timeline: {
+                events: [
+                    event,
+                ],
+                limited: false,
+            },
+        };
+
+        httpBackend.when("GET", "/sync").respond(200, syncData);
+        await Promise.race([
+            Promise.all([
+                httpBackend.flushAllExpected(),
+            ]),
+            new Promise((_, reject) => {
+                client.once("sync.unexpectedError", reject);
+            }),
+        ]);
+    }, 20000); // additional timeout as this test can take quite a while
+});

spec/integ/devicelist-integ-spec.js

@@ -16,9 +16,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {TestClient} from '../TestClient';
+import { TestClient } from '../TestClient';
 import * as testUtils from '../test-utils';
-import {logger} from '../../src/logger';
+import { logger } from '../../src/logger';
 
 const ROOM_ID = "!room:id";
 
@@ -67,7 +67,6 @@ function getSyncResponse(roomMembers) {
     return syncResponse;
 }
 
-
 describe("DeviceList management:", function() {
     if (!global.Olm) {
         logger.warn('not running deviceList tests: Olm not present');
@@ -98,7 +97,7 @@ describe("DeviceList management:", function() {
     });
 
     it("Alice shouldn't do a second /query for non-e2e-capable devices", function() {
-        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
+        aliceTestClient.expectKeyQuery({ device_keys: { '@alice:localhost': {} } });
         return aliceTestClient.start().then(function() {
             const syncResponse = getSyncResponse(['@bob:xyz']);
             aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
@@ -137,11 +136,10 @@ describe("DeviceList management:", function() {
         });
     });
 
-
     it("We should not get confused by out-of-order device query responses",
        () => {
-           // https://github.com/vector-im/riot-web/issues/3126
-           aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
+           // https://github.com/vector-im/element-web/issues/3126
+           aliceTestClient.expectKeyQuery({ device_keys: { '@alice:localhost': {} } });
            return aliceTestClient.start().then(() => {
                aliceTestClient.httpBackend.when('GET', '/sync').respond(
                    200, getSyncResponse(['@bob:xyz', '@chris:abc']));
@@ -160,14 +158,14 @@ describe("DeviceList management:", function() {
                );
 
                aliceTestClient.httpBackend.when('PUT', '/send/').respond(
-                   200, {event_id: '$event1'});
+                   200, { event_id: '$event1' });
 
                return Promise.all([
                    aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
                    aliceTestClient.httpBackend.flush('/keys/query', 1).then(
                        () => aliceTestClient.httpBackend.flush('/send/', 1),
                    ),
-                   aliceTestClient.client._crypto._deviceList.saveIfDirty(),
+                   aliceTestClient.client.crypto._deviceList.saveIfDirty(),
                ]);
            }).then(() => {
                aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
@@ -199,12 +197,12 @@ describe("DeviceList management:", function() {
                    },
                    token: '3',
                }).respond(200, {
-                   device_keys: {'@chris:abc': {}},
+                   device_keys: { '@chris:abc': {} },
                });
                return aliceTestClient.httpBackend.flush('/keys/query', 1);
            }).then((flushed) => {
                expect(flushed).toEqual(0);
-               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+               return aliceTestClient.client.crypto._deviceList.saveIfDirty();
            }).then(() => {
                aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
                    const bobStat = data.trackingStatus['@bob:xyz'];
@@ -228,7 +226,7 @@ describe("DeviceList management:", function() {
                    },
                    token: '2',
                }).respond(200, {
-                   device_keys: {'@bob:xyz': {}},
+                   device_keys: { '@bob:xyz': {} },
                });
                return aliceTestClient.httpBackend.flush('/keys/query', 1);
            }).then((flushed) => {
@@ -237,7 +235,7 @@ describe("DeviceList management:", function() {
                // wait for the client to stop processing the response
                return aliceTestClient.client.downloadKeys(['@bob:xyz']);
            }).then(() => {
-               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+               return aliceTestClient.client.crypto._deviceList.saveIfDirty();
            }).then(() => {
                aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
                    const bobStat = data.trackingStatus['@bob:xyz'];
@@ -258,7 +256,7 @@ describe("DeviceList management:", function() {
                // wait for the client to stop processing the response
                return aliceTestClient.client.downloadKeys(['@chris:abc']);
            }).then(() => {
-               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+               return aliceTestClient.client.crypto._deviceList.saveIfDirty();
            }).then(() => {
                aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
                    const bobStat = data.trackingStatus['@bob:xyz'];
@@ -271,7 +269,7 @@ describe("DeviceList management:", function() {
            });
        }).timeout(3000);
 
-    // https://github.com/vector-im/riot-web/issues/4983
+    // https://github.com/vector-im/element-web/issues/4983
     describe("Alice should know she has stale device lists", () => {
         beforeEach(async function() {
             await aliceTestClient.start();
@@ -288,7 +286,7 @@ describe("DeviceList management:", function() {
                 },
             );
             await aliceTestClient.httpBackend.flush('/keys/query', 1);
-            await aliceTestClient.client._crypto._deviceList.saveIfDirty();
+            await aliceTestClient.client.crypto._deviceList.saveIfDirty();
 
             aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
                 const bobStat = data.trackingStatus['@bob:xyz'];
@@ -323,9 +321,8 @@ describe("DeviceList management:", function() {
                 },
             );
 
-
             await aliceTestClient.flushSync();
-            await aliceTestClient.client._crypto._deviceList.saveIfDirty();
+            await aliceTestClient.client.crypto._deviceList.saveIfDirty();
 
             aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
                 const bobStat = data.trackingStatus['@bob:xyz'];
@@ -361,7 +358,7 @@ describe("DeviceList management:", function() {
             );
 
             await aliceTestClient.flushSync();
-            await aliceTestClient.client._crypto._deviceList.saveIfDirty();
+            await aliceTestClient.client.crypto._deviceList.saveIfDirty();
 
             aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
                 const bobStat = data.trackingStatus['@bob:xyz'];
@@ -382,7 +379,7 @@ describe("DeviceList management:", function() {
                 anotherTestClient.httpBackend.when('GET', '/sync').respond(
                     200, getSyncResponse([]));
                 await anotherTestClient.flushSync();
-                await anotherTestClient.client._crypto._deviceList.saveIfDirty();
+                await anotherTestClient.client.crypto._deviceList.saveIfDirty();
 
                 anotherTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
                     const bobStat = data.trackingStatus['@bob:xyz'];

spec/integ/matrix-client-crypto.spec.js

@@ -28,11 +28,10 @@ limitations under the License.
 // load olm before the sdk if possible
 import '../olm-loader';
 
-import {logger} from '../../src/logger';
+import { logger } from '../../src/logger';
 import * as testUtils from "../test-utils";
-import * as utils from "../../src/utils";
-import {TestClient} from "../TestClient";
-import {CRYPTO_ENABLED} from "../../src/client";
+import { TestClient } from "../TestClient";
+import { CRYPTO_ENABLED } from "../../src/client";
 
 let aliTestClient;
 const roomId = "!room:localhost";
@@ -70,13 +69,13 @@ function expectAliQueryKeys() {
     aliTestClient.httpBackend.when("POST", "/keys/query")
             .respond(200, function(path, content) {
         expect(content.device_keys[bobUserId]).toEqual(
-            {},
+            [],
             "Expected Alice to key query for " + bobUserId + ", got " +
             Object.keys(content.device_keys),
         );
         const result = {};
         result[bobUserId] = bobKeys;
-        return {device_keys: result};
+        return { device_keys: result };
     });
     return aliTestClient.httpBackend.flush("/keys/query", 1);
 }
@@ -98,13 +97,13 @@ function expectBobQueryKeys() {
         "POST", "/keys/query",
     ).respond(200, function(path, content) {
         expect(content.device_keys[aliUserId]).toEqual(
-            {},
+            [],
             "Expected Bob to key query for " + aliUserId + ", got " +
             Object.keys(content.device_keys),
         );
         const result = {};
         result[aliUserId] = aliKeys;
-        return {device_keys: result};
+        return { device_keys: result };
     });
     return bobTestClient.httpBackend.flush("/keys/query", 1);
 }
@@ -133,7 +132,7 @@ function expectAliClaimKeys() {
             result[bobUserId] = {};
             result[bobUserId][bobDeviceId] = {};
             result[bobUserId][bobDeviceId][keyId] = keys[keyId];
-            return {one_time_keys: result};
+            return { one_time_keys: result };
         });
     }).then(() => {
         // it can take a while to process the key query, so give it some extra
@@ -145,7 +144,6 @@ function expectAliClaimKeys() {
     });
 }
 
-
 function aliDownloadsKeys() {
     // can't query keys before bob has uploaded them
     expect(bobTestClient.getSigningKey()).toBeTruthy();
@@ -161,7 +159,7 @@ function aliDownloadsKeys() {
     // check that the localStorage is updated as we expect (not sure this is
     // an integration test, but meh)
     return Promise.all([p1, p2]).then(() => {
-        return aliTestClient.client._crypto._deviceList.saveIfDirty();
+        return aliTestClient.client.crypto._deviceList.saveIfDirty();
     }).then(() => {
         aliTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
             const devices = data.devices[bobUserId];
@@ -244,7 +242,7 @@ function bobSendsReplyMessage() {
 function expectAliSendMessageRequest() {
     return expectSendMessageRequest(aliTestClient.httpBackend).then(function(content) {
         aliMessages.push(content);
-        expect(utils.keys(content.ciphertext)).toEqual([bobTestClient.getDeviceKey()]);
+        expect(Object.keys(content.ciphertext)).toEqual([bobTestClient.getDeviceKey()]);
         const ciphertext = content.ciphertext[bobTestClient.getDeviceKey()];
         expect(ciphertext).toBeTruthy();
         return ciphertext;
@@ -261,7 +259,7 @@ function expectBobSendMessageRequest() {
         bobMessages.push(content);
         const aliKeyId = "curve25519:" + aliDeviceId;
         const aliDeviceCurve25519Key = aliTestClient.deviceKeys.keys[aliKeyId];
-        expect(utils.keys(content.ciphertext)).toEqual([aliDeviceCurve25519Key]);
+        expect(Object.keys(content.ciphertext)).toEqual([aliDeviceCurve25519Key]);
         const ciphertext = content.ciphertext[aliDeviceCurve25519Key];
         expect(ciphertext).toBeTruthy();
         return ciphertext;
@@ -270,7 +268,7 @@ function expectBobSendMessageRequest() {
 
 function sendMessage(client) {
     return client.sendMessage(
-        roomId, {msgtype: "m.text", body: "Hello, World"},
+        roomId, { msgtype: "m.text", body: "Hello, World" },
     );
 }
 
@@ -358,7 +356,6 @@ function recvMessage(httpBackend, client, sender, message) {
     });
 }
 
-
 /**
  * Send an initial sync response to the client (which just includes the member
  * list for our test room).
@@ -396,7 +393,6 @@ function firstSync(testClient) {
     return testClient.flushSync();
 }
 
-
 describe("MatrixClient crypto", function() {
     if (!CRYPTO_ENABLED) {
         return;
@@ -478,7 +474,7 @@ describe("MatrixClient crypto", function() {
         ).respond(200, function(path, content) {
             const result = {};
             result[bobUserId] = bobKeys;
-            return {device_keys: result};
+            return { device_keys: result };
         });
 
         return Promise.all([
@@ -520,7 +516,7 @@ describe("MatrixClient crypto", function() {
         ).respond(200, function(path, content) {
             const result = {};
             result[bobUserId] = bobKeys;
-            return {device_keys: result};
+            return { device_keys: result };
         });
 
         return Promise.all([
@@ -534,7 +530,6 @@ describe("MatrixClient crypto", function() {
         });
     });
 
-
     it("Bob starts his client and uploads device keys and one-time keys", function() {
         return Promise.resolve()
             .then(() => bobTestClient.start())
@@ -546,7 +541,7 @@ describe("MatrixClient crypto", function() {
     });
 
     it("Ali sends a message", function() {
-        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
         return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
@@ -556,7 +551,7 @@ describe("MatrixClient crypto", function() {
     });
 
     it("Bob receives a message", function() {
-        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
         return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
@@ -567,7 +562,7 @@ describe("MatrixClient crypto", function() {
     });
 
     it("Bob receives a message with a bogus sender", function() {
-        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
         return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
@@ -621,7 +616,7 @@ describe("MatrixClient crypto", function() {
     });
 
     it("Ali blocks Bob's device", function() {
-        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
         return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
@@ -641,7 +636,7 @@ describe("MatrixClient crypto", function() {
     });
 
     it("Bob receives two pre-key messages", function() {
-        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
         return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
@@ -654,8 +649,8 @@ describe("MatrixClient crypto", function() {
     });
 
     it("Bob replies to the message", function() {
-        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
-        bobTestClient.expectKeyQuery({device_keys: {[bobUserId]: {}}});
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
+        bobTestClient.expectKeyQuery({ device_keys: { [bobUserId]: {} } });
         return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
@@ -673,7 +668,7 @@ describe("MatrixClient crypto", function() {
     it("Ali does a key query when encryption is enabled", function() {
         // enabling encryption in the room should make alice download devices
         // for both members.
-        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        aliTestClient.expectKeyQuery({ device_keys: { [aliUserId]: {} } });
         return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => firstSync(aliTestClient))

spec/integ/matrix-client-event-emitter.spec.js

@@ -1,5 +1,5 @@
 import * as utils from "../test-utils";
-import {TestClient} from "../TestClient";
+import { TestClient } from "../TestClient";
 
 describe("MatrixClient events", function() {
     let client;

spec/integ/matrix-client-event-timeline.spec.js

@@ -1,7 +1,7 @@
 import * as utils from "../test-utils";
-import {EventTimeline} from "../../src/matrix";
-import {logger} from "../../src/logger";
-import {TestClient} from "../TestClient";
+import { EventTimeline } from "../../src/matrix";
+import { logger } from "../../src/logger";
+import { TestClient } from "../TestClient";
 
 const userId = "@alice:localhost";
 const userName = "Alice";
@@ -127,7 +127,7 @@ describe("getEventTimeline support", function() {
             "DEVICE",
             accessToken,
             undefined,
-            {timelineSupport: true},
+            { timelineSupport: true },
         );
         client = testClient.client;
         httpBackend = testClient.httpBackend;
@@ -141,7 +141,6 @@ describe("getEventTimeline support", function() {
         });
     });
 
-
     it("scrollback should be able to scroll back to before a gappy /sync",
       function() {
         // need a client with timelineSupport disabled to make this work
@@ -218,7 +217,7 @@ describe("MatrixClient event timelines", function() {
             "DEVICE",
             accessToken,
             undefined,
-            {timelineSupport: true},
+            { timelineSupport: true },
         );
         client = testClient.client;
         httpBackend = testClient.httpBackend;
@@ -516,7 +515,7 @@ describe("MatrixClient event timelines", function() {
                 client.getEventTimeline(timelineSet, EVENTS[0].event_id,
                 ).then(function(tl0) {
                     tl = tl0;
-                    return client.paginateEventTimeline(tl, {backwards: true});
+                    return client.paginateEventTimeline(tl, { backwards: true });
                 }).then(function(success) {
                     expect(success).toBeTruthy();
                     expect(tl.getEvents().length).toEqual(3);
@@ -532,7 +531,6 @@ describe("MatrixClient event timelines", function() {
             ]);
         });
 
-
         it("should allow you to paginate forwards", function() {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
@@ -569,7 +567,7 @@ describe("MatrixClient event timelines", function() {
                 ).then(function(tl0) {
                     tl = tl0;
                     return client.paginateEventTimeline(
-                        tl, {backwards: false, limit: 20});
+                        tl, { backwards: false, limit: 20 });
                 }).then(function(success) {
                     expect(success).toBeTruthy();
                     expect(tl.getEvents().length).toEqual(3);
@@ -591,7 +589,7 @@ describe("MatrixClient event timelines", function() {
         const event = utils.mkMessage({
             room: roomId, user: userId, msg: "a body",
         });
-        event.unsigned = {transaction_id: TXN_ID};
+        event.unsigned = { transaction_id: TXN_ID };
 
         beforeEach(function() {
             // set up handlers for both the message send, and the
@@ -680,7 +678,6 @@ describe("MatrixClient event timelines", function() {
         });
     });
 
-
     it("should handle gappy syncs after redactions", function() {
         // https://github.com/vector-im/vector-web/issues/1389
 

spec/integ/matrix-client-methods.spec.js

@@ -1,7 +1,7 @@
 import * as utils from "../test-utils";
-import {CRYPTO_ENABLED} from "../../src/client";
-import {Filter, MemoryStore, Room} from "../../src/matrix";
-import {TestClient} from "../TestClient";
+import { CRYPTO_ENABLED } from "../../src/client";
+import { Filter, MemoryStore, Room } from "../../src/matrix";
+import { TestClient } from "../TestClient";
 
 describe("MatrixClient", function() {
     let client = null;
@@ -285,7 +285,6 @@ describe("MatrixClient", function() {
         });
     });
 
-
     describe("downloadKeys", function() {
         if (!CRYPTO_ENABLED) {
             return;
@@ -337,7 +336,7 @@ describe("MatrixClient", function() {
                 var b = JSON.parse(JSON.stringify(o));
                 delete(b.signatures);
                 delete(b.unsigned);
-                return client._crypto._olmDevice.sign(anotherjson.stringify(b));
+                return client.crypto._olmDevice.sign(anotherjson.stringify(b));
             };
 
             logger.log("Ed25519: " + ed25519key);
@@ -346,10 +345,10 @@ describe("MatrixClient", function() {
             */
 
             httpBackend.when("POST", "/keys/query").check(function(req) {
-                expect(req.data).toEqual({device_keys: {
-                    'boris': {},
-                    'chaz': {},
-                }});
+                expect(req.data).toEqual({ device_keys: {
+                    'boris': [],
+                    'chaz': [],
+                } });
             }).respond(200, {
                 device_keys: {
                     boris: borisKeys,
@@ -379,12 +378,12 @@ describe("MatrixClient", function() {
     });
 
     describe("deleteDevice", function() {
-        const auth = {a: 1};
+        const auth = { a: 1 };
         it("should pass through an auth dict", function() {
             httpBackend.when(
                 "DELETE", "/_matrix/client/r0/devices/my_device",
             ).check(function(req) {
-                expect(req.data).toEqual({auth: auth});
+                expect(req.data).toEqual({ auth: auth });
             }).respond(200);
 
             const prom = client.deleteDevice("my_device", auth);

spec/integ/matrix-client-opts.spec.js

@@ -1,8 +1,9 @@
 import * as utils from "../test-utils";
 import HttpBackend from "matrix-mock-request";
-import {MatrixClient} from "../../src/matrix";
-import {MatrixScheduler} from "../../src/scheduler";
-import {MemoryStore} from "../../src/store/memory";
+import { MatrixClient } from "../../src/matrix";
+import { MatrixScheduler } from "../../src/scheduler";
+import { MemoryStore } from "../../src/store/memory";
+import { MatrixError } from "../../src/http-api";
 
 describe("MatrixClient opts", function() {
     const baseUrl = "http://localhost.or.something";
@@ -132,10 +133,10 @@ describe("MatrixClient opts", function() {
         });
 
         it("shouldn't retry sending events", function(done) {
-            httpBackend.when("PUT", "/txn1").fail(500, {
+            httpBackend.when("PUT", "/txn1").fail(500, new MatrixError({
                 errcode: "M_SOMETHING",
                 error: "Ruh roh",
-            });
+            }));
             client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                 expect(false).toBe(true, "sendTextMessage resolved but shouldn't");
             }, function(err) {

spec/integ/matrix-client-retrying.spec.js

@@ -1,7 +1,7 @@
-import {EventStatus} from "../../src/matrix";
-import {MatrixScheduler} from "../../src/scheduler";
-import {Room} from "../../src/models/room";
-import {TestClient} from "../TestClient";
+import { EventStatus } from "../../src/matrix";
+import { MatrixScheduler } from "../../src/scheduler";
+import { Room } from "../../src/models/room";
+import { TestClient } from "../TestClient";
 
 describe("MatrixClient retrying", function() {
     let client = null;
@@ -19,7 +19,7 @@ describe("MatrixClient retrying", function() {
             "DEVICE",
             accessToken,
             undefined,
-            {scheduler},
+            { scheduler },
         );
         httpBackend = testClient.httpBackend;
         client = testClient.client;
@@ -90,7 +90,7 @@ describe("MatrixClient retrying", function() {
         // wait for the localecho of ev1 to be updated
         const p3 = new Promise((resolve, reject) => {
             room.on("Room.localEchoUpdated", (ev0) => {
-                if(ev0 === ev1) {
+                if (ev0 === ev1) {
                     resolve();
                 }
             });

spec/integ/matrix-client-room-timeline.spec.js

@@ -1,7 +1,6 @@
 import * as utils from "../test-utils";
-import {EventStatus} from "../../src/models/event";
-import {TestClient} from "../TestClient";
-
+import { EventStatus } from "../../src/models/event";
+import { TestClient } from "../TestClient";
 
 describe("MatrixClient room timelines", function() {
     let client = null;
@@ -104,7 +103,7 @@ describe("MatrixClient room timelines", function() {
             "DEVICE",
             accessToken,
             undefined,
-            {timelineSupport: true},
+            { timelineSupport: true },
         );
         httpBackend = testClient.httpBackend;
         client = testClient.client;
@@ -166,7 +165,7 @@ describe("MatrixClient room timelines", function() {
                 body: "I am a fish", user: userId, room: roomId,
             });
             ev.event_id = eventId;
-            ev.unsigned = {transaction_id: "txn1"};
+            ev.unsigned = { transaction_id: "txn1" };
             setNextSyncData([ev]);
 
             client.on("sync", function(state) {
@@ -198,7 +197,7 @@ describe("MatrixClient room timelines", function() {
                 body: "I am a fish", user: userId, room: roomId,
             });
             ev.event_id = eventId;
-            ev.unsigned = {transaction_id: "txn1"};
+            ev.unsigned = { transaction_id: "txn1" };
             setNextSyncData([ev]);
 
             client.on("sync", function(state) {
@@ -396,8 +395,8 @@ describe("MatrixClient room timelines", function() {
     describe("new events", function() {
         it("should be added to the right place in the timeline", function() {
             const eventData = [
-                utils.mkMessage({user: userId, room: roomId}),
-                utils.mkMessage({user: userId, room: roomId}),
+                utils.mkMessage({ user: userId, room: roomId }),
+                utils.mkMessage({ user: userId, room: roomId }),
             ];
             setNextSyncData(eventData);
 
@@ -434,11 +433,11 @@ describe("MatrixClient room timelines", function() {
 
         it("should set the right event.sender values", function() {
             const eventData = [
-                utils.mkMessage({user: userId, room: roomId}),
+                utils.mkMessage({ user: userId, room: roomId }),
                 utils.mkMembership({
                     user: userId, room: roomId, mship: "join", name: "New Name",
                 }),
-                utils.mkMessage({user: userId, room: roomId}),
+                utils.mkMessage({ user: userId, room: roomId }),
             ];
             eventData[1].__prev_event = USER_MEMBERSHIP_EVENT;
             setNextSyncData(eventData);
@@ -546,7 +545,7 @@ describe("MatrixClient room timelines", function() {
     describe("gappy sync", function() {
         it("should copy the last known state to the new timeline", function() {
             const eventData = [
-                utils.mkMessage({user: userId, room: roomId}),
+                utils.mkMessage({ user: userId, room: roomId }),
             ];
             setNextSyncData(eventData);
             NEXT_SYNC_DATA.rooms.join[roomId].timeline.limited = true;
@@ -579,7 +578,7 @@ describe("MatrixClient room timelines", function() {
 
         it("should emit a 'Room.timelineReset' event", function() {
             const eventData = [
-                utils.mkMessage({user: userId, room: roomId}),
+                utils.mkMessage({ user: userId, room: roomId }),
             ];
             setNextSyncData(eventData);
             NEXT_SYNC_DATA.rooms.join[roomId].timeline.limited = true;

spec/integ/matrix-client-syncing.spec.js

@@ -1,7 +1,7 @@
-import {MatrixEvent} from "../../src/models/event";
-import {EventTimeline} from "../../src/models/event-timeline";
+import { MatrixEvent } from "../../src/models/event";
+import { EventTimeline } from "../../src/models/event-timeline";
 import * as utils from "../test-utils";
-import {TestClient} from "../TestClient";
+import { TestClient } from "../TestClient";
 
 describe("MatrixClient syncing", function() {
     let client = null;
@@ -122,7 +122,6 @@ describe("MatrixClient syncing", function() {
                 resolveInvitesToProfiles: true,
             });
 
-
             return Promise.all([
                 httpBackend.flushAllExpected(),
                 awaitSyncEvent(),
@@ -677,8 +676,8 @@ describe("MatrixClient syncing", function() {
         it("should create and use an appropriate filter", function() {
             httpBackend.when("POST", "/filter").check(function(req) {
                 expect(req.data).toEqual({
-                    room: { timeline: {limit: 1},
-                            include_leave: true }});
+                    room: { timeline: { limit: 1 },
+                            include_leave: true } });
             }).respond(200, { filter_id: "another_id" });
 
             const prom = new Promise((resolve) => {

spec/integ/megolm-integ.spec.js

@@ -16,10 +16,9 @@ limitations under the License.
 */
 
 import anotherjson from "another-json";
-import * as utils from "../../src/utils";
 import * as testUtils from "../test-utils";
-import {TestClient} from "../TestClient";
-import {logger} from "../../src/logger";
+import { TestClient } from "../TestClient";
+import { logger } from "../../src/logger";
 
 const ROOM_ID = "!room:id";
 
@@ -32,7 +31,7 @@ const ROOM_ID = "!room:id";
  */
 function createOlmSession(olmAccount, recipientTestClient) {
     return recipientTestClient.awaitOneTimeKeyUpload().then((keys) => {
-        const otkId = utils.keys(keys)[0];
+        const otkId = Object.keys(keys)[0];
         const otk = keys[otkId];
 
         const session = new global.Olm.Session();
@@ -197,7 +196,6 @@ function getSyncResponse(roomMembers) {
     return syncResponse;
 }
 
-
 describe("megolm", function() {
     if (!global.Olm) {
         logger.warn('not running megolm tests: Olm not present');
@@ -257,7 +255,7 @@ describe("megolm", function() {
         const testOneTimeKeys = JSON.parse(testOlmAccount.one_time_keys());
         testOlmAccount.mark_keys_as_published();
 
-        const keyId = utils.keys(testOneTimeKeys.curve25519)[0];
+        const keyId = Object.keys(testOneTimeKeys.curve25519)[0];
         const oneTimeKey = testOneTimeKeys.curve25519[keyId];
         const keyResult = {
             'key': oneTimeKey,
@@ -269,7 +267,7 @@ describe("megolm", function() {
             'ed25519:DEVICE_ID': sig,
         };
 
-        const claimResponse = {one_time_keys: {}};
+        const claimResponse = { one_time_keys: {} };
         claimResponse.one_time_keys[userId] = {
             'DEVICE_ID': {},
         };
@@ -346,7 +344,7 @@ describe("megolm", function() {
     });
 
     it("Alice receives a megolm message before the session keys", function() {
-        // https://github.com/vector-im/riot-web/issues/2273
+        // https://github.com/vector-im/element-web/issues/2273
         let roomKeyEncrypted;
 
         return aliceTestClient.start().then(() => {
@@ -484,8 +482,9 @@ describe("megolm", function() {
             return aliceTestClient.flushSync().then(() => {
                 return aliceTestClient.flushSync();
             });
-        }).then(function() {
+        }).then(async function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
+            await room.decryptCriticalEvents();
             const event = room.getLiveTimeline().getEvents()[0];
             expect(event.getContent().body).toEqual('42');
         });
@@ -494,7 +493,7 @@ describe("megolm", function() {
     it('Alice sends a megolm message', function() {
         let p2pSession;
 
-        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
+        aliceTestClient.expectKeyQuery({ device_keys: { '@alice:localhost': {} } });
         return aliceTestClient.start().then(() => {
             // establish an olm session with alice
             return createOlmSession(testOlmAccount, aliceTestClient);
@@ -577,7 +576,7 @@ describe("megolm", function() {
     });
 
     it("We shouldn't attempt to send to blocked devices", function() {
-        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
+        aliceTestClient.expectKeyQuery({ device_keys: { '@alice:localhost': {} } });
         return aliceTestClient.start().then(() => {
             // establish an olm session with alice
             return createOlmSession(testOlmAccount, aliceTestClient);
@@ -634,7 +633,7 @@ describe("megolm", function() {
         let p2pSession;
         let megolmSessionId;
 
-        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
+        aliceTestClient.expectKeyQuery({ device_keys: { '@alice:localhost': {} } });
         return aliceTestClient.start().then(() => {
             // establish an olm session with alice
             return createOlmSession(testOlmAccount, aliceTestClient);
@@ -726,7 +725,7 @@ describe("megolm", function() {
         });
     });
 
-    // https://github.com/vector-im/riot-web/issues/2676
+    // https://github.com/vector-im/element-web/issues/2676
     it("Alice should send to her other devices", function() {
         // for this test, we make the testOlmAccount be another of Alice's devices.
         // it ought to get included in messages Alice sends.
@@ -841,13 +840,12 @@ describe("megolm", function() {
         });
     });
 
-
     it('Alice should wait for device list to complete when sending a megolm message',
     function() {
         let downloadPromise;
         let sendPromise;
 
-        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
+        aliceTestClient.expectKeyQuery({ device_keys: { '@alice:localhost': {} } });
         return aliceTestClient.start().then(() => {
             // establish an olm session with alice
             return createOlmSession(testOlmAccount, aliceTestClient);
@@ -887,11 +885,10 @@ describe("megolm", function() {
         });
     });
 
-
     it("Alice exports megolm keys and imports them to a new device", function() {
         let messageEncrypted;
 
-        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
+        aliceTestClient.expectKeyQuery({ device_keys: { '@alice:localhost': {} } });
         return aliceTestClient.start().then(() => {
             // establish an olm session with alice
             return createOlmSession(testOlmAccount, aliceTestClient);
@@ -933,8 +930,9 @@ describe("megolm", function() {
 
             aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
             return aliceTestClient.flushSync();
-        }).then(function() {
+        }).then(async function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
+            await room.decryptCriticalEvents();
             const event = room.getLiveTimeline().getEvents()[0];
             expect(event.getContent().body).toEqual('42');
 
@@ -971,4 +969,60 @@ describe("megolm", function() {
             expect(event.getContent().body).toEqual('42');
         });
     });
+
+    it("Alice receives an untrusted megolm key, only to receive the trusted one shortly after", function() {
+        const testClient = new TestClient(
+            "@alice:localhost", "device2", "access_token2",
+        );
+        const groupSession = new Olm.OutboundGroupSession();
+        groupSession.create();
+        const inboundGroupSession = new Olm.InboundGroupSession();
+        inboundGroupSession.create(groupSession.session_key());
+        const rawEvent = encryptMegolmEvent({
+            senderKey: testSenderKey,
+            groupSession: groupSession,
+            room_id: ROOM_ID,
+        });
+        return testClient.client.initCrypto().then(() => {
+            const keys = [{
+                room_id: ROOM_ID,
+                algorithm: 'm.megolm.v1.aes-sha2',
+                session_id: groupSession.session_id(),
+                session_key: inboundGroupSession.export_session(0),
+                sender_key: testSenderKey,
+            }];
+            return testClient.client.importRoomKeys(keys, { untrusted: true });
+        }).then(() => {
+            const event = testUtils.mkEvent({
+                event: true,
+                ...rawEvent,
+                room: ROOM_ID,
+            });
+            return event.attemptDecryption(testClient.client.crypto, true).then(() => {
+                expect(event.isKeySourceUntrusted()).toBeTruthy();
+            });
+        }).then(() => {
+            const event = testUtils.mkEvent({
+                type: 'm.room_key',
+                content: {
+                    room_id: ROOM_ID,
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    session_id: groupSession.session_id(),
+                    session_key: groupSession.session_key(),
+                },
+                event: true,
+            });
+            event._senderCurve25519Key = testSenderKey;
+            return testClient.client.crypto._onRoomKeyEvent(event);
+        }).then(() => {
+            const event = testUtils.mkEvent({
+                event: true,
+                ...rawEvent,
+                room: ROOM_ID,
+            });
+            return event.attemptDecryption(testClient.client.crypto, true).then(() => {
+                expect(event.isKeySourceUntrusted()).toBeFalsy();
+            });
+        });
+    });
 });

spec/olm-loader.js

@@ -15,12 +15,21 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {logger} from '../src/logger';
+import { logger } from '../src/logger';
+import * as utils from "../src/utils";
 
 // try to load the olm library.
 try {
-    global.Olm = require('olm');
+    global.Olm = require('@matrix-org/olm');
     logger.log('loaded libolm');
 } catch (e) {
     logger.warn("unable to run crypto tests: libolm not available");
 }
+
+// also try to set node crypto
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    logger.log('nodejs was compiled without crypto support: some tests will fail');
+}

spec/test-utils.js

@@ -1,8 +1,8 @@
 // load olm before the sdk if possible
 import './olm-loader';
 
-import {logger} from '../src/logger';
-import {MatrixEvent} from "../src/models/event";
+import { logger } from '../src/logger';
+import { MatrixEvent } from "../src/models/event";
 
 /**
  * Return a promise that is resolved when the client next emits a
@@ -177,7 +177,6 @@ export function mkMessage(opts) {
     return mkEvent(opts);
 }
 
-
 /**
  * A mock implementation of webstorage
  *
@@ -204,7 +203,6 @@ MockStorageApi.prototype = {
     },
 };
 
-
 /**
  * If an event is being decrypted, wait for it to finish being decrypted.
  *
@@ -212,21 +210,23 @@ MockStorageApi.prototype = {
  * @returns {Promise} promise which resolves (to `event`) when the event has been decrypted
  */
 export function awaitDecryption(event) {
-    if (!event.isBeingDecrypted()) {
-        return Promise.resolve(event);
-    }
+    // An event is not always decrypted ahead of time
+    // getClearContent is a good signal to know whether an event has been decrypted
+    // already
+    if (event.getClearContent() !== null) {
+        return event;
+    } else {
+        logger.log(`${Date.now()} event ${event.getId()} is being decrypted; waiting`);
 
-    logger.log(`${Date.now()} event ${event.getId()} is being decrypted; waiting`);
-
-    return new Promise((resolve, reject) => {
-        event.once('Event.decrypted', (ev) => {
-            logger.log(`${Date.now()} event ${event.getId()} now decrypted`);
-            resolve(ev);
+        return new Promise((resolve, reject) => {
+            event.once('Event.decrypted', (ev) => {
+                logger.log(`${Date.now()} event ${event.getId()} now decrypted`);
+                resolve(ev);
+            });
         });
-    });
+    }
 }
 
-
 export function HttpResponse(
     httpLookups, acceptKeepalives, ignoreUnhandledSync,
 ) {
@@ -357,12 +357,12 @@ export function setHttpResponses(
     );
 
     const httpReq = httpResponseObj.request.bind(httpResponseObj);
-    client._http = [
+    client.http = [
         "authedRequest", "authedRequestWithPrefix", "getContentUri",
         "request", "requestWithPrefix", "uploadContent",
     ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
-    client._http.authedRequest.mockImplementation(httpReq);
-    client._http.authedRequestWithPrefix.mockImplementation(httpReq);
-    client._http.requestWithPrefix.mockImplementation(httpReq);
-    client._http.request.mockImplementation(httpReq);
+    client.http.authedRequest.mockImplementation(httpReq);
+    client.http.authedRequestWithPrefix.mockImplementation(httpReq);
+    client.http.requestWithPrefix.mockImplementation(httpReq);
+    client.http.request.mockImplementation(httpReq);
 }

spec/unit/NamespacedValue.spec.ts

@@ -0,0 +1,78 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { NamespacedValue, UnstableValue } from "../../src/NamespacedValue";
+
+describe("NamespacedValue", () => {
+    it("should prefer stable over unstable", () => {
+        const ns = new NamespacedValue("stable", "unstable");
+        expect(ns.name).toBe(ns.stable);
+        expect(ns.altName).toBe(ns.unstable);
+    });
+
+    it("should return unstable if there is no stable", () => {
+        const ns = new NamespacedValue(null, "unstable");
+        expect(ns.name).toBe(ns.unstable);
+        expect(ns.altName).toBeFalsy();
+    });
+
+    it("should have a falsey unstable if needed", () => {
+        const ns = new NamespacedValue("stable", null);
+        expect(ns.name).toBe(ns.stable);
+        expect(ns.altName).toBeFalsy();
+    });
+
+    it("should match against either stable or unstable", () => {
+        const ns = new NamespacedValue("stable", "unstable");
+        expect(ns.matches("no")).toBe(false);
+        expect(ns.matches(ns.stable)).toBe(true);
+        expect(ns.matches(ns.unstable)).toBe(true);
+    });
+
+    it("should not permit falsey values for both parts", () => {
+        try {
+            new UnstableValue(null, null);
+            // noinspection ExceptionCaughtLocallyJS
+            throw new Error("Failed to fail");
+        } catch (e) {
+            expect(e.message).toBe("One of stable or unstable values must be supplied");
+        }
+    });
+});
+
+describe("UnstableValue", () => {
+    it("should prefer unstable over stable", () => {
+        const ns = new UnstableValue("stable", "unstable");
+        expect(ns.name).toBe(ns.unstable);
+        expect(ns.altName).toBe(ns.stable);
+    });
+
+    it("should return unstable if there is no stable", () => {
+        const ns = new UnstableValue(null, "unstable");
+        expect(ns.name).toBe(ns.unstable);
+        expect(ns.altName).toBeFalsy();
+    });
+
+    it("should not permit falsey unstable values", () => {
+        try {
+            new UnstableValue("stable", null);
+            // noinspection ExceptionCaughtLocallyJS
+            throw new Error("Failed to fail");
+        } catch (e) {
+            expect(e.message).toBe("Unstable value must be supplied");
+        }
+    });
+});

spec/unit/ReEmitter.spec.ts

@@ -0,0 +1,72 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventEmitter } from "events";
+import { ReEmitter } from "../../src/ReEmitter";
+
+const EVENTNAME = "UnknownEntry";
+
+class EventSource extends EventEmitter {
+    doTheThing() {
+        this.emit(EVENTNAME, "foo", "bar");
+    }
+
+    doAnError() {
+        this.emit('error');
+    }
+}
+
+class EventTarget extends EventEmitter {
+
+}
+
+describe("ReEmitter", function() {
+    it("Re-Emits events with the same args", function() {
+        const src = new EventSource();
+        const tgt = new EventTarget();
+
+        const handler = jest.fn();
+        tgt.on(EVENTNAME, handler);
+
+        const reEmitter = new ReEmitter(tgt);
+        reEmitter.reEmit(src, [EVENTNAME]);
+
+        src.doTheThing();
+
+        // Args should be the args passed to 'emit' after the event name, and
+        // also the source object of the event which re-emitter adds
+        expect(handler).toHaveBeenCalledWith("foo", "bar", src);
+    });
+
+    it("Doesn't throw if no handler for 'error' event", function() {
+        const src = new EventSource();
+        const tgt = new EventTarget();
+
+        const reEmitter = new ReEmitter(tgt);
+        reEmitter.reEmit(src, ['error']);
+
+        // without the workaround in ReEmitter, this would throw
+        src.doAnError();
+
+        const handler = jest.fn();
+        tgt.on('error', handler);
+
+        src.doAnError();
+
+        // Now we've attached an error handler, it should be called
+        expect(handler).toHaveBeenCalled();
+    });
+});

spec/unit/autodiscovery.spec.js

@@ -17,7 +17,7 @@ limitations under the License.
 
 import MockHttpBackend from "matrix-mock-request";
 import * as sdk from "../../src";
-import {AutoDiscovery} from "../../src/autodiscovery";
+import { AutoDiscovery } from "../../src/autodiscovery";
 
 describe("AutoDiscovery", function() {
     let httpBackend = null;

spec/unit/content-repo.spec.js

@@ -1,4 +1,4 @@
-import {getHttpUriForMxc, getIdenticonUri} from "../../src/content-repo";
+import { getHttpUriForMxc } from "../../src/content-repo";
 
 describe("ContentRepo", function() {
     const baseUrl = "https://my.home.server";
@@ -56,31 +56,4 @@ describe("ContentRepo", function() {
             );
         });
     });
-
-    describe("getIdenticonUri", function() {
-        it("should do nothing for null input", function() {
-            expect(getIdenticonUri(null)).toEqual(null);
-        });
-
-        it("should set w/h by default to 96", function() {
-            expect(getIdenticonUri(baseUrl, "foobar")).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foobar" +
-                "?width=96&height=96",
-            );
-        });
-
-        it("should be able to set custom w/h", function() {
-            expect(getIdenticonUri(baseUrl, "foobar", 32, 64)).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foobar" +
-                "?width=32&height=64",
-            );
-        });
-
-        it("should URL encode the identicon string", function() {
-            expect(getIdenticonUri(baseUrl, "foo#bar", 32, 64)).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foo%23bar" +
-                "?width=32&height=64",
-            );
-        });
-    });
 });

spec/unit/crypto.spec.js

@@ -1,15 +1,16 @@
 import '../olm-loader';
-import {Crypto} from "../../src/crypto";
-import {WebStorageSessionStore} from "../../src/store/session/webstorage";
-import {MemoryCryptoStore} from "../../src/crypto/store/memory-crypto-store";
-import {MockStorageApi} from "../MockStorageApi";
-import {TestClient} from "../TestClient";
-import {MatrixEvent} from "../../src/models/event";
-import {Room} from "../../src/models/room";
+import { Crypto } from "../../src/crypto";
+import { WebStorageSessionStore } from "../../src/store/session/webstorage";
+import { MemoryCryptoStore } from "../../src/crypto/store/memory-crypto-store";
+import { MockStorageApi } from "../MockStorageApi";
+import { TestClient } from "../TestClient";
+import { MatrixEvent } from "../../src/models/event";
+import { Room } from "../../src/models/room";
 import * as olmlib from "../../src/crypto/olmlib";
-import {sleep} from "../../src/utils";
-import {EventEmitter} from "events";
-import {CRYPTO_ENABLED} from "../../src/client";
+import { sleep } from "../../src/utils";
+import { EventEmitter } from "events";
+import { CRYPTO_ENABLED } from "../../src/client";
+import { DeviceInfo } from "../../src/crypto/deviceinfo";
 
 const Olm = global.Olm;
 
@@ -26,6 +27,66 @@ describe("Crypto", function() {
         expect(Crypto.getOlmVersion()[0]).toEqual(3);
     });
 
+    describe("encrypted events", function() {
+        it("provides encryption information", async function() {
+            const client = (new TestClient(
+                "@alice:example.com", "deviceid",
+            )).client;
+            await client.initCrypto();
+
+            // unencrypted event
+            const event = {
+                getId: () => "$event_id",
+                getSenderKey: () => null,
+                getWireContent: () => {return {};},
+            };
+
+            let encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeFalsy();
+
+            // unknown sender (e.g. deleted device), forwarded megolm key (untrusted)
+            event.getSenderKey = () => 'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI';
+            event.getWireContent = () => {return { algorithm: olmlib.MEGOLM_ALGORITHM };};
+            event.getForwardingCurve25519KeyChain = () => ["not empty"];
+            event.isKeySourceUntrusted = () => false;
+            event.getClaimedEd25519Key =
+                () => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeFalsy();
+            expect(encryptionInfo.sender).toBeFalsy();
+
+            // known sender, megolm key from backup
+            event.getForwardingCurve25519KeyChain = () => [];
+            event.isKeySourceUntrusted = () => true;
+            const device = new DeviceInfo("FLIBBLE");
+            device.keys["curve25519:FLIBBLE"] =
+                'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI';
+            device.keys["ed25519:FLIBBLE"] =
+                'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
+            client.crypto._deviceList.getDeviceByIdentityKey = () => device;
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeFalsy();
+            expect(encryptionInfo.sender).toBeTruthy();
+            expect(encryptionInfo.mismatchedSender).toBeFalsy();
+
+            // known sender, trusted megolm key, but bad ed25519key
+            event.isKeySourceUntrusted = () => false;
+            device.keys["ed25519:FLIBBLE"] =
+                'BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB';
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeTruthy();
+            expect(encryptionInfo.sender).toBeTruthy();
+            expect(encryptionInfo.mismatchedSender).toBeTruthy();
+
+            client.stopClient();
+        });
+    });
 
     describe('Session management', function() {
         const otkResponse = {
@@ -152,7 +213,7 @@ describe("Crypto", function() {
 
                 async function keyshareEventForEvent(event, index) {
                     const eventContent = event.getWireContent();
-                    const key = await aliceClient._crypto._olmDevice
+                    const key = await aliceClient.crypto._olmDevice
                         .getInboundGroupSessionKey(
                             roomId, eventContent.sender_key, eventContent.session_id,
                             index,
@@ -212,19 +273,19 @@ describe("Crypto", function() {
                 await Promise.all(events.map(async (event) => {
                     // alice encrypts each event, and then bob tries to decrypt
                     // them without any keys, so that they'll be in pending
-                    await aliceClient._crypto.encryptEvent(event, aliceRoom);
+                    await aliceClient.crypto.encryptEvent(event, aliceRoom);
                     event._clearEvent = {};
                     event._senderCurve25519Key = null;
                     event._claimedEd25519Key = null;
                     try {
-                        await bobClient._crypto.decryptEvent(event);
+                        await bobClient.crypto.decryptEvent(event);
                     } catch (e) {
                         // we expect this to fail because we don't have the
                         // decryption keys yet
                     }
                 }));
 
-                const bobDecryptor = bobClient._crypto._getRoomDecryptor(
+                const bobDecryptor = bobClient.crypto._getRoomDecryptor(
                     roomId, olmlib.MEGOLM_ALGORITHM,
                 );
 
@@ -241,7 +302,7 @@ describe("Crypto", function() {
                 expect(events[0].getContent().msgtype).toBe("m.bad.encrypted");
                 expect(events[1].getContent().msgtype).not.toBe("m.bad.encrypted");
 
-                const cryptoStore = bobClient._cryptoStore;
+                const cryptoStore = bobClient.cryptoStore;
                 const eventContent = events[0].getWireContent();
                 const senderKey = eventContent.sender_key;
                 const sessionId = eventContent.session_id;
@@ -283,7 +344,7 @@ describe("Crypto", function() {
                 },
             });
             await aliceClient.cancelAndResendEventRoomKeyRequest(event);
-            const cryptoStore = aliceClient._cryptoStore;
+            const cryptoStore = aliceClient.cryptoStore;
             const roomKeyRequestBody = {
                 algorithm: olmlib.MEGOLM_ALGORITHM,
                 room_id: "!someroom",
@@ -313,6 +374,10 @@ describe("Crypto", function() {
             // make a room key request, and record the transaction ID for the
             // sendToDevice call
             await aliceClient.cancelAndResendEventRoomKeyRequest(event);
+            // key requests get queued until the sync has finished, but we don't
+            // let the client set up enough for that to happen, so gut-wrench a bit
+            // to force it to send now.
+            aliceClient.crypto._outgoingRoomKeyRequestManager.sendQueuedRequests();
             jest.runAllTimers();
             await Promise.resolve();
             expect(aliceClient.sendToDevice).toBeCalledTimes(1);

spec/unit/crypto/CrossSigningInfo.spec.js

@@ -22,9 +22,11 @@ import {
 import {
     IndexedDBCryptoStore,
 } from '../../../src/crypto/store/indexeddb-crypto-store';
-import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
+import { MemoryCryptoStore } from '../../../src/crypto/store/memory-crypto-store';
 import 'fake-indexeddb/auto';
 import 'jest-localstorage-mock';
+import { OlmDevice } from "../../../src/crypto/OlmDevice";
+import { logger } from '../../../src/logger';
 
 const userId = "@alice:example.com";
 
@@ -37,7 +39,7 @@ const testKey = new Uint8Array([
 ]);
 
 const types = [
-    { type: "master", shouldCache: false },
+    { type: "master", shouldCache: true },
     { type: "self_signing", shouldCache: true },
     { type: "user_signing", shouldCache: true },
     { type: "invalid", shouldCache: false },
@@ -50,7 +52,7 @@ const masterKeyPub = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk";
 
 describe("CrossSigningInfo.getCrossSigningKey", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 
@@ -64,7 +66,7 @@ describe("CrossSigningInfo.getCrossSigningKey", function() {
     });
 
     it.each(types)("should throw if the callback returns falsey",
-                   async ({type, shouldCache}) => {
+                   async ({ type, shouldCache }) => {
         const info = new CrossSigningInfo(userId, {
             getCrossSigningKey: () => false,
         });
@@ -82,9 +84,16 @@ describe("CrossSigningInfo.getCrossSigningKey", function() {
         const info = new CrossSigningInfo(userId, {
             getCrossSigningKey: () => testKey,
         });
-        const [pubKey, ab] = await info.getCrossSigningKey("master", masterKeyPub);
+        const [pubKey, pkSigning] = await info.getCrossSigningKey("master", masterKeyPub);
         expect(pubKey).toEqual(masterKeyPub);
-        expect(ab).toEqual({a: 106712, b: 106712});
+        // check that the pkSigning object corresponds to the pubKey
+        const signature = pkSigning.sign("message");
+        const util = new global.Olm.Utility();
+        try {
+            util.ed25519_verify(pubKey, "message", signature);
+        } finally {
+            util.free();
+        }
     });
 
     it.each(types)("should request a key from the cache callback (if set)" +
@@ -217,6 +226,7 @@ describe.each([
     ["MemoryCryptoStore", () => {
         const store = new IndexedDBCryptoStore(undefined, "tests");
         store._backend = new MemoryCryptoStore();
+        store._backendPromise = Promise.resolve(store._backend);
         return store;
     }],
 ])("CrossSigning > createCryptoStoreCacheCallbacks [%s]", function(name, dbFactory) {
@@ -232,8 +242,9 @@ describe.each([
 
     it("should cache data to the store and retrieve it", async () => {
         await store.startup();
+        const olmDevice = new OlmDevice(store);
         const { getCrossSigningKeyCache, storeCrossSigningKeyCache } =
-          createCryptoStoreCacheCallbacks(store);
+              createCryptoStoreCacheCallbacks(store, olmDevice);
         await storeCrossSigningKeyCache("self_signing", testKey);
 
         // If we've not saved anything, don't expect anything
@@ -242,6 +253,6 @@ describe.each([
         expect(nokey).toBeNull();
 
         const key = await getCrossSigningKeyCache("self_signing", "");
-        expect(key).toEqual(testKey);
+        expect(new Uint8Array(key)).toEqual(testKey);
     });
 });

spec/unit/crypto/DeviceList.spec.js

@@ -16,10 +16,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {logger} from "../../../src/logger";
+import { logger } from "../../../src/logger";
 import * as utils from "../../../src/utils";
-import {MemoryCryptoStore} from "../../../src/crypto/store/memory-crypto-store";
-import {DeviceList} from "../../../src/crypto/DeviceList";
+import { MemoryCryptoStore } from "../../../src/crypto/store/memory-crypto-store";
+import { DeviceList } from "../../../src/crypto/DeviceList";
 
 const signedDeviceList = {
     "failures": {},
@@ -51,6 +51,36 @@ const signedDeviceList = {
     },
 };
 
+const signedDeviceList2 = {
+    "failures": {},
+    "device_keys": {
+        "@test2:sw1v.org": {
+            "QJVRHWAKGH": {
+                "signatures": {
+                    "@test2:sw1v.org": {
+                        "ed25519:QJVRHWAKGH":
+                            "w1xxdLe1iIqzEFHLRVYQeuiM6t2N2ZRiI8s5nDKxf054BP8" +
+                            "1CPEX/AQXh5BhkKAVMlKnwg4T9zU1/wBALeajk3",
+                    },
+                },
+                "user_id": "@test2:sw1v.org",
+                "keys": {
+                    "ed25519:QJVRHWAKGH":
+                        "Ig0/C6T+bBII1l2By2Wnnvtjp1nm/iXBlLU5/QESFXL",
+                    "curve25519:QJVRHWAKGH":
+                        "YR3eQnUvTQzGlWih4rsmJkKxpDxzgkgIgsBd1DEZIbm",
+                },
+                "algorithms": [
+                    "m.olm.v1.curve25519-aes-sha2",
+                    "m.megolm.v1.aes-sha2",
+                ],
+                "device_id": "QJVRHWAKGH",
+                "unsigned": {},
+            },
+        },
+    },
+};
+
 describe('DeviceList', function() {
     let downloadSpy;
     let cryptoStore;
@@ -69,14 +99,16 @@ describe('DeviceList', function() {
         }
     });
 
-    function createTestDeviceList() {
+    function createTestDeviceList(keyDownloadChunkSize = 250) {
         const baseApis = {
             downloadKeysForUsers: downloadSpy,
+            getUserId: () => '@test1:sw1v.org',
+            deviceId: 'HGKAWHRVJQ',
         };
         const mockOlm = {
             verifySignature: function(key, message, signature) {},
         };
-        const dl = new DeviceList(baseApis, cryptoStore, mockOlm);
+        const dl = new DeviceList(baseApis, cryptoStore, mockOlm, keyDownloadChunkSize);
         deviceLists.push(dl);
         return dl;
     }
@@ -148,4 +180,30 @@ describe('DeviceList', function() {
             expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
         });
     });
+
+    it("should download device keys in batches", function() {
+        const dl = createTestDeviceList(1);
+
+        dl.startTrackingDeviceList('@test1:sw1v.org');
+        dl.startTrackingDeviceList('@test2:sw1v.org');
+
+        const queryDefer1 = utils.defer();
+        downloadSpy.mockReturnValueOnce(queryDefer1.promise);
+        const queryDefer2 = utils.defer();
+        downloadSpy.mockReturnValueOnce(queryDefer2.promise);
+
+        const prom1 = dl.refreshOutdatedDeviceLists();
+        expect(downloadSpy).toBeCalledTimes(2);
+        expect(downloadSpy).toHaveBeenNthCalledWith(1, ['@test1:sw1v.org'], {});
+        expect(downloadSpy).toHaveBeenNthCalledWith(2, ['@test2:sw1v.org'], {});
+        queryDefer1.resolve(utils.deepCopy(signedDeviceList));
+        queryDefer2.resolve(utils.deepCopy(signedDeviceList2));
+
+        return prom1.then(() => {
+            const storedKeys1 = dl.getRawStoredDevicesForUser('@test1:sw1v.org');
+            expect(Object.keys(storedKeys1)).toEqual(['HGKAWHRVJQ']);
+            const storedKeys2 = dl.getRawStoredDevicesForUser('@test2:sw1v.org');
+            expect(Object.keys(storedKeys2)).toEqual(['QJVRHWAKGH']);
+        });
+    });
 });

spec/unit/crypto/algorithms/megolm.spec.js

@@ -1,14 +1,14 @@
 import '../../../olm-loader';
 import * as algorithms from "../../../../src/crypto/algorithms";
-import {MemoryCryptoStore} from "../../../../src/crypto/store/memory-crypto-store";
-import {MockStorageApi} from "../../../MockStorageApi";
+import { MemoryCryptoStore } from "../../../../src/crypto/store/memory-crypto-store";
+import { MockStorageApi } from "../../../MockStorageApi";
 import * as testUtils from "../../../test-utils";
-import {OlmDevice} from "../../../../src/crypto/OlmDevice";
-import {Crypto} from "../../../../src/crypto";
-import {logger} from "../../../../src/logger";
-import {MatrixEvent} from "../../../../src/models/event";
-import {TestClient} from "../../../TestClient";
-import {Room} from "../../../../src/models/room";
+import { OlmDevice } from "../../../../src/crypto/OlmDevice";
+import { Crypto } from "../../../../src/crypto";
+import { logger } from "../../../../src/logger";
+import { MatrixEvent } from "../../../../src/models/event";
+import { TestClient } from "../../../TestClient";
+import { Room } from "../../../../src/models/room";
 import * as olmlib from "../../../../src/crypto/olmlib";
 
 const MegolmDecryption = algorithms.DECRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
@@ -50,7 +50,6 @@ describe("MegolmDecryption", function() {
             roomId: ROOM_ID,
         });
 
-
         // we stub out the olm encryption bits
         mockOlmLib = {};
         mockOlmLib.ensureOlmSessionsForDevices = jest.fn();
@@ -136,9 +135,9 @@ describe("MegolmDecryption", function() {
                 mockCrypto.getStoredDevice.mockReturnValue(deviceInfo);
 
                 mockOlmLib.ensureOlmSessionsForDevices.mockResolvedValue({
-                    '@alice:foo': {'alidevice': {
+                    '@alice:foo': { 'alidevice': {
                         sessionId: 'alisession',
-                    }},
+                    } },
                 });
 
                 const awaitEncryptForDevice = new Promise((res, rej) => {
@@ -258,6 +257,9 @@ describe("MegolmDecryption", function() {
         });
 
         it("re-uses sessions for sequential messages", async function() {
+            mockCrypto._backupManager = {
+                backupGroupSession: () => {},
+            };
             const mockStorage = new MockStorageApi();
             const cryptoStore = new MemoryCryptoStore(mockStorage);
 
@@ -297,6 +299,10 @@ describe("MegolmDecryption", function() {
                 },
             }));
 
+            mockCrypto.checkDeviceTrust.mockReturnValue({
+                isVerified: () => false,
+            });
+
             const megolmEncryption = new MegolmEncryption({
                 userId: '@user:id',
                 crypto: mockCrypto,
@@ -309,7 +315,7 @@ describe("MegolmDecryption", function() {
             });
             const mockRoom = {
                 getEncryptionTargetMembers: jest.fn().mockReturnValue(
-                    [{userId: "@alice:home.server"}],
+                    [{ userId: "@alice:home.server" }],
                 ),
                 getBlacklistUnverifiedDevices: jest.fn().mockReturnValue(false),
             };
@@ -359,9 +365,9 @@ describe("MegolmDecryption", function() {
             bobClient1.initCrypto(),
             bobClient2.initCrypto(),
         ]);
-        const aliceDevice = aliceClient._crypto._olmDevice;
-        const bobDevice1 = bobClient1._crypto._olmDevice;
-        const bobDevice2 = bobClient2._crypto._olmDevice;
+        const aliceDevice = aliceClient.crypto._olmDevice;
+        const bobDevice1 = bobClient1.crypto._olmDevice;
+        const bobDevice2 = bobClient2.crypto._olmDevice;
 
         const encryptionCfg = {
             "algorithm": "m.megolm.v1.aes-sha2",
@@ -369,7 +375,7 @@ describe("MegolmDecryption", function() {
         const roomId = "!someroom";
         const room = new Room(roomId, aliceClient, "@alice:example.com", {});
         room.getEncryptionTargetMembers = async function() {
-            return [{userId: "@bob:example.com"}];
+            return [{ userId: "@bob:example.com" }];
         };
         room.setBlacklistUnverifiedDevices(true);
         aliceClient.store.storeRoom(room);
@@ -398,10 +404,10 @@ describe("MegolmDecryption", function() {
             },
         };
 
-        aliceClient._crypto._deviceList.storeDevicesForUser(
+        aliceClient.crypto._deviceList.storeDevicesForUser(
             "@bob:example.com", BOB_DEVICES,
         );
-        aliceClient._crypto._deviceList.downloadKeys = async function(userIds) {
+        aliceClient.crypto._deviceList.downloadKeys = async function(userIds) {
             return this._getDevicesFromStore(userIds);
         };
 
@@ -442,7 +448,7 @@ describe("MegolmDecryption", function() {
                 body: "secret",
             },
         });
-        await aliceClient._crypto.encryptEvent(event, room);
+        await aliceClient.crypto.encryptEvent(event, room);
 
         expect(run).toBe(true);
 
@@ -462,8 +468,8 @@ describe("MegolmDecryption", function() {
             aliceClient.initCrypto(),
             bobClient.initCrypto(),
         ]);
-        const aliceDevice = aliceClient._crypto._olmDevice;
-        const bobDevice = bobClient._crypto._olmDevice;
+        const aliceDevice = aliceClient.crypto._olmDevice;
+        const bobDevice = bobClient.crypto._olmDevice;
 
         const encryptionCfg = {
             "algorithm": "m.megolm.v1.aes-sha2",
@@ -502,10 +508,10 @@ describe("MegolmDecryption", function() {
             },
         };
 
-        aliceClient._crypto._deviceList.storeDevicesForUser(
+        aliceClient.crypto._deviceList.storeDevicesForUser(
             "@bob:example.com", BOB_DEVICES,
         );
-        aliceClient._crypto._deviceList.downloadKeys = async function(userIds) {
+        aliceClient.crypto._deviceList.downloadKeys = async function(userIds) {
             return this._getDevicesFromStore(userIds);
         };
 
@@ -540,7 +546,7 @@ describe("MegolmDecryption", function() {
             event_id: "$event",
             content: {},
         });
-        await aliceClient._crypto.encryptEvent(event, aliceRoom);
+        await aliceClient.crypto.encryptEvent(event, aliceRoom);
         await sendPromise;
     });
 
@@ -555,11 +561,11 @@ describe("MegolmDecryption", function() {
             aliceClient.initCrypto(),
             bobClient.initCrypto(),
         ]);
-        const bobDevice = bobClient._crypto._olmDevice;
+        const bobDevice = bobClient.crypto._olmDevice;
 
         const roomId = "!someroom";
 
-        aliceClient._crypto._onToDeviceEvent(new MatrixEvent({
+        aliceClient.crypto._onToDeviceEvent(new MatrixEvent({
             type: "org.matrix.room_key.withheld",
             sender: "@bob:example.com",
             content: {
@@ -572,7 +578,7 @@ describe("MegolmDecryption", function() {
             },
         }));
 
-        await expect(aliceClient._crypto.decryptEvent(new MatrixEvent({
+        await expect(aliceClient.crypto.decryptEvent(new MatrixEvent({
             type: "m.room.encrypted",
             sender: "@bob:example.com",
             event_id: "$event",
@@ -598,14 +604,14 @@ describe("MegolmDecryption", function() {
             aliceClient.initCrypto(),
             bobClient.initCrypto(),
         ]);
-        aliceClient._crypto.downloadKeys = async () => {};
-        const bobDevice = bobClient._crypto._olmDevice;
+        aliceClient.crypto.downloadKeys = async () => {};
+        const bobDevice = bobClient.crypto._olmDevice;
 
         const roomId = "!someroom";
 
         const now = Date.now();
 
-        aliceClient._crypto._onToDeviceEvent(new MatrixEvent({
+        aliceClient.crypto._onToDeviceEvent(new MatrixEvent({
             type: "org.matrix.room_key.withheld",
             sender: "@bob:example.com",
             content: {
@@ -622,7 +628,7 @@ describe("MegolmDecryption", function() {
             setTimeout(resolve, 100);
         });
 
-        await expect(aliceClient._crypto.decryptEvent(new MatrixEvent({
+        await expect(aliceClient.crypto.decryptEvent(new MatrixEvent({
             type: "m.room.encrypted",
             sender: "@bob:example.com",
             event_id: "$event",
@@ -649,15 +655,15 @@ describe("MegolmDecryption", function() {
             aliceClient.initCrypto(),
             bobClient.initCrypto(),
         ]);
-        const bobDevice = bobClient._crypto._olmDevice;
-        aliceClient._crypto.downloadKeys = async () => {};
+        const bobDevice = bobClient.crypto._olmDevice;
+        aliceClient.crypto.downloadKeys = async () => {};
 
         const roomId = "!someroom";
 
         const now = Date.now();
 
         // pretend we got an event that we can't decrypt
-        aliceClient._crypto._onToDeviceEvent(new MatrixEvent({
+        aliceClient.crypto._onToDeviceEvent(new MatrixEvent({
             type: "m.room.encrypted",
             sender: "@bob:example.com",
             content: {
@@ -672,7 +678,7 @@ describe("MegolmDecryption", function() {
             setTimeout(resolve, 100);
         });
 
-        await expect(aliceClient._crypto.decryptEvent(new MatrixEvent({
+        await expect(aliceClient.crypto.decryptEvent(new MatrixEvent({
             type: "m.room.encrypted",
             sender: "@bob:example.com",
             event_id: "$event",

spec/unit/crypto/algorithms/olm.spec.js

@@ -16,12 +16,12 @@ limitations under the License.
 */
 
 import '../../../olm-loader';
-import {MemoryCryptoStore} from "../../../../src/crypto/store/memory-crypto-store";
-import {MockStorageApi} from "../../../MockStorageApi";
-import {logger} from "../../../../src/logger";
-import {OlmDevice} from "../../../../src/crypto/OlmDevice";
+import { MemoryCryptoStore } from "../../../../src/crypto/store/memory-crypto-store";
+import { MockStorageApi } from "../../../MockStorageApi";
+import { logger } from "../../../../src/logger";
+import { OlmDevice } from "../../../../src/crypto/OlmDevice";
 import * as olmlib from "../../../../src/crypto/olmlib";
-import {DeviceInfo} from "../../../../src/crypto/deviceinfo";
+import { DeviceInfo } from "../../../../src/crypto/deviceinfo";
 
 function makeOlmDevice() {
     const mockStorage = new MockStorageApi();
@@ -190,5 +190,91 @@ describe("OlmDevice", function() {
             // new session and will have called claimOneTimeKeys
             expect(count).toBe(2);
         });
+
+        it("avoids deadlocks when two tasks are ensuring the same devices", async function() {
+            // This test checks whether `ensureOlmSessionsForDevices` properly
+            // handles multiple tasks in flight ensuring some set of devices in
+            // common without deadlocks.
+
+            let claimRequestCount = 0;
+            const baseApis = {
+                claimOneTimeKeys: () => {
+                    // simulate a very slow server (.5 seconds to respond)
+                    claimRequestCount++;
+                    return new Promise((resolve, reject) => {
+                        setTimeout(reject, 500);
+                    });
+                },
+            };
+
+            const deviceBobA = DeviceInfo.fromStorage({
+                keys: {
+                    "curve25519:BOB-A": "akey",
+                },
+            }, "BOB-A");
+            const deviceBobB = DeviceInfo.fromStorage({
+                keys: {
+                    "curve25519:BOB-B": "bkey",
+                },
+            }, "BOB-B");
+
+            // There's no required ordering of devices per user, so here we
+            // create two different orderings so that each task reserves a
+            // device the other task needs before continuing.
+            const devicesByUserAB = {
+                "@bob:example.com": [
+                    deviceBobA,
+                    deviceBobB,
+                ],
+            };
+            const devicesByUserBA = {
+                "@bob:example.com": [
+                    deviceBobB,
+                    deviceBobA,
+                ],
+            };
+
+            function alwaysSucceed(promise) {
+                // swallow any exception thrown by a promise, so that
+                // Promise.all doesn't abort
+                return promise.catch(() => {});
+            }
+
+            const task1 = alwaysSucceed(olmlib.ensureOlmSessionsForDevices(
+                aliceOlmDevice, baseApis, devicesByUserAB,
+            ));
+
+            // After a single tick through the first task, it should have
+            // claimed ownership of all devices to avoid deadlocking others.
+            expect(Object.keys(aliceOlmDevice._sessionsInProgress).length).toBe(2);
+
+            const task2 = alwaysSucceed(olmlib.ensureOlmSessionsForDevices(
+                aliceOlmDevice, baseApis, devicesByUserBA,
+            ));
+
+            // The second task should not have changed the ownership count, as
+            // it's waiting on the first task.
+            expect(Object.keys(aliceOlmDevice._sessionsInProgress).length).toBe(2);
+
+            // Track the tasks, but don't await them yet.
+            const promises = Promise.all([
+                task1,
+                task2,
+            ]);
+
+            await new Promise((resolve) => {
+                setTimeout(resolve, 200);
+            });
+
+            // After .2s, the first task should have made an initial claim request.
+            expect(claimRequestCount).toBe(1);
+
+            await promises;
+
+            // After waiting for both tasks to complete, the first task should
+            // have failed, so the second task should have tried to create a
+            // new session and will have called claimOneTimeKeys
+            expect(claimRequestCount).toBe(2);
+        });
     });
 });

spec/unit/crypto/backup.spec.js

@@ -16,17 +16,19 @@ limitations under the License.
 */
 
 import '../../olm-loader';
-import {logger} from "../../../src/logger";
+import { logger } from "../../../src/logger";
 import * as olmlib from "../../../src/crypto/olmlib";
-import {MatrixClient} from "../../../src/client";
-import {MatrixEvent} from "../../../src/models/event";
+import { MatrixClient } from "../../../src/client";
+import { MatrixEvent } from "../../../src/models/event";
 import * as algorithms from "../../../src/crypto/algorithms";
-import {WebStorageSessionStore} from "../../../src/store/session/webstorage";
-import {MemoryCryptoStore} from "../../../src/crypto/store/memory-crypto-store";
-import {MockStorageApi} from "../../MockStorageApi";
+import { WebStorageSessionStore } from "../../../src/store/session/webstorage";
+import { MemoryCryptoStore } from "../../../src/crypto/store/memory-crypto-store";
+import { MockStorageApi } from "../../MockStorageApi";
 import * as testUtils from "../../test-utils";
-import {OlmDevice} from "../../../src/crypto/OlmDevice";
-import {Crypto} from "../../../src/crypto";
+import { OlmDevice } from "../../../src/crypto/OlmDevice";
+import { Crypto } from "../../../src/crypto";
+import { resetCrossSigningKeys } from "./crypto-utils";
+import { BackupManager } from "../../../src/crypto/backup";
 
 const Olm = global.Olm;
 
@@ -72,7 +74,7 @@ const KEY_BACKUP_DATA = {
 };
 
 const BACKUP_INFO = {
-    algorithm: "m.megolm_backup.v1",
+    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
     version: 1,
     auth_data: {
         public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
@@ -137,6 +139,7 @@ describe("MegolmBackup", function() {
     let megolmDecryption;
     beforeEach(async function() {
         mockCrypto = testUtils.mock(Crypto, 'Crypto');
+        mockCrypto._backupManager = testUtils.mock(BackupManager, "BackupManager");
         mockCrypto.backupKey = new Olm.PkEncryption();
         mockCrypto.backupKey.set_recipient_key(
             "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
@@ -214,12 +217,14 @@ describe("MegolmBackup", function() {
             };
             mockCrypto.cancelRoomKeyRequest = function() {};
 
-            mockCrypto.backupGroupSession = jest.fn();
+            mockCrypto._backupManager = {
+                backupGroupSession: jest.fn(),
+            };
 
             return event.attemptDecryption(mockCrypto).then(() => {
                 return megolmDecryption.onRoomKeyEvent(event);
             }).then(() => {
-                expect(mockCrypto.backupGroupSession).toHaveBeenCalled();
+                expect(mockCrypto._backupManager.backupGroupSession).toHaveBeenCalled();
             });
         });
 
@@ -263,7 +268,7 @@ describe("MegolmBackup", function() {
                 })
                 .then(() => {
                     client.enableKeyBackup({
-                        algorithm: "m.megolm_backup.v1",
+                        algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
                         version: 1,
                         auth_data: {
                             public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
@@ -271,7 +276,7 @@ describe("MegolmBackup", function() {
                     });
                     let numCalls = 0;
                     return new Promise((resolve, reject) => {
-                        client._http.authedRequest = function(
+                        client.http.authedRequest = function(
                             callback, method, path, queryParams, data, opts,
                         ) {
                             ++numCalls;
@@ -291,12 +296,9 @@ describe("MegolmBackup", function() {
                             resolve();
                             return Promise.resolve({});
                         };
-                        client._crypto.backupGroupSession(
-                            "roomId",
+                        client.crypto._backupManager.backupGroupSession(
                             "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI",
-                            [],
                             groupSession.session_id(),
-                            groupSession.session_key(),
                         );
                     }).then(() => {
                         expect(numCalls).toBe(1);
@@ -332,41 +334,50 @@ describe("MegolmBackup", function() {
             client.on("crossSigning.getKey", function(e) {
                 e.done(privateKeys[e.type]);
             });
-            await client.resetCrossSigningKeys();
+            await resetCrossSigningKeys(client);
             let numCalls = 0;
-            await new Promise((resolve, reject) => {
-                client._http.authedRequest = function(
-                    callback, method, path, queryParams, data, opts,
-                ) {
-                    ++numCalls;
-                    expect(numCalls).toBeLessThanOrEqual(1);
-                    if (numCalls >= 2) {
-                        // exit out of retry loop if there's something wrong
-                        reject(new Error("authedRequest called too many timmes"));
-                        return Promise.resolve({});
-                    }
-                    expect(method).toBe("POST");
-                    expect(path).toBe("/room_keys/version");
-                    try {
-                        // make sure auth_data is signed by the master key
-                        olmlib.pkVerify(
-                            data.auth_data, client.getCrossSigningId(), "@alice:bar",
-                        );
-                    } catch (e) {
-                        reject(e);
-                        return Promise.resolve({});
-                    }
-                    resolve();
-                    return Promise.resolve({});
-                };
+            await Promise.all([
+                new Promise((resolve, reject) => {
+                    let backupInfo;
+                    client.http.authedRequest = function(
+                        callback, method, path, queryParams, data, opts,
+                    ) {
+                        ++numCalls;
+                        expect(numCalls).toBeLessThanOrEqual(2);
+                        if (numCalls === 1) {
+                            expect(method).toBe("POST");
+                            expect(path).toBe("/room_keys/version");
+                            try {
+                                // make sure auth_data is signed by the master key
+                                olmlib.pkVerify(
+                                    data.auth_data, client.getCrossSigningId(), "@alice:bar",
+                                );
+                            } catch (e) {
+                                reject(e);
+                                return Promise.resolve({});
+                            }
+                            backupInfo = data;
+                            return Promise.resolve({});
+                        } else if (numCalls === 2) {
+                            expect(method).toBe("GET");
+                            expect(path).toBe("/room_keys/version");
+                            resolve();
+                            return Promise.resolve(backupInfo);
+                        } else {
+                            // exit out of retry loop if there's something wrong
+                            reject(new Error("authedRequest called too many times"));
+                            return Promise.resolve({});
+                        }
+                    };
+                }),
                 client.createKeyBackupVersion({
-                    algorithm: "m.megolm_backup.v1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
                     auth_data: {
                         public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
                     },
-                });
-            });
-            expect(numCalls).toBe(1);
+                }),
+            ]);
+            expect(numCalls).toBe(2);
         });
 
         it('retries when a backup fails', function() {
@@ -433,7 +444,7 @@ describe("MegolmBackup", function() {
                 })
                 .then(() => {
                     client.enableKeyBackup({
-                        algorithm: "foobar",
+                        algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
                         version: 1,
                         auth_data: {
                             public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
@@ -441,7 +452,7 @@ describe("MegolmBackup", function() {
                     });
                     let numCalls = 0;
                     return new Promise((resolve, reject) => {
-                        client._http.authedRequest = function(
+                        client.http.authedRequest = function(
                             callback, method, path, queryParams, data, opts,
                         ) {
                             ++numCalls;
@@ -467,12 +478,9 @@ describe("MegolmBackup", function() {
                                 );
                             }
                         };
-                        client._crypto.backupGroupSession(
-                            "roomId",
+                        client.crypto._backupManager.backupGroupSession(
                             "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI",
-                            [],
                             groupSession.session_id(),
-                            groupSession.session_key(),
                         );
                     }).then(() => {
                         expect(numCalls).toBe(2);
@@ -505,7 +513,7 @@ describe("MegolmBackup", function() {
         });
 
         it('can restore from backup', function() {
-            client._http.authedRequest = function() {
+            client.http.authedRequest = function() {
                 return Promise.resolve(KEY_BACKUP_DATA);
             };
             return client.restoreKeyBackupWithRecoveryKey(
@@ -517,11 +525,12 @@ describe("MegolmBackup", function() {
                 return megolmDecryption.decryptEvent(ENCRYPTED_EVENT);
             }).then((res) => {
                 expect(res.clearEvent.content).toEqual('testytest');
+                expect(res.untrusted).toBeTruthy(); // keys from backup are untrusted
             });
         });
 
         it('can restore backup by room', function() {
-            client._http.authedRequest = function() {
+            client.http.authedRequest = function() {
                 return Promise.resolve({
                     rooms: {
                         [ROOM_ID]: {
@@ -541,5 +550,47 @@ describe("MegolmBackup", function() {
                 expect(res.clearEvent.content).toEqual('testytest');
             });
         });
+
+        it('has working cache functions', async function() {
+            const key = Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]);
+            await client.crypto.storeSessionBackupPrivateKey(key);
+            const result = await client.crypto.getSessionBackupPrivateKey();
+            expect(new Uint8Array(result)).toEqual(key);
+        });
+
+        it('caches session backup keys as it encounters them', async function() {
+            const cachedNull = await client.crypto.getSessionBackupPrivateKey();
+            expect(cachedNull).toBeNull();
+            client.http.authedRequest = function() {
+                return Promise.resolve(KEY_BACKUP_DATA);
+            };
+            await new Promise((resolve) => {
+                client.restoreKeyBackupWithRecoveryKey(
+                    "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d",
+                    ROOM_ID,
+                    SESSION_ID,
+                    BACKUP_INFO,
+                    { cacheCompleteCallback: resolve },
+                );
+            });
+            const cachedKey = await client.crypto.getSessionBackupPrivateKey();
+            expect(cachedKey).not.toBeNull();
+        });
+
+        it("fails if an known algorithm is used", async function() {
+            const BAD_BACKUP_INFO = Object.assign({}, BACKUP_INFO, {
+                algorithm: "this.algorithm.does.not.exist",
+            });
+            client.http.authedRequest = function() {
+                return Promise.resolve(KEY_BACKUP_DATA);
+            };
+
+            await expect(client.restoreKeyBackupWithRecoveryKey(
+                "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d",
+                ROOM_ID,
+                SESSION_ID,
+                BAD_BACKUP_INFO,
+            )).rejects.toThrow();
+        });
     });
 });

spec/unit/crypto/cross-signing.spec.js

@@ -18,8 +18,11 @@ limitations under the License.
 import '../../olm-loader';
 import anotherjson from 'another-json';
 import * as olmlib from "../../../src/crypto/olmlib";
-import {TestClient} from '../../TestClient';
-import {HttpResponse, setHttpResponses} from '../../test-utils';
+import { TestClient } from '../../TestClient';
+import { HttpResponse, setHttpResponses } from '../../test-utils';
+import { resetCrossSigningKeys } from "./crypto-utils";
+import { MatrixError } from '../../../src/http-api';
+import { logger } from '../../../src/logger';
 
 async function makeTestClient(userInfo, options, keys) {
     if (!keys) keys = {};
@@ -47,7 +50,7 @@ async function makeTestClient(userInfo, options, keys) {
 
 describe("Cross Signing", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 
@@ -57,30 +60,85 @@ describe("Cross Signing", function() {
 
     it("should sign the master key with the device key", async function() {
         const alice = await makeTestClient(
-            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
         alice.uploadDeviceSigningKeys = jest.fn(async (auth, keys) => {
             await olmlib.verifySignature(
-                alice._crypto._olmDevice, keys.master_key, "@alice:example.com",
-                "Osborne2", alice._crypto._olmDevice.deviceEd25519Key,
+                alice.crypto._olmDevice, keys.master_key, "@alice:example.com",
+                "Osborne2", alice.crypto._olmDevice.deviceEd25519Key,
             );
         });
         alice.uploadKeySignatures = async () => {};
+        alice.setAccountData = async () => {};
+        alice.getAccountDataFromServer = async () => {};
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await alice.bootstrapCrossSigning({
+            authUploadDeviceSigningKeys: async func => await func({}),
+        });
         expect(alice.uploadDeviceSigningKeys).toHaveBeenCalled();
     });
 
+    it("should abort bootstrap if device signing auth fails", async function() {
+        const alice = await makeTestClient(
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
+        );
+        alice.uploadDeviceSigningKeys = async (auth, keys) => {
+            const errorResponse = {
+                session: "sessionId",
+                flows: [
+                    {
+                        stages: [
+                            "m.login.password",
+                        ],
+                    },
+                ],
+                params: {},
+            };
+
+            // If we're not just polling for flows, add on error rejecting the
+            // auth attempt.
+            if (auth) {
+                Object.assign(errorResponse, {
+                    completed: [],
+                    error: "Invalid password",
+                    errcode: "M_FORBIDDEN",
+                });
+            }
+
+            const error = new MatrixError(errorResponse);
+            error.httpStatus == 401;
+            throw error;
+        };
+        alice.uploadKeySignatures = async () => {};
+        alice.setAccountData = async () => {};
+        alice.getAccountDataFromServer = async () => { };
+        const authUploadDeviceSigningKeys = async func => await func({});
+
+        // Try bootstrap, expecting `authUploadDeviceSigningKeys` to pass
+        // through failure, stopping before actually applying changes.
+        let bootstrapDidThrow = false;
+        try {
+            await alice.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys,
+            });
+        } catch (e) {
+            if (e.errcode === "M_FORBIDDEN") {
+                bootstrapDidThrow = true;
+            }
+        }
+        expect(bootstrapDidThrow).toBeTruthy();
+    });
+
     it("should upload a signature when a user is verified", async function() {
         const alice = await makeTestClient(
-            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
         alice.uploadDeviceSigningKeys = async () => {};
         alice.uploadKeySignatures = async () => {};
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         // Alice downloads Bob's device key
-        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+        alice.crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
             keys: {
                 master: {
                     user_id: "@bob:example.com",
@@ -117,7 +175,7 @@ describe("Cross Signing", function() {
         ]);
 
         const alice = await makeTestClient(
-            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
             {
                 cryptoCallbacks: {
                     // will be called to sign our own device
@@ -135,30 +193,36 @@ describe("Cross Signing", function() {
         const keyChangePromise = new Promise((resolve, reject) => {
             alice.once("crossSigning.keysChanged", async (e) => {
                 resolve(e);
-                await alice.checkOwnCrossSigningTrust();
+                await alice.checkOwnCrossSigningTrust({
+                    allowPrivateKeyRequests: true,
+                });
             });
         });
 
         const uploadSigsPromise = new Promise((resolve, reject) => {
             alice.uploadKeySignatures = jest.fn(async (content) => {
-                await olmlib.verifySignature(
-                    alice._crypto._olmDevice,
-                    content["@alice:example.com"][
-                        "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
-                    ],
-                    "@alice:example.com",
-                    "Osborne2", alice._crypto._olmDevice.deviceEd25519Key,
-                );
-                olmlib.pkVerify(
-                    content["@alice:example.com"]["Osborne2"],
-                    "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ",
-                    "@alice:example.com",
-                );
-                resolve();
+                try {
+                    await olmlib.verifySignature(
+                        alice.crypto._olmDevice,
+                        content["@alice:example.com"][
+                            "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
+                            ],
+                        "@alice:example.com",
+                        "Osborne2", alice.crypto._olmDevice.deviceEd25519Key,
+                    );
+                    olmlib.pkVerify(
+                        content["@alice:example.com"]["Osborne2"],
+                        "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ",
+                        "@alice:example.com",
+                    );
+                    resolve();
+                } catch (e) {
+                    reject(e);
+                }
             });
         });
 
-        const deviceInfo = alice._crypto._deviceList._devices["@alice:example.com"]
+        const deviceInfo = alice.crypto._deviceList._devices["@alice:example.com"]
             .Osborne2;
         const aliceDevice = {
             user_id: "@alice:example.com",
@@ -166,7 +230,7 @@ describe("Cross Signing", function() {
         };
         aliceDevice.keys = deviceInfo.keys;
         aliceDevice.algorithms = deviceInfo.algorithms;
-        await alice._crypto._signObject(aliceDevice);
+        await alice.crypto._signObject(aliceDevice);
         olmlib.pkSign(aliceDevice, selfSigningKey, "@alice:example.com");
 
         // feed sync result that includes master key, ssk, device key
@@ -268,12 +332,12 @@ describe("Cross Signing", function() {
 
     it("should use trust chain to determine device verification", async function() {
         const alice = await makeTestClient(
-            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
         alice.uploadDeviceSigningKeys = async () => {};
         alice.uploadKeySignatures = async () => {};
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         // Alice downloads Bob's ssk and device key
         const bobMasterSigning = new global.Olm.PkSigning();
         const bobMasterPrivkey = bobMasterSigning.generate_seed();
@@ -294,7 +358,7 @@ describe("Cross Signing", function() {
                 ["ed25519:" + bobMasterPubkey]: sskSig,
             },
         };
-        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+        alice.crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
             keys: {
                 master: {
                     user_id: "@bob:example.com",
@@ -323,7 +387,7 @@ describe("Cross Signing", function() {
                 ["ed25519:" + bobPubkey]: sig,
             },
         };
-        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+        alice.crypto._deviceList.storeDevicesForUser("@bob:example.com", {
             Dynabook: bobDevice,
         });
         // Bob's device key should be TOFU
@@ -353,17 +417,17 @@ describe("Cross Signing", function() {
     it("should trust signatures received from other devices", async function() {
         const aliceKeys = {};
         const alice = await makeTestClient(
-            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
             null,
             aliceKeys,
         );
-        alice._crypto._deviceList.startTrackingDeviceList("@bob:example.com");
-        alice._crypto._deviceList.stopTrackingAllDeviceLists = () => {};
+        alice.crypto._deviceList.startTrackingDeviceList("@bob:example.com");
+        alice.crypto._deviceList.stopTrackingAllDeviceLists = () => {};
         alice.uploadDeviceSigningKeys = async () => {};
         alice.uploadKeySignatures = async () => {};
 
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
 
         const selfSigningKey = new Uint8Array([
             0x1e, 0xf4, 0x01, 0x6d, 0x4f, 0xa1, 0x73, 0x66,
@@ -373,14 +437,14 @@ describe("Cross Signing", function() {
         ]);
 
         const keyChangePromise = new Promise((resolve, reject) => {
-            alice._crypto._deviceList.once("userCrossSigningUpdated", (userId) => {
+            alice.crypto._deviceList.once("userCrossSigningUpdated", (userId) => {
                 if (userId === "@bob:example.com") {
                     resolve();
                 }
             });
         });
 
-        const deviceInfo = alice._crypto._deviceList._devices["@alice:example.com"]
+        const deviceInfo = alice.crypto._deviceList._devices["@alice:example.com"]
             .Osborne2;
         const aliceDevice = {
             user_id: "@alice:example.com",
@@ -388,7 +452,7 @@ describe("Cross Signing", function() {
         };
         aliceDevice.keys = deviceInfo.keys;
         aliceDevice.algorithms = deviceInfo.algorithms;
-        await alice._crypto._signObject(aliceDevice);
+        await alice.crypto._signObject(aliceDevice);
 
         const bobOlmAccount = new global.Olm.Account();
         bobOlmAccount.create();
@@ -515,12 +579,12 @@ describe("Cross Signing", function() {
 
     it("should dis-trust an unsigned device", async function() {
         const alice = await makeTestClient(
-            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
         alice.uploadDeviceSigningKeys = async () => {};
         alice.uploadKeySignatures = async () => {};
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         // Alice downloads Bob's ssk and device key
         // (NOTE: device key is not signed by ssk)
         const bobMasterSigning = new global.Olm.PkSigning();
@@ -542,7 +606,7 @@ describe("Cross Signing", function() {
                 ["ed25519:" + bobMasterPubkey]: sskSig,
             },
         };
-        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+        alice.crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
             keys: {
                 master: {
                     user_id: "@bob:example.com",
@@ -565,7 +629,7 @@ describe("Cross Signing", function() {
                 "ed25519:Dynabook": "someOtherPubkey",
             },
         };
-        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+        alice.crypto._deviceList.storeDevicesForUser("@bob:example.com", {
             Dynabook: bobDevice,
         });
         // Bob's device key should be untrusted
@@ -584,11 +648,11 @@ describe("Cross Signing", function() {
 
     it("should dis-trust a user when their ssk changes", async function() {
         const alice = await makeTestClient(
-            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
         alice.uploadDeviceSigningKeys = async () => {};
         alice.uploadKeySignatures = async () => {};
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         // Alice downloads Bob's keys
         const bobMasterSigning = new global.Olm.PkSigning();
         const bobMasterPrivkey = bobMasterSigning.generate_seed();
@@ -609,7 +673,7 @@ describe("Cross Signing", function() {
                 ["ed25519:" + bobMasterPubkey]: sskSig,
             },
         };
-        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+        alice.crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
             keys: {
                 master: {
                     user_id: "@bob:example.com",
@@ -637,7 +701,7 @@ describe("Cross Signing", function() {
         bobDevice.signatures = {};
         bobDevice.signatures["@bob:example.com"] = {};
         bobDevice.signatures["@bob:example.com"]["ed25519:" + bobPubkey] = sig;
-        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+        alice.crypto._deviceList.storeDevicesForUser("@bob:example.com", {
             Dynabook: bobDevice,
         });
         // Alice verifies Bob's SSK
@@ -669,7 +733,7 @@ describe("Cross Signing", function() {
                 ["ed25519:" + bobMasterPubkey2]: sskSig2,
             },
         };
-        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+        alice.crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
             keys: {
                 master: {
                     user_id: "@bob:example.com",
@@ -706,7 +770,7 @@ describe("Cross Signing", function() {
         // Alice gets new signature for device
         const sig2 = bobSigning2.sign(bobDeviceString);
         bobDevice.signatures["@bob:example.com"]["ed25519:" + bobPubkey2] = sig2;
-        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+        alice.crypto._deviceList.storeDevicesForUser("@bob:example.com", {
             Dynabook: bobDevice,
         });
 
@@ -722,7 +786,7 @@ describe("Cross Signing", function() {
         let upgradeResolveFunc;
 
         const alice = await makeTestClient(
-            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
             {
                 cryptoCallbacks: {
                     shouldUpgradeDeviceVerifications: (verifs) => {
@@ -734,27 +798,27 @@ describe("Cross Signing", function() {
             },
         );
         const bob = await makeTestClient(
-            {userId: "@bob:example.com", deviceId: "Dynabook"},
+            { userId: "@bob:example.com", deviceId: "Dynabook" },
         );
 
         bob.uploadDeviceSigningKeys = async () => {};
         bob.uploadKeySignatures = async () => {};
         // set Bob's cross-signing key
-        await bob.resetCrossSigningKeys();
-        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+        await resetCrossSigningKeys(bob);
+        alice.crypto._deviceList.storeDevicesForUser("@bob:example.com", {
             Dynabook: {
                 algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
                 keys: {
-                    "curve25519:Dynabook": bob._crypto._olmDevice.deviceCurve25519Key,
-                    "ed25519:Dynabook": bob._crypto._olmDevice.deviceEd25519Key,
+                    "curve25519:Dynabook": bob.crypto._olmDevice.deviceCurve25519Key,
+                    "ed25519:Dynabook": bob.crypto._olmDevice.deviceEd25519Key,
                 },
                 verified: 1,
                 known: true,
             },
         });
-        alice._crypto._deviceList.storeCrossSigningForUser(
+        alice.crypto._deviceList.storeCrossSigningForUser(
             "@bob:example.com",
-            bob._crypto._crossSigningInfo.toStorage(),
+            bob.crypto._crossSigningInfo.toStorage(),
         );
 
         alice.uploadDeviceSigningKeys = async () => {};
@@ -766,7 +830,7 @@ describe("Cross Signing", function() {
         let upgradePromise = new Promise((resolve) => {
             upgradeResolveFunc = resolve;
         });
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         await upgradePromise;
 
         const bobTrust = alice.checkUserTrust("@bob:example.com");
@@ -774,7 +838,7 @@ describe("Cross Signing", function() {
         expect(bobTrust.isTofu()).toBeTruthy();
 
         // "forget" that Bob is trusted
-        delete alice._crypto._deviceList._crossSigningInfo["@bob:example.com"]
+        delete alice.crypto._deviceList._crossSigningInfo["@bob:example.com"]
             .keys.master.signatures["@alice:example.com"];
 
         const bobTrust2 = alice.checkUserTrust("@bob:example.com");
@@ -784,9 +848,9 @@ describe("Cross Signing", function() {
         upgradePromise = new Promise((resolve) => {
             upgradeResolveFunc = resolve;
         });
-        alice._crypto._deviceList.emit("userCrossSigningUpdated", "@bob:example.com");
+        alice.crypto._deviceList.emit("userCrossSigningUpdated", "@bob:example.com");
         await new Promise((resolve) => {
-            alice._crypto.on("userTrustStatusChanged", resolve);
+            alice.crypto.on("userTrustStatusChanged", resolve);
         });
         await upgradePromise;
 

spec/unit/crypto/crypto-utils.js

@@ -0,0 +1,43 @@
+import { IndexedDBCryptoStore } from '../../../src/crypto/store/indexeddb-crypto-store';
+
+// needs to be phased out and replaced with bootstrapSecretStorage,
+// but that is doing too much extra stuff for it to be an easy transition.
+export async function resetCrossSigningKeys(client, {
+    level,
+    authUploadDeviceSigningKeys = async func => await func(),
+} = {}) {
+    const crypto = client.crypto;
+
+    const oldKeys = Object.assign({}, crypto._crossSigningInfo.keys);
+    try {
+        await crypto._crossSigningInfo.resetKeys(level);
+        await crypto._signObject(crypto._crossSigningInfo.keys.master);
+        // write a copy locally so we know these are trusted keys
+        await crypto._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+            (txn) => {
+                crypto._cryptoStore.storeCrossSigningKeys(
+                    txn, crypto._crossSigningInfo.keys);
+            },
+        );
+    } catch (e) {
+        // If anything failed here, revert the keys so we know to try again from the start
+        // next time.
+        crypto._crossSigningInfo.keys = oldKeys;
+        throw e;
+    }
+    crypto._baseApis.emit("crossSigning.keysChanged", {});
+    await crypto._afterCrossSigningLocalKeyChange();
+}
+
+export async function createSecretStorageKey() {
+    const decryption = new global.Olm.PkDecryption();
+    const storagePublicKey = decryption.generate_key();
+    const storagePrivateKey = decryption.get_private_key();
+    decryption.free();
+    return {
+        // `pubkey` not used anymore with symmetric 4S
+        keyInfo: { pubkey: storagePublicKey },
+        privateKey: storagePrivateKey,
+    };
+}

spec/unit/crypto/outgoing-room-key-requests.spec.js

@@ -0,0 +1,87 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {
+    IndexedDBCryptoStore,
+} from '../../../src/crypto/store/indexeddb-crypto-store';
+import { MemoryCryptoStore } from '../../../src/crypto/store/memory-crypto-store';
+import 'fake-indexeddb/auto';
+import 'jest-localstorage-mock';
+
+import {
+    ROOM_KEY_REQUEST_STATES,
+} from '../../../src/crypto/OutgoingRoomKeyRequestManager';
+
+const requests = [
+    {
+        requestId: "A",
+        requestBody: { session_id: "A", room_id: "A" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "B",
+        requestBody: { session_id: "B", room_id: "B" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "C",
+        requestBody: { session_id: "C", room_id: "C" },
+        state: ROOM_KEY_REQUEST_STATES.UNSENT,
+    },
+];
+
+describe.each([
+    ["IndexedDBCryptoStore",
+     () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+    ["LocalStorageCryptoStore",
+     () => new IndexedDBCryptoStore(undefined, "tests")],
+    ["MemoryCryptoStore", () => {
+        const store = new IndexedDBCryptoStore(undefined, "tests");
+        store._backend = new MemoryCryptoStore();
+        store._backendPromise = Promise.resolve(store._backend);
+        return store;
+    }],
+])("Outgoing room key requests [%s]", function(name, dbFactory) {
+    let store;
+
+    beforeAll(async () => {
+        store = dbFactory();
+        await store.startup();
+        await Promise.all(requests.map((request) =>
+            store.getOrAddOutgoingRoomKeyRequest(request),
+        ));
+    });
+
+    it("getAllOutgoingRoomKeyRequestsByState retrieves all entries in a given state",
+    async () => {
+        const r = await
+            store.getAllOutgoingRoomKeyRequestsByState(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(r).toHaveLength(2);
+        requests.filter((e) => e.state == ROOM_KEY_REQUEST_STATES.SENT).forEach((e) => {
+            expect(r).toContainEqual(e);
+        });
+    });
+
+    test("getOutgoingRoomKeyRequestByState retrieves any entry in a given state",
+    async () => {
+        const r =
+            await store.getOutgoingRoomKeyRequestByState([ROOM_KEY_REQUEST_STATES.SENT]);
+        expect(r).not.toBeNull();
+        expect(r).not.toBeUndefined();
+        expect(r.state).toEqual(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(requests).toContainEqual(r);
+    });
+});

spec/unit/crypto/secrets.spec.js

@@ -16,10 +16,22 @@ limitations under the License.
 
 import '../../olm-loader';
 import * as olmlib from "../../../src/crypto/olmlib";
-import {SECRET_STORAGE_ALGORITHM_V1} from "../../../src/crypto/SecretStorage";
-import {MatrixEvent} from "../../../src/models/event";
-import {TestClient} from '../../TestClient';
-import {makeTestClients} from './verification/util';
+import { SECRET_STORAGE_ALGORITHM_V1_AES } from "../../../src/crypto/SecretStorage";
+import { MatrixEvent } from "../../../src/models/event";
+import { TestClient } from '../../TestClient';
+import { makeTestClients } from './verification/util';
+import { encryptAES } from "../../../src/crypto/aes";
+import { resetCrossSigningKeys, createSecretStorageKey } from "./crypto-utils";
+import { logger } from '../../../src/logger';
+
+import * as utils from "../../../src/utils";
+
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    logger.log('nodejs was compiled without crypto support');
+}
 
 async function makeTestClient(userInfo, options) {
     const client = (new TestClient(
@@ -35,14 +47,21 @@ async function makeTestClient(userInfo, options) {
     await client.initCrypto();
 
     // No need to download keys for these tests
-    client._crypto.downloadKeys = async function() {};
+    client.crypto.downloadKeys = async function() {};
 
     return client;
 }
 
+// Wrapper around pkSign to return a signed object. pkSign returns the
+// signature, rather than the signed object.
+function sign(obj, key, userId) {
+    olmlib.pkSign(obj, key, userId);
+    return obj;
+}
+
 describe("Secrets", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 
@@ -51,9 +70,8 @@ describe("Secrets", function() {
     });
 
     it("should store and retrieve a secret", async function() {
-        const decryption = new global.Olm.PkDecryption();
-        const pubkey = decryption.generate_key();
-        const privkey = decryption.get_private_key();
+        const key = new Uint8Array(16);
+        for (let i = 0; i < 16; i++) key[i] = i;
 
         const signing = new global.Olm.PkSigning();
         const signingKey = signing.generate_seed();
@@ -69,11 +87,11 @@ describe("Secrets", function() {
 
         const getKey = jest.fn(e => {
             expect(Object.keys(e.keys)).toEqual(["abc"]);
-            return ['abc', privkey];
+            return ['abc', key];
         });
 
         const alice = await makeTestClient(
-            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
             {
                 cryptoCallbacks: {
                     getCrossSigningKey: t => signingKey,
@@ -81,11 +99,11 @@ describe("Secrets", function() {
                 },
             },
         );
-        alice._crypto._crossSigningInfo.setKeys({
+        alice.crypto._crossSigningInfo.setKeys({
             master: signingkeyInfo,
         });
 
-        const secretStorage = alice._crypto._secretStorage;
+        const secretStorage = alice.crypto._secretStorage;
 
         alice.setAccountData = async function(eventType, contents, callback) {
             alice.store.storeAccountDataEvents([
@@ -100,10 +118,9 @@ describe("Secrets", function() {
         };
 
         const keyAccountData = {
-            algorithm: SECRET_STORAGE_ALGORITHM_V1,
-            pubkey: pubkey,
+            algorithm: SECRET_STORAGE_ALGORITHM_V1_AES,
         };
-        await alice._crypto._crossSigningInfo.signObject(keyAccountData, 'master');
+        await alice.crypto._crossSigningInfo.signObject(keyAccountData, 'master');
 
         alice.store.storeAccountDataEvents([
             new MatrixEvent({
@@ -112,11 +129,11 @@ describe("Secrets", function() {
             }),
         ]);
 
-        expect(await secretStorage.isStored("foo")).toBe(false);
+        expect(await secretStorage.isStored("foo")).toBeFalsy();
 
         await secretStorage.store("foo", "bar", ["abc"]);
 
-        expect(await secretStorage.isStored("foo")).toBe(true);
+        expect(await secretStorage.isStored("foo")).toBeTruthy();
         expect(await secretStorage.get("foo")).toBe("bar");
 
         expect(getKey).toHaveBeenCalled();
@@ -124,7 +141,7 @@ describe("Secrets", function() {
 
     it("should throw if given a key that doesn't exist", async function() {
         const alice = await makeTestClient(
-            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
 
         try {
@@ -138,7 +155,7 @@ describe("Secrets", function() {
 
     it("should refuse to encrypt with zero keys", async function() {
         const alice = await makeTestClient(
-            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
 
         try {
@@ -149,13 +166,21 @@ describe("Secrets", function() {
     });
 
     it("should encrypt with default key if keys is null", async function() {
+        const key = new Uint8Array(16);
+        for (let i = 0; i < 16; i++) key[i] = i;
+        const getKey = jest.fn(e => {
+            expect(Object.keys(e.keys)).toEqual([newKeyId]);
+            return [newKeyId, key];
+        });
+
         let keys = {};
         const alice = await makeTestClient(
-            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
             {
                 cryptoCallbacks: {
                     getCrossSigningKey: t => keys[t],
                     saveCrossSigningKeys: k => keys = k,
+                    getSecretStorageKey: getKey,
                 },
             },
         );
@@ -167,10 +192,10 @@ describe("Secrets", function() {
                 }),
             ]);
         };
-        alice.resetCrossSigningKeys();
+        resetCrossSigningKeys(alice);
 
-        const newKeyId = await alice.addSecretStorageKey(
-            SECRET_STORAGE_ALGORITHM_V1,
+        const { keyId: newKeyId } = await alice.addSecretStorageKey(
+            SECRET_STORAGE_ALGORITHM_V1_AES,
         );
         // we don't await on this because it waits for the event to come down the sync
         // which won't happen in the test setup
@@ -183,7 +208,7 @@ describe("Secrets", function() {
 
     it("should refuse to encrypt if no keys given and no default key", async function() {
         const alice = await makeTestClient(
-            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            { userId: "@alice:example.com", deviceId: "Osborne2" },
         );
 
         try {
@@ -196,24 +221,24 @@ describe("Secrets", function() {
     it("should request secrets from other clients", async function() {
         const [osborne2, vax] = await makeTestClients(
             [
-                {userId: "@alice:example.com", deviceId: "Osborne2"},
-                {userId: "@alice:example.com", deviceId: "VAX"},
+                { userId: "@alice:example.com", deviceId: "Osborne2" },
+                { userId: "@alice:example.com", deviceId: "VAX" },
             ],
             {
                 cryptoCallbacks: {
-                    onSecretRequested: e => {
-                        expect(e.name).toBe("foo");
+                    onSecretRequested: (userId, deviceId, requestId, secretName, deviceTrust) => {
+                        expect(secretName).toBe("foo");
                         return "bar";
                     },
                 },
             },
         );
 
-        const vaxDevice = vax.client._crypto._olmDevice;
-        const osborne2Device = osborne2.client._crypto._olmDevice;
-        const secretStorage = osborne2.client._crypto._secretStorage;
+        const vaxDevice = vax.client.crypto._olmDevice;
+        const osborne2Device = osborne2.client.crypto._olmDevice;
+        const secretStorage = osborne2.client.crypto._secretStorage;
 
-        osborne2.client._crypto._deviceList.storeDevicesForUser("@alice:example.com", {
+        osborne2.client.crypto._deviceList.storeDevicesForUser("@alice:example.com", {
             "VAX": {
                 user_id: "@alice:example.com",
                 device_id: "VAX",
@@ -224,7 +249,7 @@ describe("Secrets", function() {
                 },
             },
         });
-        vax.client._crypto._deviceList.storeDevicesForUser("@alice:example.com", {
+        vax.client.crypto._deviceList.storeDevicesForUser("@alice:example.com", {
             "Osborne2": {
                 user_id: "@alice:example.com",
                 device_id: "Osborne2",
@@ -240,7 +265,7 @@ describe("Secrets", function() {
         const otks = (await osborne2Device.getOneTimeKeys()).curve25519;
         await osborne2Device.markKeysAsPublished();
 
-        await vax.client._crypto._olmDevice.createOutboundSession(
+        await vax.client.crypto._olmDevice.createOutboundSession(
             osborne2Device.deviceCurve25519Key,
             Object.values(otks)[0],
         );
@@ -251,89 +276,399 @@ describe("Secrets", function() {
         expect(secret).toBe("bar");
     });
 
-    it("bootstraps when no storage or cross-signing keys locally", async function() {
-        const bob = await makeTestClient(
-            {
-                userId: "@bob:example.com",
-                deviceId: "bob1",
-            },
+    describe("bootstrap", function() {
+        // keys used in some of the tests
+        const XSK = new Uint8Array(
+            olmlib.decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="),
         );
-        bob.uploadDeviceSigningKeys = async () => {};
-        bob.uploadKeySignatures = async () => {};
-        bob.setAccountData = async function(eventType, contents, callback) {
-            const event = new MatrixEvent({
-                type: eventType,
-                content: contents,
+        const XSPubKey = "DRb8pFVJyEJ9OWvXeUoM0jq/C2Wt+NxzBZVuk2nRb+0";
+        const USK = new Uint8Array(
+            olmlib.decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="),
+        );
+        const USPubKey = "CUpoiTtHiyXpUmd+3ohb7JVxAlUaOG1NYs9Jlx8soQU";
+        const SSK = new Uint8Array(
+            olmlib.decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="),
+        );
+        const SSPubKey = "0DfNsRDzEvkCLA0gD3m7VAGJ5VClhjEsewI35xq873Q";
+        const SSSSKey = new Uint8Array(
+            olmlib.decodeBase64(
+                "XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0=",
+            ),
+        );
+
+        it("bootstraps when no storage or cross-signing keys locally", async function() {
+            const key = new Uint8Array(16);
+            for (let i = 0; i < 16; i++) key[i] = i;
+            const getKey = jest.fn(e => {
+                return [Object.keys(e.keys)[0], key];
             });
-            this.store.storeAccountDataEvents([
-                event,
-            ]);
-            this.emit("accountData", event);
-        };
 
-        await bob.bootstrapSecretStorage();
-
-        const crossSigning = bob._crypto._crossSigningInfo;
-        const secretStorage = bob._crypto._secretStorage;
-
-        expect(crossSigning.getId()).toBeTruthy();
-        expect(await crossSigning.isStoredInSecretStorage(secretStorage)).toBeTruthy();
-        expect(await secretStorage.hasKey()).toBeTruthy();
-    });
-
-    it("bootstraps when cross-signing keys in secret storage", async function() {
-        const decryption = new global.Olm.PkDecryption();
-        const storagePublicKey = decryption.generate_key();
-        const storagePrivateKey = decryption.get_private_key();
-
-        const bob = await makeTestClient(
-            {
-                userId: "@bob:example.com",
-                deviceId: "bob1",
-            },
-            {
-                cryptoCallbacks: {
-                    getSecretStorageKey: async request => {
-                        const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
-                        expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
-                        return [defaultKeyId, storagePrivateKey];
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: getKey,
                     },
                 },
-            },
-        );
+            );
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
 
-        bob.uploadDeviceSigningKeys = async () => {};
-        bob.uploadKeySignatures = async () => {};
-        bob.setAccountData = async function(eventType, contents, callback) {
-            const event = new MatrixEvent({
-                type: eventType,
-                content: contents,
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey,
             });
-            this.store.storeAccountDataEvents([
-                event,
-            ]);
-            this.emit("accountData", event);
-        };
-        bob._crypto.checkKeyBackup = async () => {};
 
-        const crossSigning = bob._crypto._crossSigningInfo;
-        const secretStorage = bob._crypto._secretStorage;
+            const crossSigning = bob.crypto._crossSigningInfo;
+            const secretStorage = bob.crypto._secretStorage;
 
-        // Set up cross-signing keys from scratch with specific storage key
-        await bob.bootstrapSecretStorage({
-            createSecretStorageKey: async () => ({ pubkey: storagePublicKey }),
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
         });
 
-        // Clear local cross-signing keys and read from secret storage
-        bob._crypto._deviceList.storeCrossSigningForUser(
-            "@bob:example.com",
-            crossSigning.toStorage(),
-        );
-        crossSigning.keys = {};
-        await bob.bootstrapSecretStorage();
+        it("bootstraps when cross-signing keys in secret storage", async function() {
+            const decryption = new global.Olm.PkDecryption();
+            const storagePublicKey = decryption.generate_key();
+            const storagePrivateKey = decryption.get_private_key();
 
-        expect(crossSigning.getId()).toBeTruthy();
-        expect(await crossSigning.isStoredInSecretStorage(secretStorage)).toBeTruthy();
-        expect(await secretStorage.hasKey()).toBeTruthy();
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: async request => {
+                            const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
+                            expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
+                            return [defaultKeyId, storagePrivateKey];
+                        },
+                    },
+                },
+            );
+
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
+            bob.crypto._backupManager.checkKeyBackup = async () => {};
+
+            const crossSigning = bob.crypto._crossSigningInfo;
+            const secretStorage = bob.crypto._secretStorage;
+
+            // Set up cross-signing keys from scratch with specific storage key
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey: async () => ({
+                    // `pubkey` not used anymore with symmetric 4S
+                    keyInfo: { pubkey: storagePublicKey },
+                    privateKey: storagePrivateKey,
+                }),
+            });
+
+            // Clear local cross-signing keys and read from secret storage
+            bob.crypto._deviceList.storeCrossSigningForUser(
+                "@bob:example.com",
+                crossSigning.toStorage(),
+            );
+            crossSigning.keys = {};
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
+        });
+
+        it("adds passphrase checking if it's lacking", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                { userId: "@alice:example.com", deviceId: "Osborne2" },
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({ keys }, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                // we never use these values, other than checking that they
+                // exist, so just use dummy values
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: { ciphertext: "bla", mac: "bla", iv: "bla" },
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: { ciphertext: "bla", mac: "bla", iv: "bla" },
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: { ciphertext: "bla", mac: "bla", iv: "bla" },
+                        },
+                    },
+                }),
+            ]);
+            alice.crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            expect(alice.getAccountData("m.secret_storage.default_key").getContent())
+                .toEqual({ key: "key_id" });
+            const keyInfo = alice.getAccountData("m.secret_storage.key.key_id")
+                .getContent();
+            expect(keyInfo.algorithm)
+                .toEqual("m.secret_storage.v1.aes-hmac-sha2");
+            expect(keyInfo.passphrase).toEqual({
+                algorithm: "m.pbkdf2",
+                iterations: 500000,
+                salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+            });
+            expect(keyInfo).toHaveProperty("iv");
+            expect(keyInfo).toHaveProperty("mac");
+            expect(alice.checkSecretStorageKey(secretStorageKeys.key_id, keyInfo))
+                .toBeTruthy();
+        });
+        it("fixes backup keys in the wrong format", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                { userId: "@alice:example.com", deviceId: "Osborne2" },
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({ keys }, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: { ciphertext: "bla", mac: "bla", iv: "bla" },
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: { ciphertext: "bla", mac: "bla", iv: "bla" },
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: { ciphertext: "bla", mac: "bla", iv: "bla" },
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.megolm_backup.v1",
+                    content: {
+                        encrypted: {
+                            key_id: await encryptAES(
+                                "123,45,6,7,89,1,234,56,78,90,12,34,5,67,8,90",
+                                secretStorageKeys.key_id, "m.megolm_backup.v1",
+                            ),
+                        },
+                    },
+                }),
+            ]);
+            alice.crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            const backupKey = alice.getAccountData("m.megolm_backup.v1")
+                .getContent();
+            expect(backupKey.encrypted).toHaveProperty("key_id");
+            expect(await alice.getSecret("m.megolm_backup.v1"))
+                .toEqual("ey0GB1kB6jhOWgwiBUMIWg==");
+        });
     });
 });

spec/unit/crypto/verification/InRoomChannel.spec.js

@@ -13,9 +13,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-import {InRoomChannel} from "../../../../src/crypto/verification/request/InRoomChannel";
+import { InRoomChannel } from "../../../../src/crypto/verification/request/InRoomChannel";
     "../../../../src/crypto/verification/request/ToDeviceChannel";
-import {MatrixEvent} from "../../../../src/models/event";
+import { MatrixEvent } from "../../../../src/models/event";
 
 describe("InRoomChannel tests", function() {
     const ALICE = "@alice:hs.tld";

spec/unit/crypto/verification/qr_code.spec.js

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 import "../../../olm-loader";
-import {logger} from "../../../../src/logger";
+import { logger } from "../../../../src/logger";
 
 const Olm = global.Olm;
 

spec/unit/crypto/verification/request.spec.js

@@ -15,10 +15,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 import "../../../olm-loader";
-import {verificationMethods} from "../../../../src/crypto";
-import {logger} from "../../../../src/logger";
-import {SAS} from "../../../../src/crypto/verification/SAS";
-import {makeTestClients, setupWebcrypto, teardownWebcrypto} from './util';
+import { verificationMethods } from "../../../../src/crypto";
+import { logger } from "../../../../src/logger";
+import { SAS } from "../../../../src/crypto/verification/SAS";
+import { makeTestClients, setupWebcrypto, teardownWebcrypto } from './util';
 
 const Olm = global.Olm;
 
@@ -42,14 +42,14 @@ describe("verification request integration tests with crypto layer", function()
     it("should request and accept a verification", async function() {
         const [alice, bob] = await makeTestClients(
             [
-                {userId: "@alice:example.com", deviceId: "Osborne2"},
-                {userId: "@bob:example.com", deviceId: "Dynabook"},
+                { userId: "@alice:example.com", deviceId: "Osborne2" },
+                { userId: "@bob:example.com", deviceId: "Dynabook" },
             ],
             {
                 verificationMethods: [verificationMethods.SAS],
             },
         );
-        alice.client._crypto._deviceList.getRawStoredDevicesForUser = function() {
+        alice.client.crypto._deviceList.getRawStoredDevicesForUser = function() {
             return {
                 Dynabook: {
                     keys: {

spec/unit/crypto/verification/sas.spec.js

@@ -15,13 +15,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 import "../../../olm-loader";
-import {makeTestClients, setupWebcrypto, teardownWebcrypto} from './util';
-import {MatrixEvent} from "../../../../src/models/event";
-import {SAS} from "../../../../src/crypto/verification/SAS";
-import {DeviceInfo} from "../../../../src/crypto/deviceinfo";
-import {verificationMethods} from "../../../../src/crypto";
+import { makeTestClients, setupWebcrypto, teardownWebcrypto } from './util';
+import { MatrixEvent } from "../../../../src/models/event";
+import { SAS } from "../../../../src/crypto/verification/SAS";
+import { DeviceInfo } from "../../../../src/crypto/deviceinfo";
+import { verificationMethods } from "../../../../src/crypto";
 import * as olmlib from "../../../../src/crypto/olmlib";
-import {logger} from "../../../../src/logger";
+import { logger } from "../../../../src/logger";
+import { resetCrossSigningKeys } from "../crypto-utils";
 
 const Olm = global.Olm;
 
@@ -44,7 +45,16 @@ describe("SAS verification", function() {
     });
 
     it("should error on an unexpected event", async function() {
-        const sas = new SAS({}, "@alice:example.com", "ABCDEFG");
+        //channel, baseApis, userId, deviceId, startEvent, request
+        const request = {
+            onVerifierCancelled: function() {},
+        };
+        const channel = {
+            send: function() {
+                return Promise.resolve();
+            },
+        };
+        const sas = new SAS(channel, {}, "@alice:example.com", "ABCDEFG", null, request);
         sas.handleEvent(new MatrixEvent({
             sender: "@alice:example.com",
             type: "es.inquisition",
@@ -69,16 +79,16 @@ describe("SAS verification", function() {
         beforeEach(async () => {
             [alice, bob] = await makeTestClients(
                 [
-                    {userId: "@alice:example.com", deviceId: "Osborne2"},
-                    {userId: "@bob:example.com", deviceId: "Dynabook"},
+                    { userId: "@alice:example.com", deviceId: "Osborne2" },
+                    { userId: "@bob:example.com", deviceId: "Dynabook" },
                 ],
                 {
                     verificationMethods: [verificationMethods.SAS],
                 },
             );
 
-            const aliceDevice = alice.client._crypto._olmDevice;
-            const bobDevice = bob.client._crypto._olmDevice;
+            const aliceDevice = alice.client.crypto._olmDevice;
+            const bobDevice = bob.client.crypto._olmDevice;
 
             ALICE_DEVICES = {
                 Osborne2: {
@@ -104,14 +114,14 @@ describe("SAS verification", function() {
                 },
             };
 
-            alice.client._crypto._deviceList.storeDevicesForUser(
+            alice.client.crypto._deviceList.storeDevicesForUser(
                 "@bob:example.com", BOB_DEVICES,
             );
             alice.client.downloadKeys = () => {
                 return Promise.resolve();
             };
 
-            bob.client._crypto._deviceList.storeDevicesForUser(
+            bob.client.crypto._deviceList.storeDevicesForUser(
                 "@alice:example.com", ALICE_DEVICES,
             );
             bob.client.downloadKeys = () => {
@@ -172,11 +182,14 @@ describe("SAS verification", function() {
 
         it("should verify a key", async () => {
             let macMethod;
+            let keyAgreement;
             const origSendToDevice = bob.client.sendToDevice.bind(bob.client);
             bob.client.sendToDevice = function(type, map) {
                 if (type === "m.key.verification.accept") {
                     macMethod = map[alice.client.getUserId()][alice.client.deviceId]
                         .message_authentication_code;
+                    keyAgreement = map[alice.client.getUserId()][alice.client.deviceId]
+                        .key_agreement_protocol;
                 }
                 return origSendToDevice(type, map);
             };
@@ -203,6 +216,7 @@ describe("SAS verification", function() {
 
             // make sure that it uses the preferred method
             expect(macMethod).toBe("hkdf-hmac-sha256");
+            expect(keyAgreement).toBe("curve25519-hkdf-sha256");
 
             // make sure Alice and Bob verified each other
             const bobDevice
@@ -275,16 +289,16 @@ describe("SAS verification", function() {
             );
             alice.httpBackend.when('POST', '/keys/signatures/upload').respond(200, {});
             alice.httpBackend.flush(undefined, 2);
-            await alice.client.resetCrossSigningKeys();
+            await resetCrossSigningKeys(alice.client);
             bob.httpBackend.when('POST', '/keys/device_signing/upload').respond(200, {});
             bob.httpBackend.when('POST', '/keys/signatures/upload').respond(200, {});
             bob.httpBackend.flush(undefined, 2);
 
-            await bob.client.resetCrossSigningKeys();
+            await resetCrossSigningKeys(bob.client);
 
-            bob.client._crypto._deviceList.storeCrossSigningForUser(
+            bob.client.crypto._deviceList.storeCrossSigningForUser(
                 "@alice:example.com", {
-                    keys: alice.client._crypto._crossSigningInfo.keys,
+                    keys: alice.client.crypto._crossSigningInfo.keys,
                 },
             );
 
@@ -322,8 +336,8 @@ describe("SAS verification", function() {
     it("should send a cancellation message on error", async function() {
         const [alice, bob] = await makeTestClients(
             [
-                {userId: "@alice:example.com", deviceId: "Osborne2"},
-                {userId: "@bob:example.com", deviceId: "Dynabook"},
+                { userId: "@alice:example.com", deviceId: "Osborne2" },
+                { userId: "@bob:example.com", deviceId: "Dynabook" },
             ],
             {
                 verificationMethods: [verificationMethods.SAS],
@@ -376,8 +390,8 @@ describe("SAS verification", function() {
         beforeEach(async function() {
             [alice, bob] = await makeTestClients(
                 [
-                    {userId: "@alice:example.com", deviceId: "Osborne2"},
-                    {userId: "@bob:example.com", deviceId: "Dynabook"},
+                    { userId: "@alice:example.com", deviceId: "Osborne2" },
+                    { userId: "@bob:example.com", deviceId: "Dynabook" },
                 ],
                 {
                     verificationMethods: [verificationMethods.SAS],

spec/unit/crypto/verification/secret_request.spec.js

@@ -14,10 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {VerificationBase} from '../../../../src/crypto/verification/Base';
-import {CrossSigningInfo} from '../../../../src/crypto/CrossSigning';
-import {encodeBase64} from "../../../../src/crypto/olmlib";
-import {setupWebcrypto, teardownWebcrypto} from './util';
+import { VerificationBase } from '../../../../src/crypto/verification/Base';
+import { CrossSigningInfo } from '../../../../src/crypto/CrossSigning';
+import { encodeBase64 } from "../../../../src/crypto/olmlib";
+import { setupWebcrypto, teardownWebcrypto } from './util';
 
 jest.useFakeTimers();
 
@@ -54,6 +54,7 @@ describe("self-verifications", () => {
             cacheCallbacks,
         );
         _crossSigningInfo.keys = {
+            master: { keys: { X: testKeyPub } },
             self_signing: { keys: { X: testKeyPub } },
             user_signing: { keys: { X: testKeyPub } },
         };
@@ -64,12 +65,20 @@ describe("self-verifications", () => {
             }),
         };
 
+        const storeSessionBackupPrivateKey = jest.fn();
+        const restoreKeyBackupWithCache = jest.fn(() => Promise.resolve());
+
         const client = {
-            _crypto: {
+            crypto: {
                 _crossSigningInfo,
                 _secretStorage,
+                storeSessionBackupPrivateKey,
+                getSessionBackupPrivateKey: () => null,
             },
+            requestSecret: _secretStorage.request.bind(_secretStorage),
             getUserId: () => userId,
+            getKeyBackupVersion: () => Promise.resolve({}),
+            restoreKeyBackupWithCache,
         };
 
         const request = {
@@ -88,15 +97,20 @@ describe("self-verifications", () => {
 
         const result = await verification.done();
 
-        /* We should request, and store, two keys */
-        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls.length).toBe(2);
-        expect(_secretStorage.request.mock.calls.length).toBe(2);
+        /* We should request, and store, 3 cross signing keys and the key backup key */
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls.length).toBe(3);
+        expect(_secretStorage.request.mock.calls.length).toBe(4);
 
         expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[0][1])
           .toEqual(testKey);
         expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[1][1])
           .toEqual(testKey);
 
+        expect(storeSessionBackupPrivateKey.mock.calls[0][0])
+          .toEqual(testKey);
+
+        expect(restoreKeyBackupWithCache).toHaveBeenCalled();
+
         expect(result).toBeInstanceOf(Array);
         expect(result[0][0]).toBe(testKeyPub);
         expect(result[1][0]).toBe(testKeyPub);

spec/unit/crypto/verification/util.js

@@ -15,27 +15,28 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {TestClient} from '../../../TestClient';
-import {MatrixEvent} from "../../../../src/models/event";
+import { TestClient } from '../../../TestClient';
+import { MatrixEvent } from "../../../../src/models/event";
 import nodeCrypto from "crypto";
+import { logger } from '../../../../src/logger';
 
 export async function makeTestClients(userInfos, options) {
     const clients = [];
     const clientMap = {};
     const sendToDevice = function(type, map) {
-        // console.log(this.getUserId(), "sends", type, map);
+        // logger.log(this.getUserId(), "sends", type, map);
         for (const [userId, devMap] of Object.entries(map)) {
             if (userId in clientMap) {
                 for (const [deviceId, msg] of Object.entries(devMap)) {
                     if (deviceId in clientMap[userId]) {
                         const event = new MatrixEvent({
-                            sender: this.getUserId(), // eslint-disable-line babel/no-invalid-this
+                            sender: this.getUserId(), // eslint-disable-line @babel/no-invalid-this
                             type: type,
                             content: msg,
                         });
                         const client = clientMap[userId][deviceId];
                         const decryptionPromise = event.isEncrypted() ?
-                            event.attemptDecryption(client._crypto) :
+                            event.attemptDecryption(client.crypto) :
                             Promise.resolve();
 
                         decryptionPromise.then(
@@ -48,9 +49,9 @@ export async function makeTestClients(userInfos, options) {
     };
     const sendEvent = function(room, type, content) {
         // make up a unique ID as the event ID
-        const eventId = "$" + this.makeTxnId(); // eslint-disable-line babel/no-invalid-this
+        const eventId = "$" + this.makeTxnId(); // eslint-disable-line @babel/no-invalid-this
         const rawEvent = {
-            sender: this.getUserId(), // eslint-disable-line babel/no-invalid-this
+            sender: this.getUserId(), // eslint-disable-line @babel/no-invalid-this
             type: type,
             content: content,
             room_id: room,
@@ -60,14 +61,14 @@ export async function makeTestClients(userInfos, options) {
         const event = new MatrixEvent(rawEvent);
         const remoteEcho = new MatrixEvent(Object.assign({}, rawEvent, {
             unsigned: {
-                transaction_id: this.makeTxnId(), // eslint-disable-line babel/no-invalid-this
+                transaction_id: this.makeTxnId(), // eslint-disable-line @babel/no-invalid-this
             },
         }));
 
         setImmediate(() => {
             for (const tc of clients) {
-                if (tc.client === this) { // eslint-disable-line babel/no-invalid-this
-                    console.log("sending remote echo!!");
+                if (tc.client === this) { // eslint-disable-line @babel/no-invalid-this
+                    logger.log("sending remote echo!!");
                     tc.client.emit("Room.timeline", remoteEcho);
                 } else {
                     tc.client.emit("Room.timeline", event);
@@ -75,7 +76,7 @@ export async function makeTestClients(userInfos, options) {
             }
         });
 
-        return Promise.resolve({event_id: eventId});
+        return Promise.resolve({ event_id: eventId });
     };
 
     for (const userInfo of userInfos) {

spec/unit/crypto/verification/verification_request.spec.js

@@ -13,13 +13,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-import {VerificationRequest, READY_TYPE, START_TYPE, DONE_TYPE} from
+import { VerificationRequest, READY_TYPE, START_TYPE, DONE_TYPE } from
     "../../../../src/crypto/verification/request/VerificationRequest";
-import {InRoomChannel} from "../../../../src/crypto/verification/request/InRoomChannel";
-import {ToDeviceChannel} from
+import { InRoomChannel } from "../../../../src/crypto/verification/request/InRoomChannel";
+import { ToDeviceChannel } from
     "../../../../src/crypto/verification/request/ToDeviceChannel";
-import {MatrixEvent} from "../../../../src/models/event";
-import {setupWebcrypto, teardownWebcrypto} from "./util";
+import { MatrixEvent } from "../../../../src/models/event";
+import { setupWebcrypto, teardownWebcrypto } from "./util";
 
 function makeMockClient(userId, deviceId) {
     let counter = 1;
@@ -40,7 +40,7 @@ function makeMockClient(userId, deviceId) {
                 content,
                 origin_server_ts: Date.now(),
             }));
-            return Promise.resolve({event_id: eventId});
+            return Promise.resolve({ event_id: eventId });
         },
 
         sendToDevice(type, msgMap) {
@@ -48,7 +48,7 @@ function makeMockClient(userId, deviceId) {
                 const deviceMap = msgMap[userId];
                 for (const deviceId of Object.keys(deviceMap)) {
                     const content = deviceMap[deviceId];
-                    const event = new MatrixEvent({content, type});
+                    const event = new MatrixEvent({ content, type });
                     deviceEvents[userId] = deviceEvents[userId] || {};
                     deviceEvents[userId][deviceId] = deviceEvents[userId][deviceId] || [];
                     deviceEvents[userId][deviceId].push(event);
@@ -90,7 +90,7 @@ class MockVerifier {
         if (this._startEvent) {
             await this._channel.send(DONE_TYPE, {});
         } else {
-            await this._channel.send(START_TYPE, {method: MOCK_METHOD});
+            await this._channel.send(START_TYPE, { method: MOCK_METHOD });
         }
     }
 
@@ -119,6 +119,8 @@ async function distributeEvent(ownRequest, theirRequest, event) {
     await theirRequest.channel.handleEvent(event, theirRequest, true);
 }
 
+jest.useFakeTimers();
+
 describe("verification request unit tests", function() {
     beforeAll(function() {
         setupWebcrypto();
@@ -224,7 +226,7 @@ describe("verification request unit tests", function() {
             new ToDeviceChannel(bob1, bob1.getUserId(), ["device1", "device2"],
                 ToDeviceChannel.makeTransactionId(), "device2"),
             new Map([[MOCK_METHOD, MockVerifier]]), bob1);
-        const to = {userId: "@bob:matrix.tld", deviceId: "device2"};
+        const to = { userId: "@bob:matrix.tld", deviceId: "device2" };
         const verifier = bob1Request.beginKeyVerification(MOCK_METHOD, to);
         expect(verifier).toBeInstanceOf(MockVerifier);
         await verifier.start();
@@ -246,4 +248,38 @@ describe("verification request unit tests", function() {
         expect(bob1Request.done).toBe(true);
         expect(bob2Request.done).toBe(true);
     });
+
+    it("request times out after 10 minutes", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        await aliceRequest.channel.handleEvent(requestEvent, aliceRequest, true,
+            true, true);
+
+        expect(aliceRequest.cancelled).toBe(false);
+        expect(aliceRequest._cancellingUserId).toBe(undefined);
+        jest.advanceTimersByTime(10 * 60 * 1000);
+        expect(aliceRequest._cancellingUserId).toBe(alice.getUserId());
+    });
+
+    it("request times out 2 minutes after receipt", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        const bobRequest = new VerificationRequest(
+            new InRoomChannel(bob, "!room"), new Map(), bob);
+
+        await bobRequest.channel.handleEvent(requestEvent, bobRequest, true);
+
+        expect(bobRequest.cancelled).toBe(false);
+        expect(bobRequest._cancellingUserId).toBe(undefined);
+        jest.advanceTimersByTime(2 * 60 * 1000);
+        expect(bobRequest._cancellingUserId).toBe(bob.getUserId());
+    });
 });

spec/unit/event-timeline.spec.js

@@ -1,6 +1,6 @@
 import * as utils from "../test-utils";
-import {EventTimeline} from "../../src/models/event-timeline";
-import {RoomState} from "../../src/models/room-state";
+import { EventTimeline } from "../../src/models/event-timeline";
+import { RoomState } from "../../src/models/room-state";
 
 function mockRoomStates(timeline) {
     timeline._startState = utils.mock(RoomState, "startState");
@@ -15,7 +15,7 @@ describe("EventTimeline", function() {
 
     beforeEach(function() {
         // XXX: this is a horrid hack; should use sinon or something instead to mock
-        const timelineSet = { room: { roomId: roomId }};
+        const timelineSet = { room: { roomId: roomId } };
         timelineSet.room.getUnfilteredTimelineSet = function() {
             return timelineSet;
         };
@@ -94,7 +94,6 @@ describe("EventTimeline", function() {
         });
     });
 
-
     describe("neighbouringTimelines", function() {
         it("neighbouring timelines should start null", function() {
             expect(timeline.getNeighbouringTimeline(EventTimeline.BACKWARDS)).toBe(null);
@@ -102,8 +101,8 @@ describe("EventTimeline", function() {
         });
 
         it("setNeighbouringTimeline should set neighbour", function() {
-            const prev = {a: "a"};
-            const next = {b: "b"};
+            const prev = { a: "a" };
+            const next = { b: "b" };
             timeline.setNeighbouringTimeline(prev, EventTimeline.BACKWARDS);
             timeline.setNeighbouringTimeline(next, EventTimeline.FORWARDS);
             expect(timeline.getNeighbouringTimeline(EventTimeline.BACKWARDS)).toBe(prev);
@@ -111,8 +110,8 @@ describe("EventTimeline", function() {
         });
 
         it("setNeighbouringTimeline should throw if called twice", function() {
-            const prev = {a: "a"};
-            const next = {b: "b"};
+            const prev = { a: "a" };
+            const next = { b: "b" };
             expect(function() {
                 timeline.setNeighbouringTimeline(prev, EventTimeline.BACKWARDS);
             }).not.toThrow();
@@ -278,7 +277,6 @@ describe("EventTimeline", function() {
                 not.toHaveBeenCalled();
         });
 
-
         it("should call setStateEvents on the right RoomState with the right " +
            "forwardLooking value for old events", function() {
             const events = [

spec/unit/event.spec.js

@@ -15,8 +15,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {logger} from "../../src/logger";
-import {MatrixEvent} from "../../src/models/event";
+import { logger } from "../../src/logger";
+import { MatrixEvent } from "../../src/models/event";
 
 describe("MatrixEvent", () => {
     describe(".attemptDecryption", () => {

spec/unit/filter-component.spec.js

@@ -0,0 +1,34 @@
+import { FilterComponent } from "../../src/filter-component";
+import { mkEvent } from '../test-utils';
+
+describe("Filter Component", function() {
+    describe("types", function() {
+        it("should filter out events with other types", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.member',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(false);
+        });
+
+        it("should validate events with the same type", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.message',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(true);
+        });
+    });
+});

spec/unit/filter.spec.js

@@ -1,4 +1,4 @@
-import {Filter} from "../../src/filter";
+import { Filter } from "../../src/filter";
 
 describe("Filter", function() {
     const filterId = "f1lt3ring15g00d4ursoul";

spec/unit/interactive-auth.spec.js

@@ -15,9 +15,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {logger} from "../../src/logger";
-import {InteractiveAuth} from "../../src/interactive-auth";
-import {MatrixError} from "../../src/http-api";
+import { logger } from "../../src/logger";
+import { InteractiveAuth } from "../../src/interactive-auth";
+import { MatrixError } from "../../src/http-api";
 
 // Trivial client object to test interactive auth
 // (we do not need TestClient here)
@@ -63,7 +63,7 @@ describe("InteractiveAuth", function() {
         });
 
         // .. which should trigger a call here
-        const requestRes = {"a": "b"};
+        const requestRes = { "a": "b" };
         doRequest.mockImplementation(function(authData) {
             logger.log('cccc');
             expect(authData).toEqual({
@@ -97,7 +97,7 @@ describe("InteractiveAuth", function() {
         // first we expect a call to doRequest
         doRequest.mockImplementation(function(authData) {
             logger.log("request1", authData);
-            expect(authData).toEqual({});
+            expect(authData).toEqual(null); // first request should be null
             const err = new MatrixError({
                 session: "sessionId",
                 flows: [
@@ -112,7 +112,7 @@ describe("InteractiveAuth", function() {
         });
 
         // .. which should be followed by a call to stateUpdated
-        const requestRes = {"a": "b"};
+        const requestRes = { "a": "b" };
         stateUpdated.mockImplementation(function(stage) {
             expect(stage).toEqual("logintype");
             expect(ia.getSessionId()).toEqual("sessionId");
@@ -143,4 +143,33 @@ describe("InteractiveAuth", function() {
             expect(stateUpdated).toBeCalledTimes(1);
         });
     });
+
+    it("should start an auth stage and reject if no auth flow", function() {
+        const doRequest = jest.fn();
+        const stateUpdated = jest.fn();
+
+        const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
+            doRequest: doRequest,
+            stateUpdated: stateUpdated,
+        });
+
+        doRequest.mockImplementation(function(authData) {
+            logger.log("request1", authData);
+            expect(authData).toEqual(null); // first request should be null
+            const err = new MatrixError({
+                session: "sessionId",
+                flows: [],
+                params: {
+                    "logintype": { param: "aa" },
+                },
+            });
+            err.httpStatus = 401;
+            throw err;
+        });
+
+        return ia.attemptAuth().catch(function(error) {
+            expect(error.message).toBe('No appropriate authentication flow found');
+        });
+    });
 });

spec/unit/login.spec.js

@@ -1,4 +1,4 @@
-import {TestClient} from '../TestClient';
+import { TestClient } from '../TestClient';
 
 describe('Login request', function() {
     let client;

spec/unit/matrix-client.spec.js

@@ -1,6 +1,18 @@
-import {logger} from "../../src/logger";
-import {MatrixClient} from "../../src/client";
-import {Filter} from "../../src/filter";
+import { logger } from "../../src/logger";
+import { MatrixClient } from "../../src/client";
+import { Filter } from "../../src/filter";
+import { DEFAULT_TREE_POWER_LEVELS_TEMPLATE } from "../../src/models/MSC3089TreeSpace";
+import {
+    EventType,
+    RoomCreateTypeField,
+    RoomType,
+    UNSTABLE_MSC3088_ENABLED,
+    UNSTABLE_MSC3088_PURPOSE,
+    UNSTABLE_MSC3089_TREE_SUBTYPE,
+} from "../../src/@types/event";
+import { MEGOLM_ALGORITHM } from "../../src/crypto/olmlib";
+import { MatrixEvent } from "../../src/models/event";
+import { Preset } from "../../src/@types/partials";
 
 jest.useFakeTimers();
 
@@ -144,12 +156,12 @@ describe("MatrixClient", function() {
             scheduler: scheduler,
             userId: userId,
         });
-        // FIXME: We shouldn't be yanking _http like this.
-        client._http = [
+        // FIXME: We shouldn't be yanking http like this.
+        client.http = [
             "authedRequest", "getContentUri", "request", "uploadContent",
         ].reduce((r, k) => { r[k] = jest.fn(); return r; }, {});
-        client._http.authedRequest.mockImplementation(httpReq);
-        client._http.request.mockImplementation(httpReq);
+        client.http.authedRequest.mockImplementation(httpReq);
+        client.http.request.mockImplementation(httpReq);
 
         // set reasonable working defaults
         acceptKeepalives = true;
@@ -166,11 +178,165 @@ describe("MatrixClient", function() {
         // means they may call /events and then fail an expect() which will fail
         // a DIFFERENT test (pollution between tests!) - we return unresolved
         // promises to stop the client from continuing to run.
-        client._http.authedRequest.mockImplementation(function() {
+        client.http.authedRequest.mockImplementation(function() {
             return new Promise(() => {});
         });
     });
 
+    it("should create (unstable) file trees", async () => {
+        const userId = "@test:example.org";
+        const roomId = "!room:example.org";
+        const roomName = "Test Tree";
+        const mockRoom = {};
+        const fn = jest.fn().mockImplementation((opts) => {
+            expect(opts).toMatchObject({
+                name: roomName,
+                preset: Preset.PrivateChat,
+                power_level_content_override: {
+                    ...DEFAULT_TREE_POWER_LEVELS_TEMPLATE,
+                    users: {
+                        [userId]: 100,
+                    },
+                },
+                creation_content: {
+                    [RoomCreateTypeField]: RoomType.Space,
+                },
+                initial_state: [
+                    {
+                        // We use `unstable` to ensure that the code is actually using the right identifier
+                        type: UNSTABLE_MSC3088_PURPOSE.unstable,
+                        state_key: UNSTABLE_MSC3089_TREE_SUBTYPE.unstable,
+                        content: {
+                            [UNSTABLE_MSC3088_ENABLED.unstable]: true,
+                        },
+                    },
+                    {
+                        type: EventType.RoomEncryption,
+                        state_key: "",
+                        content: {
+                            algorithm: MEGOLM_ALGORITHM,
+                        },
+                    },
+                ],
+            });
+            return { room_id: roomId };
+        });
+        client.getUserId = () => userId;
+        client.createRoom = fn;
+        client.getRoom = (getRoomId) => {
+            expect(getRoomId).toEqual(roomId);
+            return mockRoom;
+        };
+        const tree = await client.unstableCreateFileTree(roomName);
+        expect(tree).toBeDefined();
+        expect(tree.roomId).toEqual(roomId);
+        expect(tree.room).toBe(mockRoom);
+        expect(fn.mock.calls.length).toBe(1);
+    });
+
+    it("should get (unstable) file trees with valid state", async () => {
+        const roomId = "!room:example.org";
+        const mockRoom = {
+            currentState: {
+                getStateEvents: (eventType, stateKey) => {
+                    if (eventType === EventType.RoomCreate) {
+                        expect(stateKey).toEqual("");
+                        return new MatrixEvent({
+                            content: {
+                                [RoomCreateTypeField]: RoomType.Space,
+                            },
+                        });
+                    } else if (eventType === UNSTABLE_MSC3088_PURPOSE.unstable) {
+                        // We use `unstable` to ensure that the code is actually using the right identifier
+                        expect(stateKey).toEqual(UNSTABLE_MSC3089_TREE_SUBTYPE.unstable);
+                        return new MatrixEvent({
+                            content: {
+                                [UNSTABLE_MSC3088_ENABLED.unstable]: true,
+                            },
+                        });
+                    } else {
+                        throw new Error("Unexpected event type or state key");
+                    }
+                },
+            },
+        };
+        client.getRoom = (getRoomId) => {
+            expect(getRoomId).toEqual(roomId);
+            return mockRoom;
+        };
+        const tree = client.unstableGetFileTreeSpace(roomId);
+        expect(tree).toBeDefined();
+        expect(tree.roomId).toEqual(roomId);
+        expect(tree.room).toBe(mockRoom);
+    });
+
+    it("should not get (unstable) file trees with invalid create contents", async () => {
+        const roomId = "!room:example.org";
+        const mockRoom = {
+            currentState: {
+                getStateEvents: (eventType, stateKey) => {
+                    if (eventType === EventType.RoomCreate) {
+                        expect(stateKey).toEqual("");
+                        return new MatrixEvent({
+                            content: {
+                                [RoomCreateTypeField]: "org.example.not_space",
+                            },
+                        });
+                    } else if (eventType === UNSTABLE_MSC3088_PURPOSE.unstable) {
+                        // We use `unstable` to ensure that the code is actually using the right identifier
+                        expect(stateKey).toEqual(UNSTABLE_MSC3089_TREE_SUBTYPE.unstable);
+                        return new MatrixEvent({
+                            content: {
+                                [UNSTABLE_MSC3088_ENABLED.unstable]: true,
+                            },
+                        });
+                    } else {
+                        throw new Error("Unexpected event type or state key");
+                    }
+                },
+            },
+        };
+        client.getRoom = (getRoomId) => {
+            expect(getRoomId).toEqual(roomId);
+            return mockRoom;
+        };
+        const tree = client.unstableGetFileTreeSpace(roomId);
+        expect(tree).toBeFalsy();
+    });
+
+    it("should not get (unstable) file trees with invalid purpose/subtype contents", async () => {
+        const roomId = "!room:example.org";
+        const mockRoom = {
+            currentState: {
+                getStateEvents: (eventType, stateKey) => {
+                    if (eventType === EventType.RoomCreate) {
+                        expect(stateKey).toEqual("");
+                        return new MatrixEvent({
+                            content: {
+                                [RoomCreateTypeField]: RoomType.Space,
+                            },
+                        });
+                    } else if (eventType === UNSTABLE_MSC3088_PURPOSE.unstable) {
+                        expect(stateKey).toEqual(UNSTABLE_MSC3089_TREE_SUBTYPE.unstable);
+                        return new MatrixEvent({
+                            content: {
+                                [UNSTABLE_MSC3088_ENABLED.unstable]: false,
+                            },
+                        });
+                    } else {
+                        throw new Error("Unexpected event type or state key");
+                    }
+                },
+            },
+        };
+        client.getRoom = (getRoomId) => {
+            expect(getRoomId).toEqual(roomId);
+            return mockRoom;
+        };
+        const tree = client.unstableGetFileTreeSpace(roomId);
+        expect(tree).toBeFalsy();
+    });
+
     it("should not POST /filter if a matching filter already exists", async function() {
         httpLookups = [];
         httpLookups.push(PUSH_RULES_RESPONSE);
@@ -178,7 +344,7 @@ describe("MatrixClient", function() {
         const filterId = "ehfewf";
         store.getFilterIdByName.mockReturnValue(filterId);
         const filter = new Filter(0, filterId);
-        filter.setDefinition({"room": {"timeline": {"limit": 8}}});
+        filter.setDefinition({ "room": { "timeline": { "limit": 8 } } });
         store.getFilter.mockReturnValue(filter);
         const syncPromise = new Promise((resolve, reject) => {
             client.on("sync", function syncListener(state) {
@@ -521,4 +687,19 @@ describe("MatrixClient", function() {
         xit("should be able to peek into a room using peekInRoom", function(done) {
         });
     });
+
+    describe("getPresence", function() {
+        it("should send a presence HTTP GET", function() {
+            httpLookups = [{
+                method: "GET",
+                path: `/presence/${encodeURIComponent(userId)}/status`,
+                data: {
+                    "presence": "unavailable",
+                    "last_active_ago": 420845,
+                },
+            }];
+            client.getPresence(userId);
+            expect(httpLookups.length).toEqual(0);
+        });
+    });
 });

spec/unit/models/MSC3089Branch.spec.ts

@@ -0,0 +1,155 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { MatrixClient } from "../../../src";
+import { Room } from "../../../src/models/room";
+import { MatrixEvent } from "../../../src/models/event";
+import { UNSTABLE_MSC3089_BRANCH } from "../../../src/@types/event";
+import { EventTimelineSet } from "../../../src/models/event-timeline-set";
+import { EventTimeline } from "../../../src/models/event-timeline";
+import { MSC3089Branch } from "../../../src/models/MSC3089Branch";
+
+describe("MSC3089Branch", () => {
+    let client: MatrixClient;
+    // @ts-ignore - TS doesn't know that this is a type
+    let indexEvent: MatrixEvent;
+    let branch: MSC3089Branch;
+
+    const branchRoomId = "!room:example.org";
+    const fileEventId = "$file";
+
+    const staticTimelineSets = {} as EventTimelineSet;
+    const staticRoom = {
+        getUnfilteredTimelineSet: () => staticTimelineSets,
+    } as any as Room; // partial
+
+    beforeEach(() => {
+        // TODO: Use utility functions to create test rooms and clients
+        client = <MatrixClient>{
+            getRoom: (roomId: string) => {
+                if (roomId === branchRoomId) {
+                    return staticRoom;
+                } else {
+                    throw new Error("Unexpected fetch for unknown room");
+                }
+            },
+        };
+        indexEvent = {
+            getRoomId: () => branchRoomId,
+            getStateKey: () => fileEventId,
+        };
+        branch = new MSC3089Branch(client, indexEvent);
+    });
+
+    it('should know the file event ID', () => {
+        expect(branch.id).toEqual(fileEventId);
+    });
+
+    it('should know if the file is active or not', () => {
+        indexEvent.getContent = () => ({});
+        expect(branch.isActive).toBe(false);
+        indexEvent.getContent = () => ({ active: false });
+        expect(branch.isActive).toBe(false);
+        indexEvent.getContent = () => ({ active: true });
+        expect(branch.isActive).toBe(true);
+        indexEvent.getContent = () => ({ active: "true" }); // invalid boolean, inactive
+        expect(branch.isActive).toBe(false);
+    });
+
+    it('should be able to delete the file', async () => {
+        const stateFn = jest.fn()
+            .mockImplementation((roomId: string, eventType: string, content: any, stateKey: string) => {
+                expect(roomId).toEqual(branchRoomId);
+                expect(eventType).toEqual(UNSTABLE_MSC3089_BRANCH.unstable); // test that we're definitely using the unstable value
+                expect(content).toMatchObject({});
+                expect(content['active']).toBeUndefined();
+                expect(stateKey).toEqual(fileEventId);
+
+                return Promise.resolve(); // return value not used
+            });
+        client.sendStateEvent = stateFn;
+
+        const redactFn = jest.fn().mockImplementation((roomId: string, eventId: string) => {
+            expect(roomId).toEqual(branchRoomId);
+            expect(eventId).toEqual(fileEventId);
+
+            return Promise.resolve(); // return value not used
+        });
+        client.redactEvent = redactFn;
+
+        await branch.delete();
+
+        expect(stateFn).toHaveBeenCalledTimes(1);
+        expect(redactFn).toHaveBeenCalledTimes(1);
+    });
+
+    it('should know its name', async () => {
+        const name = "My File.txt";
+        indexEvent.getContent = () => ({ active: true, name: name });
+
+        const res = branch.getName();
+
+        expect(res).toEqual(name);
+    });
+
+    it('should be able to change its name', async () => {
+        const name = "My File.txt";
+        indexEvent.getContent = () => ({ active: true, retained: true });
+        const stateFn = jest.fn()
+            .mockImplementation((roomId: string, eventType: string, content: any, stateKey: string) => {
+                expect(roomId).toEqual(branchRoomId);
+                expect(eventType).toEqual(UNSTABLE_MSC3089_BRANCH.unstable); // test that we're definitely using the unstable value
+                expect(content).toMatchObject({
+                    retained: true, // canary for copying state
+                    active: true,
+                    name: name,
+                });
+                expect(stateKey).toEqual(fileEventId);
+
+                return Promise.resolve(); // return value not used
+            });
+        client.sendStateEvent = stateFn;
+
+        await branch.setName(name);
+
+        expect(stateFn).toHaveBeenCalledTimes(1);
+    });
+
+    it('should be able to return event information', async () => {
+        const mxcLatter = "example.org/file";
+        const fileContent = { isFile: "not quite", url: "mxc://" + mxcLatter };
+        const eventsArr = [
+            { getId: () => "$not-file", getContent: () => ({}) },
+            { getId: () => fileEventId, getContent: () => ({ file: fileContent }) },
+        ];
+        client.getEventTimeline = () => Promise.resolve({
+            getEvents: () => eventsArr,
+        }) as any as Promise<EventTimeline>; // partial
+        client.mxcUrlToHttp = (mxc: string) => {
+            expect(mxc).toEqual("mxc://" + mxcLatter);
+            return `https://example.org/_matrix/media/v1/download/${mxcLatter}`;
+        };
+        client.decryptEventIfNeeded = () => Promise.resolve();
+
+        const res = await branch.getFileInfo();
+        expect(res).toBeDefined();
+        expect(res).toMatchObject({
+            info: fileContent,
+            // Escape regex from MDN guides: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions
+            httpUrl: expect.stringMatching(`.+${mxcLatter.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`),
+        });
+    });
+});

spec/unit/models/MSC3089TreeSpace.spec.ts

@@ -0,0 +1,857 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { MatrixClient } from "../../../src";
+import { Room } from "../../../src/models/room";
+import { MatrixEvent } from "../../../src/models/event";
+import { EventType, MsgType, UNSTABLE_MSC3089_BRANCH, UNSTABLE_MSC3089_LEAF } from "../../../src/@types/event";
+import {
+    DEFAULT_TREE_POWER_LEVELS_TEMPLATE,
+    MSC3089TreeSpace,
+    TreePermissions,
+} from "../../../src/models/MSC3089TreeSpace";
+import { DEFAULT_ALPHABET } from "../../../src/utils";
+import { MockBlob } from "../../MockBlob";
+
+describe("MSC3089TreeSpace", () => {
+    let client: MatrixClient;
+    let room: Room;
+    let tree: MSC3089TreeSpace;
+    const roomId = "!tree:localhost";
+    const targetUser = "@target:example.org";
+
+    let powerLevels;
+
+    beforeEach(() => {
+        // TODO: Use utility functions to create test rooms and clients
+        client = <MatrixClient>{
+            getRoom: (fetchRoomId: string) => {
+                if (fetchRoomId === roomId) {
+                    return room;
+                } else {
+                    throw new Error("Unexpected fetch for unknown room");
+                }
+            },
+        };
+        room = <Room>{
+            currentState: {
+                getStateEvents: (evType: EventType, stateKey: string) => {
+                    if (evType === EventType.RoomPowerLevels && stateKey === "") {
+                        return powerLevels;
+                    } else {
+                        throw new Error("Accessed unexpected state event type or key");
+                    }
+                },
+            },
+        };
+        tree = new MSC3089TreeSpace(client, roomId);
+        makePowerLevels(DEFAULT_TREE_POWER_LEVELS_TEMPLATE);
+    });
+
+    function makePowerLevels(content: any) {
+        powerLevels = new MatrixEvent({
+            type: EventType.RoomPowerLevels,
+            state_key: "",
+            sender: "@creator:localhost",
+            event_id: "$powerlevels",
+            room_id: roomId,
+            content: content,
+        });
+    }
+
+    it('should populate the room reference', () => {
+        expect(tree.room).toBe(room);
+    });
+
+    it('should proxy the ID member to room ID', () => {
+        expect(tree.id).toEqual(tree.roomId);
+        expect(tree.id).toEqual(roomId);
+    });
+
+    it('should support setting the name of the space', async () => {
+        const newName = "NEW NAME";
+        const fn = jest.fn()
+            .mockImplementation((stateRoomId: string, eventType: EventType, content: any, stateKey: string) => {
+                expect(stateRoomId).toEqual(roomId);
+                expect(eventType).toEqual(EventType.RoomName);
+                expect(stateKey).toEqual("");
+                expect(content).toMatchObject({ name: newName });
+                return Promise.resolve();
+            });
+        client.sendStateEvent = fn;
+        await tree.setName(newName);
+        expect(fn.mock.calls.length).toBe(1);
+    });
+
+    it('should support inviting users to the space', async () => {
+        const target = targetUser;
+        const fn = jest.fn().mockImplementation((inviteRoomId: string, userId: string) => {
+            expect(inviteRoomId).toEqual(roomId);
+            expect(userId).toEqual(target);
+            return Promise.resolve();
+        });
+        client.invite = fn;
+        await tree.invite(target);
+        expect(fn.mock.calls.length).toBe(1);
+    });
+
+    async function evaluatePowerLevels(pls: any, role: TreePermissions, expectedPl: number) {
+        makePowerLevels(pls);
+        const fn = jest.fn()
+            .mockImplementation((stateRoomId: string, eventType: EventType, content: any, stateKey: string) => {
+                expect(stateRoomId).toEqual(roomId);
+                expect(eventType).toEqual(EventType.RoomPowerLevels);
+                expect(stateKey).toEqual("");
+                expect(content).toMatchObject({
+                    ...pls,
+                    users: {
+                        [targetUser]: expectedPl,
+                    },
+                });
+                return Promise.resolve();
+            });
+        client.sendStateEvent = fn;
+        await tree.setPermissions(targetUser, role);
+        expect(fn.mock.calls.length).toBe(1);
+    }
+
+    it('should support setting Viewer permissions', () => {
+        return evaluatePowerLevels({
+            ...DEFAULT_TREE_POWER_LEVELS_TEMPLATE,
+            users_default: 1024,
+        }, TreePermissions.Viewer, 1024);
+    });
+
+    it('should support setting Editor permissions', () => {
+        return evaluatePowerLevels({
+            ...DEFAULT_TREE_POWER_LEVELS_TEMPLATE,
+            events_default: 1024,
+        }, TreePermissions.Editor, 1024);
+    });
+
+    it('should support setting Owner permissions', () => {
+        return evaluatePowerLevels({
+            ...DEFAULT_TREE_POWER_LEVELS_TEMPLATE,
+            events: {
+                [EventType.RoomPowerLevels]: 1024,
+            },
+        }, TreePermissions.Owner, 1024);
+    });
+
+    it('should support demoting permissions', () => {
+        return evaluatePowerLevels({
+            ...DEFAULT_TREE_POWER_LEVELS_TEMPLATE,
+            users_default: 1024,
+            users: {
+                [targetUser]: 2222,
+            },
+        }, TreePermissions.Viewer, 1024);
+    });
+
+    it('should support promoting permissions', () => {
+        return evaluatePowerLevels({
+            ...DEFAULT_TREE_POWER_LEVELS_TEMPLATE,
+            events_default: 1024,
+            users: {
+                [targetUser]: 5,
+            },
+        }, TreePermissions.Editor, 1024);
+    });
+
+    it('should support defaults: Viewer', () => {
+        return evaluatePowerLevels({}, TreePermissions.Viewer, 0);
+    });
+
+    it('should support defaults: Editor', () => {
+        return evaluatePowerLevels({}, TreePermissions.Editor, 50);
+    });
+
+    it('should support defaults: Owner', () => {
+        return evaluatePowerLevels({}, TreePermissions.Owner, 100);
+    });
+
+    it('should create subdirectories', async () => {
+        const subspaceName = "subdirectory";
+        const subspaceId = "!subspace:localhost";
+        const domain = "domain.example.com";
+        client.getRoom = (roomId: string) => {
+            if (roomId === tree.roomId) {
+                return tree.room;
+            } else if (roomId === subspaceId) {
+                return {} as Room; // we don't need anything important off of this
+            } else {
+                throw new Error("Unexpected getRoom call");
+            }
+        };
+        client.getDomain = () => domain;
+        const createFn = jest.fn().mockImplementation(async (name: string) => {
+            expect(name).toEqual(subspaceName);
+            return new MSC3089TreeSpace(client, subspaceId);
+        });
+        const sendStateFn = jest.fn()
+            .mockImplementation(async (roomId: string, eventType: EventType, content: any, stateKey: string) => {
+                expect([tree.roomId, subspaceId]).toContain(roomId);
+                if (roomId === subspaceId) {
+                    expect(eventType).toEqual(EventType.SpaceParent);
+                    expect(stateKey).toEqual(tree.roomId);
+                } else {
+                    expect(eventType).toEqual(EventType.SpaceChild);
+                    expect(stateKey).toEqual(subspaceId);
+                }
+                expect(content).toMatchObject({ via: [domain] });
+
+                // return value not used
+            });
+        client.unstableCreateFileTree = createFn;
+        client.sendStateEvent = sendStateFn;
+
+        const directory = await tree.createDirectory(subspaceName);
+        expect(directory).toBeDefined();
+        expect(directory).not.toBeNull();
+        expect(directory).not.toBe(tree);
+        expect(directory.roomId).toEqual(subspaceId);
+        expect(createFn).toHaveBeenCalledTimes(1);
+        expect(sendStateFn).toHaveBeenCalledTimes(2);
+
+        const content = expect.objectContaining({ via: [domain] });
+        expect(sendStateFn).toHaveBeenCalledWith(subspaceId, EventType.SpaceParent, content, tree.roomId);
+        expect(sendStateFn).toHaveBeenCalledWith(tree.roomId, EventType.SpaceChild, content, subspaceId);
+    });
+
+    it('should find subdirectories', () => {
+        const firstChildRoom = "!one:example.org";
+        const secondChildRoom = "!two:example.org";
+        const thirdChildRoom = "!three:example.org"; // to ensure it doesn't end up in the subdirectories
+        room.currentState = {
+            getStateEvents: (eventType: EventType, stateKey?: string) => {
+                expect(eventType).toEqual(EventType.SpaceChild);
+                expect(stateKey).toBeUndefined();
+                return [
+                    // Partial implementations of Room
+                    { getStateKey: () => firstChildRoom },
+                    { getStateKey: () => secondChildRoom },
+                    { getStateKey: () => thirdChildRoom },
+                ];
+            },
+        };
+        client.getRoom = () => ({} as Room); // to appease the TreeSpace constructor
+
+        const getFn = jest.fn().mockImplementation((roomId: string) => {
+            if (roomId === thirdChildRoom) {
+                throw new Error("Mock not-a-space room case called (expected)");
+            }
+            expect([firstChildRoom, secondChildRoom]).toContain(roomId);
+            return new MSC3089TreeSpace(client, roomId);
+        });
+        client.unstableGetFileTreeSpace = getFn;
+
+        const subdirectories = tree.getDirectories();
+        expect(subdirectories).toBeDefined();
+        expect(subdirectories.length).toBe(2);
+        expect(subdirectories[0].roomId).toBe(firstChildRoom);
+        expect(subdirectories[1].roomId).toBe(secondChildRoom);
+        expect(getFn).toHaveBeenCalledTimes(3);
+        expect(getFn).toHaveBeenCalledWith(firstChildRoom);
+        expect(getFn).toHaveBeenCalledWith(secondChildRoom);
+        expect(getFn).toHaveBeenCalledWith(thirdChildRoom); // check to make sure it tried
+    });
+
+    it('should find specific directories', () => {
+        client.getRoom = () => ({} as Room); // to appease the TreeSpace constructor
+
+        // Only mocking used API
+        const firstSubdirectory = { roomId: "!first:example.org" } as any as MSC3089TreeSpace;
+        const searchedSubdirectory = { roomId: "!find_me:example.org" } as any as MSC3089TreeSpace;
+        const thirdSubdirectory = { roomId: "!third:example.org" } as any as MSC3089TreeSpace;
+        tree.getDirectories = () => [firstSubdirectory, searchedSubdirectory, thirdSubdirectory];
+
+        let result = tree.getDirectory(searchedSubdirectory.roomId);
+        expect(result).toBe(searchedSubdirectory);
+
+        result = tree.getDirectory("not a subdirectory");
+        expect(result).toBeFalsy();
+    });
+
+    it('should be able to delete itself', async () => {
+        const delete1 = jest.fn().mockImplementation(() => Promise.resolve());
+        const subdir1 = { delete: delete1 } as any as MSC3089TreeSpace; // mock tested bits
+
+        const delete2 = jest.fn().mockImplementation(() => Promise.resolve());
+        const subdir2 = { delete: delete2 } as any as MSC3089TreeSpace; // mock tested bits
+
+        const joinMemberId = "@join:example.org";
+        const knockMemberId = "@knock:example.org";
+        const inviteMemberId = "@invite:example.org";
+        const leaveMemberId = "@leave:example.org";
+        const banMemberId = "@ban:example.org";
+        const selfUserId = "@self:example.org";
+
+        tree.getDirectories = () => [subdir1, subdir2];
+        room.currentState = {
+            getStateEvents: (eventType: EventType, stateKey?: string) => {
+                expect(eventType).toEqual(EventType.RoomMember);
+                expect(stateKey).toBeUndefined();
+                return [
+                    // Partial implementations
+                    { getContent: () => ({ membership: "join" }), getStateKey: () => joinMemberId },
+                    { getContent: () => ({ membership: "knock" }), getStateKey: () => knockMemberId },
+                    { getContent: () => ({ membership: "invite" }), getStateKey: () => inviteMemberId },
+                    { getContent: () => ({ membership: "leave" }), getStateKey: () => leaveMemberId },
+                    { getContent: () => ({ membership: "ban" }), getStateKey: () => banMemberId },
+
+                    // ensure we don't kick ourselves
+                    { getContent: () => ({ membership: "join" }), getStateKey: () => selfUserId },
+                ];
+            },
+        };
+
+        // These two functions are tested by input expectations, so no expectations in the function bodies
+        const kickFn = jest.fn().mockImplementation((userId) => Promise.resolve());
+        const leaveFn = jest.fn().mockImplementation(() => Promise.resolve());
+        client.kick = kickFn;
+        client.leave = leaveFn;
+        client.getUserId = () => selfUserId;
+
+        await tree.delete();
+
+        expect(delete1).toHaveBeenCalledTimes(1);
+        expect(delete2).toHaveBeenCalledTimes(1);
+        expect(kickFn).toHaveBeenCalledTimes(3);
+        expect(kickFn).toHaveBeenCalledWith(tree.roomId, joinMemberId, expect.any(String));
+        expect(kickFn).toHaveBeenCalledWith(tree.roomId, knockMemberId, expect.any(String));
+        expect(kickFn).toHaveBeenCalledWith(tree.roomId, inviteMemberId, expect.any(String));
+        expect(leaveFn).toHaveBeenCalledTimes(1);
+    });
+
+    describe('get and set order', () => {
+        // Danger: these are partial implementations for testing purposes only
+
+        // @ts-ignore - "MatrixEvent is a value but used as a type", which is true but not important
+        let childState: { [roomId: string]: MatrixEvent[] } = {};
+        // @ts-ignore - "MatrixEvent is a value but used as a type", which is true but not important
+        let parentState: MatrixEvent[] = [];
+        let parentRoom: Room;
+        let childTrees: MSC3089TreeSpace[];
+        let rooms: { [roomId: string]: Room };
+        let clientSendStateFn: jest.MockedFunction<typeof client.sendStateEvent>;
+        const staticDomain = "static.example.org";
+
+        function addSubspace(roomId: string, createTs?: number, order?: string) {
+            const content = {
+                via: [staticDomain],
+            };
+            if (order) content['order'] = order;
+            parentState.push({
+                getType: () => EventType.SpaceChild,
+                getStateKey: () => roomId,
+                getContent: () => content,
+            });
+            childState[roomId] = [
+                {
+                    getType: () => EventType.SpaceParent,
+                    getStateKey: () => tree.roomId,
+                    getContent: () => ({
+                        via: [staticDomain],
+                    }),
+                },
+            ];
+            if (createTs) {
+                childState[roomId].push({
+                    getType: () => EventType.RoomCreate,
+                    getStateKey: () => "",
+                    getContent: () => ({}),
+                    getTs: () => createTs,
+                });
+            }
+            rooms[roomId] = makeMockChildRoom(roomId);
+            childTrees.push(new MSC3089TreeSpace(client, roomId));
+        }
+
+        function expectOrder(childRoomId: string, order: number) {
+            const child = childTrees.find(c => c.roomId === childRoomId);
+            expect(child).toBeDefined();
+            expect(child.getOrder()).toEqual(order);
+        }
+
+        function makeMockChildRoom(roomId: string): Room {
+            return {
+                currentState: {
+                    getStateEvents: (eventType: EventType, stateKey?: string) => {
+                        expect([EventType.SpaceParent, EventType.RoomCreate]).toContain(eventType);
+                        if (eventType === EventType.RoomCreate) {
+                            expect(stateKey).toEqual("");
+                            return childState[roomId].find(e => e.getType() === EventType.RoomCreate);
+                        } else {
+                            expect(stateKey).toBeUndefined();
+                            return childState[roomId].filter(e => e.getType() === eventType);
+                        }
+                    },
+                },
+            } as Room; // partial
+        }
+
+        beforeEach(() => {
+            childState = {};
+            parentState = [];
+            parentRoom = {
+                ...tree.room,
+                roomId: tree.roomId,
+                currentState: {
+                    getStateEvents: (eventType: EventType, stateKey?: string) => {
+                        expect([
+                            EventType.SpaceChild,
+                            EventType.RoomCreate,
+                            EventType.SpaceParent,
+                        ]).toContain(eventType);
+
+                        if (eventType === EventType.RoomCreate) {
+                            expect(stateKey).toEqual("");
+                            return parentState.filter(e => e.getType() === EventType.RoomCreate)[0];
+                        } else {
+                            if (stateKey !== undefined) {
+                                expect(Object.keys(rooms)).toContain(stateKey);
+                                expect(stateKey).not.toEqual(tree.roomId);
+                                return parentState.find(e => e.getType() === eventType && e.getStateKey() === stateKey);
+                            } // else fine
+                            return parentState.filter(e => e.getType() === eventType);
+                        }
+                    },
+                },
+            } as Room;
+            childTrees = [];
+            rooms = {};
+            rooms[tree.roomId] = parentRoom;
+            (<any>tree).room = parentRoom; // override readonly
+            client.getRoom = (r) => rooms[r];
+
+            clientSendStateFn = jest.fn()
+                .mockImplementation((roomId: string, eventType: EventType, content: any, stateKey: string) => {
+                    expect(roomId).toEqual(tree.roomId);
+                    expect(eventType).toEqual(EventType.SpaceChild);
+                    expect(content).toMatchObject(expect.objectContaining({
+                        via: expect.any(Array),
+                        order: expect.any(String),
+                    }));
+                    expect(Object.keys(rooms)).toContain(stateKey);
+                    expect(stateKey).not.toEqual(tree.roomId);
+
+                    const stateEvent = parentState.find(e => e.getType() === eventType && e.getStateKey() === stateKey);
+                    expect(stateEvent).toBeDefined();
+                    stateEvent.getContent = () => content;
+
+                    return Promise.resolve(); // return value not used
+                });
+            client.sendStateEvent = clientSendStateFn;
+        });
+
+        it('should know when something is top level', () => {
+            const a = "!a:example.org";
+            addSubspace(a);
+
+            expect(tree.isTopLevel).toBe(true);
+            expect(childTrees[0].isTopLevel).toBe(false); // a bit of a hack to get at this, but it's fine
+        });
+
+        it('should return -1 for top level spaces', () => {
+            // The tree is what we've defined as top level, so it should work
+            expect(tree.getOrder()).toEqual(-1);
+        });
+
+        it('should throw when setting an order at the top level space', async () => {
+            try {
+                // The tree is what we've defined as top level, so it should work
+                await tree.setOrder(2);
+
+                // noinspection ExceptionCaughtLocallyJS
+                throw new Error("Failed to fail");
+            } catch (e) {
+                expect(e.message).toEqual("Cannot set order of top level spaces currently");
+            }
+        });
+
+        it('should return a stable order for unordered children', () => {
+            const a = "!a:example.org";
+            const b = "!b:example.org";
+            const c = "!c:example.org";
+
+            // Add in reverse order to make sure it gets ordered correctly
+            addSubspace(c, 3);
+            addSubspace(b, 2);
+            addSubspace(a, 1);
+
+            expectOrder(a, 0);
+            expectOrder(b, 1);
+            expectOrder(c, 2);
+        });
+
+        it('should return a stable order for ordered children', () => {
+            const a = "!a:example.org";
+            const b = "!b:example.org";
+            const c = "!c:example.org";
+
+            // Add in reverse order to make sure it gets ordered correctly
+            addSubspace(a, 1, "Z");
+            addSubspace(b, 2, "Y");
+            addSubspace(c, 3, "X");
+
+            expectOrder(c, 0);
+            expectOrder(b, 1);
+            expectOrder(a, 2);
+        });
+
+        it('should return a stable order for partially ordered children', () => {
+            const a = "!a:example.org";
+            const b = "!b:example.org";
+            const c = "!c:example.org";
+            const d = "!d:example.org";
+
+            // Add in reverse order to make sure it gets ordered correctly
+            addSubspace(a, 1);
+            addSubspace(b, 2);
+            addSubspace(c, 3, "Y");
+            addSubspace(d, 4, "X");
+
+            expectOrder(d, 0);
+            expectOrder(c, 1);
+            expectOrder(b, 3); // note order diff due to room ID comparison expectation
+            expectOrder(a, 2);
+        });
+
+        it('should return a stable order if the create event timestamps are the same', () => {
+            const a = "!a:example.org";
+            const b = "!b:example.org";
+            const c = "!c:example.org";
+
+            // Add in reverse order to make sure it gets ordered correctly
+            addSubspace(c, 3);
+            addSubspace(b, 3); // same as C
+            addSubspace(a, 3); // same as C
+
+            expectOrder(a, 0);
+            expectOrder(b, 1);
+            expectOrder(c, 2);
+        });
+
+        it('should return a stable order if there are no known create events', () => {
+            const a = "!a:example.org";
+            const b = "!b:example.org";
+            const c = "!c:example.org";
+
+            // Add in reverse order to make sure it gets ordered correctly
+            addSubspace(c);
+            addSubspace(b);
+            addSubspace(a);
+
+            expectOrder(a, 0);
+            expectOrder(b, 1);
+            expectOrder(c, 2);
+        });
+
+        // XXX: These tests rely on `getOrder()` re-calculating and not caching values.
+
+        it('should allow reordering within unordered children', async () => {
+            const a = "!a:example.org";
+            const b = "!b:example.org";
+            const c = "!c:example.org";
+
+            // Add in reverse order to make sure it gets ordered correctly
+            addSubspace(c, 3);
+            addSubspace(b, 2);
+            addSubspace(a, 1);
+
+            // Order of this state is validated by other tests.
+
+            const treeA = childTrees.find(c => c.roomId === a);
+            expect(treeA).toBeDefined();
+            await treeA.setOrder(1);
+
+            expect(clientSendStateFn).toHaveBeenCalledTimes(3);
+            expect(clientSendStateFn).toHaveBeenCalledWith(tree.roomId, EventType.SpaceChild, expect.objectContaining({
+                via: [staticDomain], // should retain domain independent of client.getDomain()
+
+                // Because of how the reordering works (maintain stable ordering before moving), we end up calling this
+                // function twice for the same room.
+                order: DEFAULT_ALPHABET[0],
+            }), a);
+            expect(clientSendStateFn).toHaveBeenCalledWith(tree.roomId, EventType.SpaceChild, expect.objectContaining({
+                via: [staticDomain], // should retain domain independent of client.getDomain()
+                order: DEFAULT_ALPHABET[1],
+            }), b);
+            expect(clientSendStateFn).toHaveBeenCalledWith(tree.roomId, EventType.SpaceChild, expect.objectContaining({
+                via: [staticDomain], // should retain domain independent of client.getDomain()
+                order: DEFAULT_ALPHABET[2],
+            }), a);
+            expectOrder(a, 1);
+            expectOrder(b, 0);
+            expectOrder(c, 2);
+        });
+
+        it('should allow reordering within ordered children', async () => {
+            const a = "!a:example.org";
+            const b = "!b:example.org";
+            const c = "!c:example.org";
+
+            // Add in reverse order to make sure it gets ordered correctly
+            addSubspace(c, 3, "Z");
+            addSubspace(b, 2, "X");
+            addSubspace(a, 1, "V");
+
+            // Order of this state is validated by other tests.
+
+            const treeA = childTrees.find(c => c.roomId === a);
+            expect(treeA).toBeDefined();
+            await treeA.setOrder(1);
+
+            expect(clientSendStateFn).toHaveBeenCalledTimes(1);
+            expect(clientSendStateFn).toHaveBeenCalledWith(tree.roomId, EventType.SpaceChild, expect.objectContaining({
+                via: [staticDomain], // should retain domain independent of client.getDomain()
+                order: 'Y',
+            }), a);
+            expectOrder(a, 1);
+            expectOrder(b, 0);
+            expectOrder(c, 2);
+        });
+
+        it('should allow reordering within partially ordered children', async () => {
+            const a = "!a:example.org";
+            const b = "!b:example.org";
+            const c = "!c:example.org";
+            const d = "!d:example.org";
+
+            // Add in reverse order to make sure it gets ordered correctly
+            addSubspace(a, 1);
+            addSubspace(b, 2);
+            addSubspace(c, 3, "Y");
+            addSubspace(d, 4, "W");
+
+            // Order of this state is validated by other tests.
+
+            const treeA = childTrees.find(c => c.roomId === a);
+            expect(treeA).toBeDefined();
+            await treeA.setOrder(2);
+
+            expect(clientSendStateFn).toHaveBeenCalledTimes(1);
+            expect(clientSendStateFn).toHaveBeenCalledWith(tree.roomId, EventType.SpaceChild, expect.objectContaining({
+                via: [staticDomain], // should retain domain independent of client.getDomain()
+                order: 'Z',
+            }), a);
+            expectOrder(a, 2);
+            expectOrder(b, 3);
+            expectOrder(c, 1);
+            expectOrder(d, 0);
+        });
+
+        it('should support moving upwards', async () => {
+            const a = "!a:example.org";
+            const b = "!b:example.org";
+            const c = "!c:example.org";
+            const d = "!d:example.org";
+
+            // Add in reverse order to make sure it gets ordered correctly
+            addSubspace(d, 4, "Z");
+            addSubspace(c, 3, "X");
+            addSubspace(b, 2, "V");
+            addSubspace(a, 1, "T");
+
+            // Order of this state is validated by other tests.
+
+            const treeB = childTrees.find(c => c.roomId === b);
+            expect(treeB).toBeDefined();
+            await treeB.setOrder(2);
+
+            expect(clientSendStateFn).toHaveBeenCalledTimes(1);
+            expect(clientSendStateFn).toHaveBeenCalledWith(tree.roomId, EventType.SpaceChild, expect.objectContaining({
+                via: [staticDomain], // should retain domain independent of client.getDomain()
+                order: 'Y',
+            }), b);
+            expectOrder(a, 0);
+            expectOrder(b, 2);
+            expectOrder(c, 1);
+            expectOrder(d, 3);
+        });
+
+        it('should support moving downwards', async () => {
+            const a = "!a:example.org";
+            const b = "!b:example.org";
+            const c = "!c:example.org";
+            const d = "!d:example.org";
+
+            // Add in reverse order to make sure it gets ordered correctly
+            addSubspace(d, 4, "Z");
+            addSubspace(c, 3, "X");
+            addSubspace(b, 2, "V");
+            addSubspace(a, 1, "T");
+
+            // Order of this state is validated by other tests.
+
+            const treeC = childTrees.find(ch => ch.roomId === c);
+            expect(treeC).toBeDefined();
+            await treeC.setOrder(1);
+
+            expect(clientSendStateFn).toHaveBeenCalledTimes(1);
+            expect(clientSendStateFn).toHaveBeenCalledWith(tree.roomId, EventType.SpaceChild, expect.objectContaining({
+                via: [staticDomain], // should retain domain independent of client.getDomain()
+                order: 'U',
+            }), c);
+            expectOrder(a, 0);
+            expectOrder(b, 2);
+            expectOrder(c, 1);
+            expectOrder(d, 3);
+        });
+
+        it('should support moving over the partial ordering boundary', async () => {
+            const a = "!a:example.org";
+            const b = "!b:example.org";
+            const c = "!c:example.org";
+            const d = "!d:example.org";
+
+            // Add in reverse order to make sure it gets ordered correctly
+            addSubspace(d, 4);
+            addSubspace(c, 3);
+            addSubspace(b, 2, "V");
+            addSubspace(a, 1, "T");
+
+            // Order of this state is validated by other tests.
+
+            const treeB = childTrees.find(ch => ch.roomId === b);
+            expect(treeB).toBeDefined();
+            await treeB.setOrder(2);
+
+            expect(clientSendStateFn).toHaveBeenCalledTimes(2);
+            expect(clientSendStateFn).toHaveBeenCalledWith(tree.roomId, EventType.SpaceChild, expect.objectContaining({
+                via: [staticDomain], // should retain domain independent of client.getDomain()
+                order: 'W',
+            }), c);
+            expect(clientSendStateFn).toHaveBeenCalledWith(tree.roomId, EventType.SpaceChild, expect.objectContaining({
+                via: [staticDomain], // should retain domain independent of client.getDomain()
+                order: 'X',
+            }), b);
+            expectOrder(a, 0);
+            expectOrder(b, 2);
+            expectOrder(c, 1);
+            expectOrder(d, 3);
+        });
+    });
+
+    it('should upload files', async () => {
+        const mxc = "mxc://example.org/file";
+        const fileInfo = {
+            mimetype: "text/plain",
+            // other fields as required by encryption, but ignored here
+        };
+        const fileEventId = "$file";
+        const fileName = "My File.txt";
+        const fileContents = "This is a test file";
+
+        // Mock out Blob for the test environment
+        (<any>global).Blob = MockBlob;
+
+        const uploadFn = jest.fn().mockImplementation((contents: Blob, opts: any) => {
+            expect(contents).toBeInstanceOf(Blob);
+            expect(contents.size).toEqual(fileContents.length);
+            expect(opts).toMatchObject({
+                includeFilename: false,
+                onlyContentUri: true, // because the tests rely on this - we shouldn't really be testing  for this.
+            });
+            return Promise.resolve(mxc);
+        });
+        client.uploadContent = uploadFn;
+
+        const sendMsgFn = jest.fn().mockImplementation((roomId: string, contents: any) => {
+            expect(roomId).toEqual(tree.roomId);
+            expect(contents).toMatchObject({
+                msgtype: MsgType.File,
+                body: fileName,
+                url: mxc,
+                file: fileInfo,
+                [UNSTABLE_MSC3089_LEAF.unstable]: {}, // test to ensure we're definitely using unstable
+            });
+
+            return Promise.resolve({ event_id: fileEventId }); // eslint-disable-line camelcase
+        });
+        client.sendMessage = sendMsgFn;
+
+        const sendStateFn = jest.fn()
+            .mockImplementation((roomId: string, eventType: string, content: any, stateKey: string) => {
+                expect(roomId).toEqual(tree.roomId);
+                expect(eventType).toEqual(UNSTABLE_MSC3089_BRANCH.unstable); // test to ensure we're definitely using unstable
+                expect(stateKey).toEqual(fileEventId);
+                expect(content).toMatchObject({
+                    active: true,
+                    name: fileName,
+                });
+
+                return Promise.resolve(); // return value not used.
+            });
+        client.sendStateEvent = sendStateFn;
+
+        const buf = Uint8Array.from(Array.from(fileContents).map((_, i) => fileContents.charCodeAt(i)));
+
+        // We clone the file info just to make sure it doesn't get mutated for the test.
+        await tree.createFile(fileName, buf, Object.assign({}, fileInfo));
+
+        expect(uploadFn).toHaveBeenCalledTimes(1);
+        expect(sendMsgFn).toHaveBeenCalledTimes(1);
+        expect(sendStateFn).toHaveBeenCalledTimes(1);
+    });
+
+    it('should support getting files', () => {
+        const fileEventId = "$file";
+        const fileEvent = { forTest: true }; // MatrixEvent mock
+        room.currentState = {
+            getStateEvents: (eventType: string, stateKey?: string) => {
+                expect(eventType).toEqual(UNSTABLE_MSC3089_BRANCH.unstable); // test to ensure we're definitely using unstable
+                expect(stateKey).toEqual(fileEventId);
+                return fileEvent;
+            },
+        };
+
+        const file = tree.getFile(fileEventId);
+        expect(file).toBeDefined();
+        expect(file.indexEvent).toBe(fileEvent);
+    });
+
+    it('should return falsy for unknown files', () => {
+        const fileEventId = "$file";
+        room.currentState = {
+            getStateEvents: (eventType: string, stateKey?: string) => {
+                expect(eventType).toEqual(UNSTABLE_MSC3089_BRANCH.unstable); // test to ensure we're definitely using unstable
+                expect(stateKey).toEqual(fileEventId);
+                return null;
+            },
+        };
+
+        const file = tree.getFile(fileEventId);
+        expect(file).toBeFalsy();
+    });
+
+    it('should list files', () => {
+        const firstFile = { getContent: () => ({ active: true }) };
+        const secondFile = { getContent: () => ({ active: false }) }; // deliberately inactive
+        room.currentState = {
+            getStateEvents: (eventType: string, stateKey?: string) => {
+                expect(eventType).toEqual(UNSTABLE_MSC3089_BRANCH.unstable); // test to ensure we're definitely using unstable
+                expect(stateKey).toBeUndefined();
+                return [firstFile, secondFile];
+            },
+        };
+
+        const files = tree.listFiles();
+        expect(files).toBeDefined();
+        expect(files.length).toEqual(1);
+        expect(files[0].indexEvent).toBe(firstFile);
+    });
+});

spec/unit/pushprocessor.spec.js

@@ -1,5 +1,5 @@
 import * as utils from "../test-utils";
-import {PushProcessor} from "../../src/pushprocessor";
+import { PushProcessor } from "../../src/pushprocessor";
 
 describe('NotificationService', function() {
     const testUserId = "@ali:matrix.org";

spec/unit/realtime-callbacks.spec.js

@@ -46,8 +46,8 @@ describe("realtime-callbacks", function() {
         it("should set 'this' to the global object", function() {
             let passed = false;
             const callback = function() {
-                expect(this).toBe(global); // eslint-disable-line babel/no-invalid-this
-                expect(this.console).toBeTruthy(); // eslint-disable-line babel/no-invalid-this
+                expect(this).toBe(global); // eslint-disable-line @babel/no-invalid-this
+                expect(this.console).toBeTruthy(); // eslint-disable-line @babel/no-invalid-this
                 passed = true;
             };
             callbacks.setTimeout(callback);

spec/unit/relations.spec.ts

@@ -0,0 +1,133 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventTimelineSet } from "../../src/models/event-timeline-set";
+import { MatrixEvent } from "../../src/models/event";
+import { Relations } from "../../src/models/relations";
+
+describe("Relations", function() {
+    it("should deduplicate annotations", function() {
+        const relations = new Relations("m.annotation", "m.reaction");
+
+        // Create an instance of an annotation
+        const eventData = {
+            "sender": "@bob:example.com",
+            "type": "m.reaction",
+            "event_id": "$cZ1biX33ENJqIm00ks0W_hgiO_6CHrsAc3ZQrnLeNTw",
+            "room_id": "!pzVjCQSoQPpXQeHpmK:example.com",
+            "content": {
+                "m.relates_to": {
+                    "event_id": "$2s4yYpEkVQrPglSCSqB_m6E8vDhWsg0yFNyOJdVIb_o",
+                    "key": "👍️",
+                    "rel_type": "m.annotation",
+                },
+            },
+        };
+        const eventA = new MatrixEvent(eventData);
+
+        // Add the event once and check results
+        {
+            relations.addEvent(eventA);
+            const annotationsByKey = relations.getSortedAnnotationsByKey();
+            expect(annotationsByKey.length).toEqual(1);
+            const [key, events] = annotationsByKey[0];
+            expect(key).toEqual("👍️");
+            expect(events.size).toEqual(1);
+        }
+
+        // Add the event again and expect the same
+        {
+            relations.addEvent(eventA);
+            const annotationsByKey = relations.getSortedAnnotationsByKey();
+            expect(annotationsByKey.length).toEqual(1);
+            const [key, events] = annotationsByKey[0];
+            expect(key).toEqual("👍️");
+            expect(events.size).toEqual(1);
+        }
+
+        // Create a fresh object with the same event content
+        const eventB = new MatrixEvent(eventData);
+
+        // Add the event again and expect the same
+        {
+            relations.addEvent(eventB);
+            const annotationsByKey = relations.getSortedAnnotationsByKey();
+            expect(annotationsByKey.length).toEqual(1);
+            const [key, events] = annotationsByKey[0];
+            expect(key).toEqual("👍️");
+            expect(events.size).toEqual(1);
+        }
+    });
+
+    it("should emit created regardless of ordering", async function() {
+        const targetEvent = new MatrixEvent({
+            "sender": "@bob:example.com",
+            "type": "m.room.message",
+            "event_id": "$2s4yYpEkVQrPglSCSqB_m6E8vDhWsg0yFNyOJdVIb_o",
+            "room_id": "!pzVjCQSoQPpXQeHpmK:example.com",
+            "content": {},
+        });
+        const relationEvent = new MatrixEvent({
+            "sender": "@bob:example.com",
+            "type": "m.reaction",
+            "event_id": "$cZ1biX33ENJqIm00ks0W_hgiO_6CHrsAc3ZQrnLeNTw",
+            "room_id": "!pzVjCQSoQPpXQeHpmK:example.com",
+            "content": {
+                "m.relates_to": {
+                    "event_id": "$2s4yYpEkVQrPglSCSqB_m6E8vDhWsg0yFNyOJdVIb_o",
+                    "key": "👍️",
+                    "rel_type": "m.annotation",
+                },
+            },
+        });
+
+        // Stub the room
+        const room = {
+            getPendingEvent() { return null; },
+            getUnfilteredTimelineSet() { return null; },
+        };
+
+        // Add the target event first, then the relation event
+        {
+            const relationsCreated = new Promise(resolve => {
+                targetEvent.once("Event.relationsCreated", resolve);
+            });
+
+            const timelineSet = new EventTimelineSet(room, {
+                unstableClientRelationAggregation: true,
+            });
+            timelineSet.addLiveEvent(targetEvent);
+            timelineSet.addLiveEvent(relationEvent);
+
+            await relationsCreated;
+        }
+
+        // Add the relation event first, then the target event
+        {
+            const relationsCreated = new Promise(resolve => {
+                targetEvent.once("Event.relationsCreated", resolve);
+            });
+
+            const timelineSet = new EventTimelineSet(room, {
+                unstableClientRelationAggregation: true,
+            });
+            timelineSet.addLiveEvent(relationEvent);
+            timelineSet.addLiveEvent(targetEvent);
+
+            await relationsCreated;
+        }
+    });
+});

spec/unit/room-member.spec.js

@@ -1,5 +1,5 @@
 import * as utils from "../test-utils";
-import {RoomMember} from "../../src/models/room-member";
+import { RoomMember } from "../../src/models/room-member";
 
 describe("RoomMember", function() {
     const roomId = "!foo:bar";
@@ -33,12 +33,6 @@ describe("RoomMember", function() {
             expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
         });
 
-        it("should return an identicon HTTP URL if allowDefault was set and there " +
-        "was no m.room.member event", function() {
-            const url = member.getAvatarUrl(hsUrl, 64, 64, "crop", true);
-            expect(url.indexOf("http")).toEqual(0); // don't care about form
-        });
-
         it("should return nothing if there is no m.room.member and allowDefault=false",
         function() {
             const url = member.getAvatarUrl(hsUrl, 64, 64, "crop", false);

spec/unit/room-state.spec.js

@@ -1,6 +1,6 @@
 import * as utils from "../test-utils";
-import {RoomState} from "../../src/models/room-state";
-import {RoomMember} from "../../src/models/room-member";
+import { RoomState } from "../../src/models/room-state";
+import { RoomMember } from "../../src/models/room-member";
 
 describe("RoomState", function() {
     const roomId = "!foo:bar";
@@ -471,13 +471,13 @@ describe("RoomState", function() {
 
         it("should update after adding joined member", function() {
             state.setStateEvents([
-                utils.mkMembership({event: true, mship: "join",
-                    user: userA, room: roomId}),
+                utils.mkMembership({ event: true, mship: "join",
+                    user: userA, room: roomId }),
             ]);
             expect(state.getJoinedMemberCount()).toEqual(1);
             state.setStateEvents([
-                utils.mkMembership({event: true, mship: "join",
-                    user: userC, room: roomId}),
+                utils.mkMembership({ event: true, mship: "join",
+                    user: userC, room: roomId }),
             ]);
             expect(state.getJoinedMemberCount()).toEqual(2);
         });
@@ -490,13 +490,13 @@ describe("RoomState", function() {
 
         it("should update after adding invited member", function() {
             state.setStateEvents([
-                utils.mkMembership({event: true, mship: "invite",
-                    user: userA, room: roomId}),
+                utils.mkMembership({ event: true, mship: "invite",
+                    user: userA, room: roomId }),
             ]);
             expect(state.getInvitedMemberCount()).toEqual(1);
             state.setStateEvents([
-                utils.mkMembership({event: true, mship: "invite",
-                    user: userC, room: roomId}),
+                utils.mkMembership({ event: true, mship: "invite",
+                    user: userC, room: roomId }),
             ]);
             expect(state.getInvitedMemberCount()).toEqual(2);
         });
@@ -509,15 +509,15 @@ describe("RoomState", function() {
 
         it("should, once used, override counting members from state", function() {
             state.setStateEvents([
-                utils.mkMembership({event: true, mship: "join",
-                    user: userA, room: roomId}),
+                utils.mkMembership({ event: true, mship: "join",
+                    user: userA, room: roomId }),
             ]);
             expect(state.getJoinedMemberCount()).toEqual(1);
             state.setJoinedMemberCount(100);
             expect(state.getJoinedMemberCount()).toEqual(100);
             state.setStateEvents([
-                utils.mkMembership({event: true, mship: "join",
-                    user: userC, room: roomId}),
+                utils.mkMembership({ event: true, mship: "join",
+                    user: userC, room: roomId }),
             ]);
             expect(state.getJoinedMemberCount()).toEqual(100);
         });
@@ -525,14 +525,14 @@ describe("RoomState", function() {
         it("should, once used, override counting members from state, " +
         "also after clone", function() {
             state.setStateEvents([
-                utils.mkMembership({event: true, mship: "join",
-                    user: userA, room: roomId}),
+                utils.mkMembership({ event: true, mship: "join",
+                    user: userA, room: roomId }),
             ]);
             state.setJoinedMemberCount(100);
             const copy = state.clone();
             copy.setStateEvents([
-                utils.mkMembership({event: true, mship: "join",
-                    user: userC, room: roomId}),
+                utils.mkMembership({ event: true, mship: "join",
+                    user: userC, room: roomId }),
             ]);
             expect(state.getJoinedMemberCount()).toEqual(100);
         });
@@ -545,15 +545,15 @@ describe("RoomState", function() {
 
         it("should, once used, override counting members from state", function() {
             state.setStateEvents([
-                utils.mkMembership({event: true, mship: "invite",
-                    user: userB, room: roomId}),
+                utils.mkMembership({ event: true, mship: "invite",
+                    user: userB, room: roomId }),
             ]);
             expect(state.getInvitedMemberCount()).toEqual(1);
             state.setInvitedMemberCount(100);
             expect(state.getInvitedMemberCount()).toEqual(100);
             state.setStateEvents([
-                utils.mkMembership({event: true, mship: "invite",
-                    user: userC, room: roomId}),
+                utils.mkMembership({ event: true, mship: "invite",
+                    user: userC, room: roomId }),
             ]);
             expect(state.getInvitedMemberCount()).toEqual(100);
         });
@@ -561,14 +561,14 @@ describe("RoomState", function() {
         it("should, once used, override counting members from state, " +
         "also after clone", function() {
             state.setStateEvents([
-                utils.mkMembership({event: true, mship: "invite",
-                    user: userB, room: roomId}),
+                utils.mkMembership({ event: true, mship: "invite",
+                    user: userB, room: roomId }),
             ]);
             state.setInvitedMemberCount(100);
             const copy = state.clone();
             copy.setStateEvents([
-                utils.mkMembership({event: true, mship: "invite",
-                    user: userC, room: roomId}),
+                utils.mkMembership({ event: true, mship: "invite",
+                    user: userC, room: roomId }),
             ]);
             expect(state.getInvitedMemberCount()).toEqual(100);
         });

spec/unit/room.spec.js

@@ -1,8 +1,9 @@
 import * as utils from "../test-utils";
-import {EventStatus, MatrixEvent} from "../../src/models/event";
-import {EventTimeline} from "../../src/models/event-timeline";
-import {RoomState} from "../../src/models/room-state";
-import {Room} from "../../src/models/room";
+import { EventStatus, MatrixEvent } from "../../src/models/event";
+import { EventTimeline } from "../../src/models/event-timeline";
+import { RoomState } from "../../src/models/room-state";
+import { Room } from "../../src/models/room";
+import { TestClient } from "../TestClient";
 
 describe("Room", function() {
     const roomId = "!foo:bar";
@@ -45,12 +46,6 @@ describe("Room", function() {
             expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
         });
 
-        it("should return an identicon HTTP URL if allowDefault was set and there " +
-        "was no m.room.avatar event", function() {
-            const url = room.getAvatarUrl(hsUrl, 64, 64, "crop", true);
-            expect(url.indexOf("http")).toEqual(0); // don't care about form
-        });
-
         it("should return nothing if there is no m.room.avatar and allowDefault=false",
         function() {
             const url = room.getAvatarUrl(hsUrl, 64, 64, "crop", false);
@@ -196,7 +191,7 @@ describe("Room", function() {
             const remoteEvent = utils.mkMessage({
                 room: roomId, user: userA, event: true,
             });
-            remoteEvent.event.unsigned = {transaction_id: "TXN_ID"};
+            remoteEvent.event.unsigned = { transaction_id: "TXN_ID" };
             const remoteEventId = remoteEvent.getId();
 
             let callCount = 0;
@@ -380,7 +375,7 @@ describe("Room", function() {
         let events = null;
 
         beforeEach(function() {
-            room = new Room(roomId, null, null, {timelineSupport: timelineSupport});
+            room = new Room(roomId, null, null, { timelineSupport: timelineSupport });
             // set events each time to avoid resusing Event objects (which
             // doesn't work because they get frozen)
             events = [
@@ -462,7 +457,7 @@ describe("Room", function() {
 
     describe("compareEventOrdering", function() {
         beforeEach(function() {
-            room = new Room(roomId, null, null, {timelineSupport: true});
+            room = new Room(roomId, null, null, { timelineSupport: true });
         });
 
         const events = [
@@ -718,7 +713,7 @@ describe("Room", function() {
             it("uses hero name from state", function() {
                 const name = "Mr B";
                 addMember(userA, "invite");
-                addMember(userB, "join", {name});
+                addMember(userB, "join", { name });
                 room.setSummary({
                     "m.heroes": [userB],
                 });
@@ -729,7 +724,7 @@ describe("Room", function() {
 
             it("uses counts from summary", function() {
                 const name = "Mr B";
-                addMember(userB, "join", {name});
+                addMember(userB, "join", { name });
                 room.setSummary({
                     "m.heroes": [userB],
                     "m.joined_member_count": 50,
@@ -742,8 +737,8 @@ describe("Room", function() {
             it("relies on heroes in case of absent counts", function() {
                 const nameB = "Mr Bean";
                 const nameC = "Mel C";
-                addMember(userB, "join", {name: nameB});
-                addMember(userC, "join", {name: nameC});
+                addMember(userB, "join", { name: nameB });
+                addMember(userC, "join", { name: nameC });
                 room.setSummary({
                     "m.heroes": [userB, userC],
                 });
@@ -753,7 +748,7 @@ describe("Room", function() {
 
             it("uses only heroes", function() {
                 const nameB = "Mr Bean";
-                addMember(userB, "join", {name: nameB});
+                addMember(userB, "join", { name: nameB });
                 addMember(userC, "join");
                 room.setSummary({
                     "m.heroes": [userB],
@@ -846,7 +841,7 @@ describe("Room", function() {
             it("should show the other user's name for private" +
             " (invite join_rules) rooms if you are invited to it.", function() {
                 setJoinRule("invite");
-                addMember(userA, "invite", {user: userB});
+                addMember(userA, "invite", { user: userB });
                 addMember(userB);
                 room.recalculate();
                 const name = room.name;
@@ -921,8 +916,8 @@ describe("Room", function() {
                "available",
             function() {
                 setJoinRule("invite");
-                addMember(userB, 'join', {name: "Alice"});
-                addMember(userA, "invite", {user: userA});
+                addMember(userB, 'join', { name: "Alice" });
+                addMember(userA, "invite", { user: userA });
                 room.recalculate();
                 const name = room.name;
                 expect(name).toEqual("Alice");
@@ -932,7 +927,7 @@ describe("Room", function() {
             function() {
                 setJoinRule("invite");
                 addMember(userB);
-                addMember(userA, "invite", {user: userA});
+                addMember(userA, "invite", { user: userA });
                 room.recalculate();
                 const name = room.name;
                 expect(name).toEqual(userB);
@@ -1182,7 +1177,10 @@ describe("Room", function() {
     describe("addPendingEvent", function() {
         it("should add pending events to the pendingEventList if " +
                       "pendingEventOrdering == 'detached'", function() {
-            const room = new Room(roomId, null, userA, {
+            const client = (new TestClient(
+                "@alice:example.com", "alicedevice",
+            )).client;
+            const room = new Room(roomId, client, userA, {
                 pendingEventOrdering: "detached",
             });
             const eventA = utils.mkMessage({
@@ -1232,7 +1230,10 @@ describe("Room", function() {
 
     describe("updatePendingEvent", function() {
         it("should remove cancelled events from the pending list", function() {
-            const room = new Room(roomId, null, userA, {
+            const client = (new TestClient(
+                "@alice:example.com", "alicedevice",
+            )).client;
+            const room = new Room(roomId, client, userA, {
                 pendingEventOrdering: "detached",
             });
             const eventA = utils.mkMessage({
@@ -1266,7 +1267,6 @@ describe("Room", function() {
             expect(callCount).toEqual(1);
         });
 
-
         it("should remove cancelled events from the timeline", function() {
             const room = new Room(roomId, null, userA);
             const eventA = utils.mkMessage({
@@ -1314,13 +1314,13 @@ describe("Room", function() {
                 isRoomEncrypted: function() {
                     return false;
                 },
-                _http: {
+                http: {
                     serverResponse,
                     authedRequest: function() {
                         if (this.serverResponse instanceof Error) {
                             return Promise.reject(this.serverResponse);
                         } else {
-                            return Promise.resolve({chunk: this.serverResponse});
+                            return Promise.resolve({ chunk: this.serverResponse });
                         }
                     },
                 },
@@ -1350,7 +1350,7 @@ describe("Room", function() {
 
         it("should load members from server on first call", async function() {
             const client = createClientMock([memberEvent]);
-            const room = new Room(roomId, client, null, {lazyLoadMembers: true});
+            const room = new Room(roomId, client, null, { lazyLoadMembers: true });
             await room.loadMembersIfNeeded();
             const memberA = room.getMember("@user_a:bar");
             expect(memberA.name).toEqual("User A");
@@ -1365,7 +1365,7 @@ describe("Room", function() {
                 room: roomId, event: true, name: "Ms A",
             });
             const client = createClientMock([memberEvent2], [memberEvent]);
-            const room = new Room(roomId, client, null, {lazyLoadMembers: true});
+            const room = new Room(roomId, client, null, { lazyLoadMembers: true });
 
             await room.loadMembersIfNeeded();
 
@@ -1375,16 +1375,16 @@ describe("Room", function() {
 
         it("should allow retry on error", async function() {
             const client = createClientMock(new Error("server says no"));
-            const room = new Room(roomId, client, null, {lazyLoadMembers: true});
+            const room = new Room(roomId, client, null, { lazyLoadMembers: true });
             let hasThrown = false;
             try {
                 await room.loadMembersIfNeeded();
-            } catch(err) {
+            } catch (err) {
                 hasThrown = true;
             }
             expect(hasThrown).toEqual(true);
 
-            client._http.serverResponse = [memberEvent];
+            client.http.serverResponse = [memberEvent];
             await room.loadMembersIfNeeded();
             const memberA = room.getMember("@user_a:bar");
             expect(memberA.name).toEqual("User A");
@@ -1403,17 +1403,17 @@ describe("Room", function() {
             const room = new Room(roomId, null, userA);
             const events = [];
             room.on("Room.myMembership", (_room, membership, oldMembership) => {
-                events.push({membership, oldMembership});
+                events.push({ membership, oldMembership });
             });
             room.updateMyMembership("invite");
             expect(room.getMyMembership()).toEqual("invite");
-            expect(events[0]).toEqual({membership: "invite", oldMembership: null});
+            expect(events[0]).toEqual({ membership: "invite", oldMembership: null });
             events.splice(0);   //clear
             room.updateMyMembership("invite");
             expect(events.length).toEqual(0);
             room.updateMyMembership("join");
             expect(room.getMyMembership()).toEqual("join");
-            expect(events[0]).toEqual({membership: "join", oldMembership: "invite"});
+            expect(events[0]).toEqual({ membership: "join", oldMembership: "invite" });
         });
     });
 
@@ -1421,7 +1421,7 @@ describe("Room", function() {
         it("should return first hero id",
         function() {
             const room = new Room(roomId, null, userA);
-            room.setSummary({'m.heroes': [userB]});
+            room.setSummary({ 'm.heroes': [userB] });
             expect(room.guessDMUserId()).toEqual(userB);
         });
         it("should return first member that isn't self",

spec/unit/scheduler.spec.js

@@ -1,9 +1,9 @@
 // This file had a function whose name is all caps, which displeases eslint
 /* eslint new-cap: "off" */
 
-import {defer} from '../../src/utils';
-import {MatrixError} from "../../src/http-api";
-import {MatrixScheduler} from "../../src/scheduler";
+import { defer } from '../../src/utils';
+import { MatrixError } from "../../src/http-api";
+import { MatrixScheduler } from "../../src/scheduler";
 import * as utils from "../test-utils";
 
 jest.useFakeTimers();
@@ -62,8 +62,8 @@ describe("MatrixScheduler", function() {
             scheduler.queueEvent(eventA),
             scheduler.queueEvent(eventB),
         ]);
-        deferB.resolve({b: true});
-        deferA.resolve({a: true});
+        deferB.resolve({ b: true });
+        deferA.resolve({ a: true });
         const [a, b] = await abPromise;
         expect(a.a).toEqual(true);
         expect(b.b).toEqual(true);
@@ -143,7 +143,7 @@ describe("MatrixScheduler", function() {
         deferA.reject({});
         try {
             await globalA;
-        } catch(err) {
+        } catch (err) {
             await Promise.resolve();
             expect(procCount).toEqual(2);
         }
@@ -156,8 +156,8 @@ describe("MatrixScheduler", function() {
         // Expect to have processFn invoked for A&B.
         // Resolve A.
         // Expect to have processFn invoked for D.
-        const eventC = utils.mkMessage({user: "@a:bar", room: roomId, event: true});
-        const eventD = utils.mkMessage({user: "@b:bar", room: roomId, event: true});
+        const eventC = utils.mkMessage({ user: "@a:bar", room: roomId, event: true });
+        const eventD = utils.mkMessage({ user: "@b:bar", room: roomId, event: true });
 
         const buckets = {};
         buckets[eventA.getId()] = "queue_A";
@@ -241,7 +241,7 @@ describe("MatrixScheduler", function() {
             expect(queue).toEqual([eventA, eventB]);
             // modify the queue
             const eventC = utils.mkMessage(
-                {user: "@a:bar", room: roomId, event: true},
+                { user: "@a:bar", room: roomId, event: true },
             );
             queue.push(eventC);
             const queueAgain = scheduler.getQueueForEvent(eventA);

spec/unit/sync-accumulator.spec.js

@@ -15,7 +15,40 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {SyncAccumulator} from "../../src/sync-accumulator";
+import { SyncAccumulator } from "../../src/sync-accumulator";
+
+// The event body & unsigned object get frozen to assert that they don't get altered
+// by the impl
+const RES_WITH_AGE = {
+    next_batch: "abc",
+    rooms: {
+        invite: {},
+        leave: {},
+        join: {
+            "!foo:bar": {
+                account_data: { events: [] },
+                ephemeral: { events: [] },
+                unread_notifications: {},
+                timeline: {
+                    events: [
+                        Object.freeze({
+                            content: {
+                                body: "This thing is happening right now!",
+                            },
+                            origin_server_ts: 123456789,
+                            sender: "@alice:localhost",
+                            type: "m.room.message",
+                            unsigned: Object.freeze({
+                                age: 50,
+                            }),
+                        }),
+                    ],
+                    prev_batch: "something",
+                },
+            },
+        },
+    },
+};
 
 describe("SyncAccumulator", function() {
     let sa;
@@ -368,6 +401,39 @@ describe("SyncAccumulator", function() {
             expect(summary["m.joined_member_count"]).toEqual(5);
             expect(summary["m.heroes"]).toEqual(["@bob:bar"]);
         });
+
+        it("should return correctly adjusted age attributes", () => {
+            const delta = 1000;
+            const startingTs = 1000;
+
+            const oldDateNow = Date.now;
+            try {
+                Date.now = jest.fn();
+                Date.now.mockReturnValue(startingTs);
+
+                sa.accumulate(RES_WITH_AGE);
+
+                Date.now.mockReturnValue(startingTs + delta);
+
+                const output = sa.getJSON();
+                expect(output.roomsData.join["!foo:bar"].timeline.events[0].unsigned.age).toEqual(
+                    RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0].unsigned.age + delta,
+                );
+                expect(Object.keys(output.roomsData.join["!foo:bar"].timeline.events[0])).toEqual(
+                    Object.keys(RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0]),
+                );
+            } finally {
+                Date.now = oldDateNow;
+            }
+        });
+
+        it("should mangle age without adding extra keys", () => {
+            sa.accumulate(RES_WITH_AGE);
+            const output = sa.getJSON();
+            expect(Object.keys(output.roomsData.join["!foo:bar"].timeline.events[0])).toEqual(
+                Object.keys(RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0]),
+            );
+        });
     });
 });
 

spec/unit/timeline-window.spec.js

@@ -1,5 +1,5 @@
-import {EventTimeline} from "../../src/models/event-timeline";
-import {TimelineIndex, TimelineWindow} from "../../src/timeline-window";
+import { EventTimeline } from "../../src/models/event-timeline";
+import { TimelineIndex, TimelineWindow } from "../../src/timeline-window";
 import * as utils from "../test-utils";
 
 const ROOM_ID = "roomId";
@@ -18,7 +18,7 @@ function createTimeline(numEvents, baseIndex) {
     }
 
     // XXX: this is a horrid hack
-    const timelineSet = { room: { roomId: ROOM_ID }};
+    const timelineSet = { room: { roomId: ROOM_ID } };
     timelineSet.room.getUnfilteredTimelineSet = function() {
         return timelineSet;
     };
@@ -46,7 +46,6 @@ function addEventsToTimeline(timeline, numEvents, atStart) {
     }
 }
 
-
 /*
  * create a pair of linked timelines
  */
@@ -58,7 +57,6 @@ function createLinkedTimelines() {
     return [tl1, tl2];
 }
 
-
 describe("TimelineIndex", function() {
     describe("minIndex", function() {
         it("should return the min index relative to BaseIndex", function() {
@@ -133,7 +131,6 @@ describe("TimelineIndex", function() {
     });
 });
 
-
 describe("TimelineWindow", function() {
     /**
      * create a dummy eventTimelineSet and client, and a TimelineWindow
@@ -142,7 +139,7 @@ describe("TimelineWindow", function() {
     let timelineSet;
     let client;
     function createWindow(timeline, opts) {
-        timelineSet = {getTimelineForEvent: () => null};
+        timelineSet = { getTimelineForEvent: () => null };
         client = {};
         client.getEventTimeline = function(timelineSet0, eventId0) {
             expect(timelineSet0).toBe(timelineSet);
@@ -171,7 +168,7 @@ describe("TimelineWindow", function() {
             const timeline = createTimeline();
             const eventId = timeline.getEvents()[1].getId();
 
-            const timelineSet = {getTimelineForEvent: () => null};
+            const timelineSet = { getTimelineForEvent: () => null };
             const client = {};
             client.getEventTimeline = function(timelineSet0, eventId0) {
                 expect(timelineSet0).toBe(timelineSet);
@@ -193,7 +190,7 @@ describe("TimelineWindow", function() {
 
             const eventId = timeline.getEvents()[1].getId();
 
-            const timelineSet = {getTimelineForEvent: () => null};
+            const timelineSet = { getTimelineForEvent: () => null };
             const client = {};
 
             const timelineWindow = new TimelineWindow(client, timelineSet);
@@ -266,7 +263,7 @@ describe("TimelineWindow", function() {
         it("should advance into next timeline", function() {
             const tls = createLinkedTimelines();
             const eventId = tls[0].getEvents()[1].getId();
-            const timelineWindow = createWindow(tls[0], {windowLimit: 5});
+            const timelineWindow = createWindow(tls[0], { windowLimit: 5 });
 
             return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = tls[0].getEvents();
@@ -311,7 +308,7 @@ describe("TimelineWindow", function() {
         it("should retreat into previous timeline", function() {
             const tls = createLinkedTimelines();
             const eventId = tls[1].getEvents()[1].getId();
-            const timelineWindow = createWindow(tls[1], {windowLimit: 5});
+            const timelineWindow = createWindow(tls[1], { windowLimit: 5 });
 
             return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = tls[1].getEvents();
@@ -357,7 +354,7 @@ describe("TimelineWindow", function() {
             const timeline = createTimeline();
             timeline.setPaginationToken("toktok", EventTimeline.FORWARDS);
 
-            const timelineWindow = createWindow(timeline, {windowLimit: 5});
+            const timelineWindow = createWindow(timeline, { windowLimit: 5 });
             const eventId = timeline.getEvents()[1].getId();
 
             client.paginateEventTimeline = function(timeline0, opts) {
@@ -385,12 +382,11 @@ describe("TimelineWindow", function() {
             });
         });
 
-
         it("should make backward pagination requests", function() {
             const timeline = createTimeline();
             timeline.setPaginationToken("toktok", EventTimeline.BACKWARDS);
 
-            const timelineWindow = createWindow(timeline, {windowLimit: 5});
+            const timelineWindow = createWindow(timeline, { windowLimit: 5 });
             const eventId = timeline.getEvents()[1].getId();
 
             client.paginateEventTimeline = function(timeline0, opts) {
@@ -422,7 +418,7 @@ describe("TimelineWindow", function() {
             const timeline = createTimeline();
             timeline.setPaginationToken("toktok", EventTimeline.FORWARDS);
 
-            const timelineWindow = createWindow(timeline, {windowLimit: 5});
+            const timelineWindow = createWindow(timeline, { windowLimit: 5 });
             const eventId = timeline.getEvents()[1].getId();
 
             let paginateCount = 0;

spec/unit/user.spec.js

@@ -1,4 +1,4 @@
-import {User} from "../../src/models/user";
+import { User } from "../../src/models/user";
 import * as utils from "../test-utils";
 
 describe("User", function() {

spec/unit/utils.spec.js

@@ -1,285 +0,0 @@
-import * as utils from "../../src/utils";
-
-describe("utils", function() {
-    describe("encodeParams", function() {
-        it("should url encode and concat with &s", function() {
-            const params = {
-                foo: "bar",
-                baz: "beer@",
-            };
-            expect(utils.encodeParams(params)).toEqual(
-                "foo=bar&baz=beer%40",
-            );
-        });
-    });
-
-    describe("encodeUri", function() {
-        it("should replace based on object keys and url encode", function() {
-            const path = "foo/bar/%something/%here";
-            const vals = {
-                "%something": "baz",
-                "%here": "beer@",
-            };
-            expect(utils.encodeUri(path, vals)).toEqual(
-                "foo/bar/baz/beer%40",
-            );
-        });
-    });
-
-    describe("forEach", function() {
-        it("should be invoked for each element", function() {
-            const arr = [];
-            utils.forEach([55, 66, 77], function(element) {
-                arr.push(element);
-            });
-            expect(arr).toEqual([55, 66, 77]);
-        });
-    });
-
-    describe("findElement", function() {
-        it("should find only 1 element if there is a match", function() {
-            const matchFn = function() {
-                return true;
-            };
-            const arr = [55, 66, 77];
-            expect(utils.findElement(arr, matchFn)).toEqual(55);
-        });
-        it("should be able to find in reverse order", function() {
-            const matchFn = function() {
-                return true;
-            };
-            const arr = [55, 66, 77];
-            expect(utils.findElement(arr, matchFn, true)).toEqual(77);
-        });
-        it("should find nothing if the function never returns true", function() {
-            const matchFn = function() {
-                return false;
-            };
-            const arr = [55, 66, 77];
-            expect(utils.findElement(arr, matchFn)).toBeFalsy();
-        });
-    });
-
-    describe("removeElement", function() {
-        it("should remove only 1 element if there is a match", function() {
-            const matchFn = function() {
-                return true;
-            };
-            const arr = [55, 66, 77];
-            utils.removeElement(arr, matchFn);
-            expect(arr).toEqual([66, 77]);
-        });
-        it("should be able to remove in reverse order", function() {
-            const matchFn = function() {
-                return true;
-            };
-            const arr = [55, 66, 77];
-            utils.removeElement(arr, matchFn, true);
-            expect(arr).toEqual([55, 66]);
-        });
-        it("should remove nothing if the function never returns true", function() {
-            const matchFn = function() {
-                return false;
-            };
-            const arr = [55, 66, 77];
-            utils.removeElement(arr, matchFn);
-            expect(arr).toEqual(arr);
-        });
-    });
-
-    describe("isFunction", function() {
-        it("should return true for functions", function() {
-            expect(utils.isFunction([])).toBe(false);
-            expect(utils.isFunction([5, 3, 7])).toBe(false);
-            expect(utils.isFunction()).toBe(false);
-            expect(utils.isFunction(null)).toBe(false);
-            expect(utils.isFunction({})).toBe(false);
-            expect(utils.isFunction("foo")).toBe(false);
-            expect(utils.isFunction(555)).toBe(false);
-
-            expect(utils.isFunction(function() {})).toBe(true);
-            const s = { foo: function() {} };
-            expect(utils.isFunction(s.foo)).toBe(true);
-        });
-    });
-
-    describe("isArray", function() {
-        it("should return true for arrays", function() {
-            expect(utils.isArray([])).toBe(true);
-            expect(utils.isArray([5, 3, 7])).toBe(true);
-
-            expect(utils.isArray()).toBe(false);
-            expect(utils.isArray(null)).toBe(false);
-            expect(utils.isArray({})).toBe(false);
-            expect(utils.isArray("foo")).toBe(false);
-            expect(utils.isArray(555)).toBe(false);
-            expect(utils.isArray(function() {})).toBe(false);
-        });
-    });
-
-    describe("checkObjectHasKeys", function() {
-        it("should throw for missing keys", function() {
-            expect(function() {
-                utils.checkObjectHasKeys({}, ["foo"]);
-            }).toThrow();
-            expect(function() {
-                utils.checkObjectHasKeys({
-                    foo: "bar",
-                }, ["foo"]);
-            }).not.toThrow();
-        });
-    });
-
-    describe("checkObjectHasNoAdditionalKeys", function() {
-        it("should throw for extra keys", function() {
-            expect(function() {
-                utils.checkObjectHasNoAdditionalKeys({
-                            foo: "bar",
-                            baz: 4,
-                        }, ["foo"]);
-            }).toThrow();
-
-            expect(function() {
-                utils.checkObjectHasNoAdditionalKeys({
-                            foo: "bar",
-                        }, ["foo"]);
-            }).not.toThrow();
-        });
-    });
-
-    describe("deepCompare", function() {
-        const assert = {
-            isTrue: function(x) {
-                expect(x).toBe(true);
-            },
-            isFalse: function(x) {
-                expect(x).toBe(false);
-            },
-        };
-
-        it("should handle primitives", function() {
-            assert.isTrue(utils.deepCompare(null, null));
-            assert.isFalse(utils.deepCompare(null, undefined));
-            assert.isTrue(utils.deepCompare("hi", "hi"));
-            assert.isTrue(utils.deepCompare(5, 5));
-            assert.isFalse(utils.deepCompare(5, 10));
-        });
-
-        it("should handle regexps", function() {
-            assert.isTrue(utils.deepCompare(/abc/, /abc/));
-            assert.isFalse(utils.deepCompare(/abc/, /123/));
-            const r = /abc/;
-            assert.isTrue(utils.deepCompare(r, r));
-        });
-
-        it("should handle dates", function() {
-            assert.isTrue(utils.deepCompare(new Date("2011-03-31"),
-                                            new Date("2011-03-31")));
-            assert.isFalse(utils.deepCompare(new Date("2011-03-31"),
-                                             new Date("1970-01-01")));
-        });
-
-        it("should handle arrays", function() {
-            assert.isTrue(utils.deepCompare([], []));
-            assert.isTrue(utils.deepCompare([1, 2], [1, 2]));
-            assert.isFalse(utils.deepCompare([1, 2], [2, 1]));
-            assert.isFalse(utils.deepCompare([1, 2], [1, 2, 3]));
-        });
-
-        it("should handle simple objects", function() {
-            assert.isTrue(utils.deepCompare({}, {}));
-            assert.isTrue(utils.deepCompare({a: 1, b: 2}, {a: 1, b: 2}));
-            assert.isTrue(utils.deepCompare({a: 1, b: 2}, {b: 2, a: 1}));
-            assert.isFalse(utils.deepCompare({a: 1, b: 2}, {a: 1, b: 3}));
-
-            assert.isTrue(utils.deepCompare({1: {name: "mhc", age: 28},
-                                             2: {name: "arb", age: 26}},
-                                            {1: {name: "mhc", age: 28},
-                                             2: {name: "arb", age: 26}}));
-
-            assert.isFalse(utils.deepCompare({1: {name: "mhc", age: 28},
-                                              2: {name: "arb", age: 26}},
-                                             {1: {name: "mhc", age: 28},
-                                              2: {name: "arb", age: 27}}));
-
-            assert.isFalse(utils.deepCompare({}, null));
-            assert.isFalse(utils.deepCompare({}, undefined));
-        });
-
-        it("should handle functions", function() {
-            // no two different function is equal really, they capture their
-            // context variables so even if they have same toString(), they
-            // won't have same functionality
-            const func = function(x) {
-                return true;
-            };
-            const func2 = function(x) {
-                return true;
-            };
-            assert.isTrue(utils.deepCompare(func, func));
-            assert.isFalse(utils.deepCompare(func, func2));
-            assert.isTrue(utils.deepCompare({ a: { b: func } }, { a: { b: func } }));
-            assert.isFalse(utils.deepCompare({ a: { b: func } }, { a: { b: func2 } }));
-        });
-    });
-
-
-    describe("extend", function() {
-        const SOURCE = { "prop2": 1, "string2": "x", "newprop": "new" };
-
-        it("should extend", function() {
-            const target = {
-                "prop1": 5, "prop2": 7, "string1": "baz", "string2": "foo",
-            };
-            const merged = {
-                "prop1": 5, "prop2": 1, "string1": "baz", "string2": "x",
-                "newprop": "new",
-            };
-            const sourceOrig = JSON.stringify(SOURCE);
-
-            utils.extend(target, SOURCE);
-            expect(JSON.stringify(target)).toEqual(JSON.stringify(merged));
-
-            // check the originial wasn't modified
-            expect(JSON.stringify(SOURCE)).toEqual(sourceOrig);
-        });
-
-        it("should ignore null", function() {
-            const target = {
-                "prop1": 5, "prop2": 7, "string1": "baz", "string2": "foo",
-            };
-            const merged = {
-                "prop1": 5, "prop2": 1, "string1": "baz", "string2": "x",
-                "newprop": "new",
-            };
-            const sourceOrig = JSON.stringify(SOURCE);
-
-            utils.extend(target, null, SOURCE);
-            expect(JSON.stringify(target)).toEqual(JSON.stringify(merged));
-
-            // check the originial wasn't modified
-            expect(JSON.stringify(SOURCE)).toEqual(sourceOrig);
-        });
-
-        it("should handle properties created with defineProperties", function() {
-            const source = Object.defineProperties({}, {
-                "enumerableProp": {
-                    get: function() {
-                        return true;
-                    },
-                    enumerable: true,
-                },
-                "nonenumerableProp": {
-                    get: function() {
-                        return true;
-                    },
-                },
-            });
-
-            const target = {};
-            utils.extend(target, source);
-            expect(target.enumerableProp).toBe(true);
-            expect(target.nonenumerableProp).toBe(undefined);
-        });
-    });
-});

spec/unit/utils.spec.ts

@@ -0,0 +1,432 @@
+import * as utils from "../../src/utils";
+import {
+    alphabetPad,
+    averageBetweenStrings,
+    baseToString,
+    DEFAULT_ALPHABET,
+    lexicographicCompare,
+    nextString,
+    prevString,
+    stringToBase,
+} from "../../src/utils";
+import { logger } from "../../src/logger";
+
+// TODO: Fix types throughout
+
+describe("utils", function() {
+    describe("encodeParams", function() {
+        it("should url encode and concat with &s", function() {
+            const params = {
+                foo: "bar",
+                baz: "beer@",
+            };
+            expect(utils.encodeParams(params)).toEqual(
+                "foo=bar&baz=beer%40",
+            );
+        });
+    });
+
+    describe("encodeUri", function() {
+        it("should replace based on object keys and url encode", function() {
+            const path = "foo/bar/%something/%here";
+            const vals = {
+                "%something": "baz",
+                "%here": "beer@",
+            };
+            expect(utils.encodeUri(path, vals)).toEqual(
+                "foo/bar/baz/beer%40",
+            );
+        });
+    });
+
+    describe("removeElement", function() {
+        it("should remove only 1 element if there is a match", function() {
+            const matchFn = function() {
+                return true;
+            };
+            const arr = [55, 66, 77];
+            utils.removeElement(arr, matchFn);
+            expect(arr).toEqual([66, 77]);
+        });
+        it("should be able to remove in reverse order", function() {
+            const matchFn = function() {
+                return true;
+            };
+            const arr = [55, 66, 77];
+            utils.removeElement(arr, matchFn, true);
+            expect(arr).toEqual([55, 66]);
+        });
+        it("should remove nothing if the function never returns true", function() {
+            const matchFn = function() {
+                return false;
+            };
+            const arr = [55, 66, 77];
+            utils.removeElement(arr, matchFn);
+            expect(arr).toEqual(arr);
+        });
+    });
+
+    describe("isFunction", function() {
+        it("should return true for functions", function() {
+            expect(utils.isFunction([])).toBe(false);
+            expect(utils.isFunction([5, 3, 7])).toBe(false);
+            expect(utils.isFunction(undefined)).toBe(false);
+            expect(utils.isFunction(null)).toBe(false);
+            expect(utils.isFunction({})).toBe(false);
+            expect(utils.isFunction("foo")).toBe(false);
+            expect(utils.isFunction(555)).toBe(false);
+
+            expect(utils.isFunction(function() {})).toBe(true);
+            const s = { foo: function() {} };
+            expect(utils.isFunction(s.foo)).toBe(true);
+        });
+    });
+
+    describe("checkObjectHasKeys", function() {
+        it("should throw for missing keys", function() {
+            expect(function() {
+                utils.checkObjectHasKeys({}, ["foo"]);
+            }).toThrow();
+            expect(function() {
+                utils.checkObjectHasKeys({
+                    foo: "bar",
+                }, ["foo"]);
+            }).not.toThrow();
+        });
+    });
+
+    describe("checkObjectHasNoAdditionalKeys", function() {
+        it("should throw for extra keys", function() {
+            expect(function() {
+                utils.checkObjectHasNoAdditionalKeys({ foo: "bar", baz: 4 }, ["foo"]);
+            }).toThrow();
+
+            expect(function() {
+                utils.checkObjectHasNoAdditionalKeys({ foo: "bar" }, ["foo"]);
+            }).not.toThrow();
+        });
+    });
+
+    describe("deepCompare", function() {
+        const assert = {
+            isTrue: function(x) {
+                expect(x).toBe(true);
+            },
+            isFalse: function(x) {
+                expect(x).toBe(false);
+            },
+        };
+
+        it("should handle primitives", function() {
+            assert.isTrue(utils.deepCompare(null, null));
+            assert.isFalse(utils.deepCompare(null, undefined));
+            assert.isTrue(utils.deepCompare("hi", "hi"));
+            assert.isTrue(utils.deepCompare(5, 5));
+            assert.isFalse(utils.deepCompare(5, 10));
+        });
+
+        it("should handle regexps", function() {
+            assert.isTrue(utils.deepCompare(/abc/, /abc/));
+            assert.isFalse(utils.deepCompare(/abc/, /123/));
+            const r = /abc/;
+            assert.isTrue(utils.deepCompare(r, r));
+        });
+
+        it("should handle dates", function() {
+            assert.isTrue(utils.deepCompare(new Date("2011-03-31"), new Date("2011-03-31")));
+            assert.isFalse(utils.deepCompare(new Date("2011-03-31"), new Date("1970-01-01")));
+        });
+
+        it("should handle arrays", function() {
+            assert.isTrue(utils.deepCompare([], []));
+            assert.isTrue(utils.deepCompare([1, 2], [1, 2]));
+            assert.isFalse(utils.deepCompare([1, 2], [2, 1]));
+            assert.isFalse(utils.deepCompare([1, 2], [1, 2, 3]));
+        });
+
+        it("should handle simple objects", function() {
+            assert.isTrue(utils.deepCompare({}, {}));
+            assert.isTrue(utils.deepCompare({ a: 1, b: 2 }, { a: 1, b: 2 }));
+            assert.isTrue(utils.deepCompare({ a: 1, b: 2 }, { b: 2, a: 1 }));
+            assert.isFalse(utils.deepCompare({ a: 1, b: 2 }, { a: 1, b: 3 }));
+
+            assert.isTrue(utils.deepCompare({
+                1: { name: "mhc", age: 28 },
+                2: { name: "arb", age: 26 },
+            }, {
+                1: { name: "mhc", age: 28 },
+                2: { name: "arb", age: 26 },
+            }));
+
+            assert.isFalse(utils.deepCompare({
+                1: { name: "mhc", age: 28 },
+                2: { name: "arb", age: 26 },
+            }, {
+                1: { name: "mhc", age: 28 },
+                2: { name: "arb", age: 27 },
+            }));
+
+            assert.isFalse(utils.deepCompare({}, null));
+            assert.isFalse(utils.deepCompare({}, undefined));
+        });
+
+        it("should handle functions", function() {
+            // no two different function is equal really, they capture their
+            // context variables so even if they have same toString(), they
+            // won't have same functionality
+            const func = function(x) {
+                return true;
+            };
+            const func2 = function(x) {
+                return true;
+            };
+            assert.isTrue(utils.deepCompare(func, func));
+            assert.isFalse(utils.deepCompare(func, func2));
+            assert.isTrue(utils.deepCompare({ a: { b: func } }, { a: { b: func } }));
+            assert.isFalse(utils.deepCompare({ a: { b: func } }, { a: { b: func2 } }));
+        });
+    });
+
+    describe("extend", function() {
+        const SOURCE = { "prop2": 1, "string2": "x", "newprop": "new" };
+
+        it("should extend", function() {
+            const target = {
+                "prop1": 5, "prop2": 7, "string1": "baz", "string2": "foo",
+            };
+            const merged = {
+                "prop1": 5, "prop2": 1, "string1": "baz", "string2": "x",
+                "newprop": "new",
+            };
+            const sourceOrig = JSON.stringify(SOURCE);
+
+            utils.extend(target, SOURCE);
+            expect(JSON.stringify(target)).toEqual(JSON.stringify(merged));
+
+            // check the originial wasn't modified
+            expect(JSON.stringify(SOURCE)).toEqual(sourceOrig);
+        });
+
+        it("should ignore null", function() {
+            const target = {
+                "prop1": 5, "prop2": 7, "string1": "baz", "string2": "foo",
+            };
+            const merged = {
+                "prop1": 5, "prop2": 1, "string1": "baz", "string2": "x",
+                "newprop": "new",
+            };
+            const sourceOrig = JSON.stringify(SOURCE);
+
+            utils.extend(target, null, SOURCE);
+            expect(JSON.stringify(target)).toEqual(JSON.stringify(merged));
+
+            // check the originial wasn't modified
+            expect(JSON.stringify(SOURCE)).toEqual(sourceOrig);
+        });
+
+        it("should handle properties created with defineProperties", function() {
+            const source = Object.defineProperties({}, {
+                "enumerableProp": {
+                    get: function() {
+                        return true;
+                    },
+                    enumerable: true,
+                },
+                "nonenumerableProp": {
+                    get: function() {
+                        return true;
+                    },
+                },
+            });
+
+            // TODO: Fix type
+            const target: any = {};
+            utils.extend(target, source);
+            expect(target.enumerableProp).toBe(true);
+            expect(target.nonenumerableProp).toBe(undefined);
+        });
+    });
+
+    describe("chunkPromises", function() {
+        it("should execute promises in chunks", async function() {
+            let promiseCount = 0;
+
+            async function fn1() {
+                await utils.sleep(1);
+                expect(promiseCount).toEqual(0);
+                ++promiseCount;
+            }
+
+            async function fn2() {
+                expect(promiseCount).toEqual(1);
+                ++promiseCount;
+            }
+
+            await utils.chunkPromises([fn1, fn2], 1);
+            expect(promiseCount).toEqual(2);
+        });
+    });
+
+    describe('DEFAULT_ALPHABET', () => {
+        it('should be usefully printable ASCII in order', () => {
+            expect(DEFAULT_ALPHABET).toEqual(
+                " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~",
+            );
+        });
+    });
+
+    describe('alphabetPad', () => {
+        it('should pad to the alphabet length', () => {
+            const len = 12;
+            expect(alphabetPad("a", len)).toEqual("a" + ("".padEnd(len - 1, DEFAULT_ALPHABET[0])));
+            expect(alphabetPad("a", len, "123")).toEqual("a" + ("".padEnd(len - 1, '1')));
+        });
+    });
+
+    describe('baseToString', () => {
+        it('should calculate the appropriate string from numbers', () => {
+            // Verify the whole alphabet
+            for (let i = BigInt(1); i <= DEFAULT_ALPHABET.length; i++) {
+                logger.log({ i }); // for debugging
+                expect(baseToString(i)).toEqual(DEFAULT_ALPHABET[Number(i) - 1]);
+            }
+
+            // Just quickly double check that repeated characters aren't treated as padding, particularly
+            // at the beginning of the alphabet where they are most vulnerable to this behaviour.
+            expect(baseToString(BigInt(1))).toEqual(DEFAULT_ALPHABET[0].repeat(1));
+            expect(baseToString(BigInt(96))).toEqual(DEFAULT_ALPHABET[0].repeat(2));
+            expect(baseToString(BigInt(9121))).toEqual(DEFAULT_ALPHABET[0].repeat(3));
+            expect(baseToString(BigInt(866496))).toEqual(DEFAULT_ALPHABET[0].repeat(4));
+            expect(baseToString(BigInt(82317121))).toEqual(DEFAULT_ALPHABET[0].repeat(5));
+            expect(baseToString(BigInt(7820126496))).toEqual(DEFAULT_ALPHABET[0].repeat(6));
+
+            expect(baseToString(BigInt(10))).toEqual(DEFAULT_ALPHABET[9]);
+            expect(baseToString(BigInt(10), "abcdefghijklmnopqrstuvwxyz")).toEqual('j');
+            expect(baseToString(BigInt(6337))).toEqual("ab");
+            expect(baseToString(BigInt(80), "abcdefghijklmnopqrstuvwxyz")).toEqual('cb');
+        });
+    });
+
+    describe('stringToBase', () => {
+        it('should calculate the appropriate number for a string', () => {
+            expect(stringToBase(DEFAULT_ALPHABET[0].repeat(1))).toEqual(BigInt(1));
+            expect(stringToBase(DEFAULT_ALPHABET[0].repeat(2))).toEqual(BigInt(96));
+            expect(stringToBase(DEFAULT_ALPHABET[0].repeat(3))).toEqual(BigInt(9121));
+            expect(stringToBase(DEFAULT_ALPHABET[0].repeat(4))).toEqual(BigInt(866496));
+            expect(stringToBase(DEFAULT_ALPHABET[0].repeat(5))).toEqual(BigInt(82317121));
+            expect(stringToBase(DEFAULT_ALPHABET[0].repeat(6))).toEqual(BigInt(7820126496));
+            expect(stringToBase("a", "abcdefghijklmnopqrstuvwxyz")).toEqual(BigInt(1));
+            expect(stringToBase("a")).toEqual(BigInt(66));
+            expect(stringToBase("c", "abcdefghijklmnopqrstuvwxyz")).toEqual(BigInt(3));
+            expect(stringToBase("ab")).toEqual(BigInt(6337));
+            expect(stringToBase("cb", "abcdefghijklmnopqrstuvwxyz")).toEqual(BigInt(80));
+        });
+    });
+
+    describe('averageBetweenStrings', () => {
+        it('should average appropriately', () => {
+            expect(averageBetweenStrings("  ", "!!")).toEqual(" P");
+            expect(averageBetweenStrings(" ", "!")).toEqual("  ");
+            expect(averageBetweenStrings('A', 'B')).toEqual('A ');
+            expect(averageBetweenStrings('AA', 'BB')).toEqual('Aq');
+            expect(averageBetweenStrings('A', 'z')).toEqual(']');
+            expect(averageBetweenStrings('a', 'z', "abcdefghijklmnopqrstuvwxyz")).toEqual('m');
+            expect(averageBetweenStrings('AA', 'zz')).toEqual('^.');
+            expect(averageBetweenStrings('aa', 'zz', "abcdefghijklmnopqrstuvwxyz")).toEqual('mz');
+            expect(averageBetweenStrings('cat', 'doggo')).toEqual("d9>Cw");
+            expect(averageBetweenStrings('cat', 'doggo', "abcdefghijklmnopqrstuvwxyz")).toEqual("cumqh");
+        });
+    });
+
+    describe('nextString', () => {
+        it('should find the next string appropriately', () => {
+            expect(nextString('A')).toEqual('B');
+            expect(nextString('b', 'abcdefghijklmnopqrstuvwxyz')).toEqual('c');
+            expect(nextString('cat')).toEqual('cau');
+            expect(nextString('cat', 'abcdefghijklmnopqrstuvwxyz')).toEqual('cau');
+        });
+    });
+
+    describe('prevString', () => {
+        it('should find the next string appropriately', () => {
+            expect(prevString('B')).toEqual('A');
+            expect(prevString('c', 'abcdefghijklmnopqrstuvwxyz')).toEqual('b');
+            expect(prevString('cau')).toEqual('cat');
+            expect(prevString('cau', 'abcdefghijklmnopqrstuvwxyz')).toEqual('cat');
+        });
+    });
+
+    // Let's just ensure the ordering is sensible for lexicographic ordering
+    describe('string averaging unified', () => {
+        it('should be truly previous and next', () => {
+            let midpoint = "cat";
+
+            // We run this test 100 times to ensure we end up with a sane sequence.
+            for (let i = 0; i < 100; i++) {
+                const next = nextString(midpoint);
+                const prev = prevString(midpoint);
+                logger.log({ i, midpoint, next, prev }); // for test debugging
+
+                expect(lexicographicCompare(midpoint, next) < 0).toBe(true);
+                expect(lexicographicCompare(midpoint, prev) > 0).toBe(true);
+                expect(averageBetweenStrings(prev, next)).toBe(midpoint);
+
+                midpoint = next;
+            }
+        });
+
+        it('should roll over', () => {
+            const lastAlpha = DEFAULT_ALPHABET[DEFAULT_ALPHABET.length - 1];
+            const firstAlpha = DEFAULT_ALPHABET[0];
+
+            const highRoll = firstAlpha + firstAlpha;
+            const lowRoll = lastAlpha;
+
+            expect(nextString(lowRoll)).toEqual(highRoll);
+            expect(prevString(highRoll)).toEqual(lowRoll);
+        });
+
+        it('should be reversible on small strings', () => {
+            // Large scale reversibility is tested for max space order value
+            const input = "cats";
+            expect(prevString(nextString(input))).toEqual(input);
+        });
+
+        // We want to explicitly make sure that Space order values are supported and roll appropriately
+        it('should properly handle rolling over at 50 characters', () => {
+            // Note: we also test reversibility of large strings here.
+
+            const maxSpaceValue = DEFAULT_ALPHABET[DEFAULT_ALPHABET.length - 1].repeat(50);
+            const fiftyFirstChar = DEFAULT_ALPHABET[0].repeat(51);
+
+            expect(nextString(maxSpaceValue)).toBe(fiftyFirstChar);
+            expect(prevString(fiftyFirstChar)).toBe(maxSpaceValue);
+
+            // We're testing that the rollover happened, which means that the next string come before
+            // the maximum space order value lexicographically.
+            expect(lexicographicCompare(maxSpaceValue, fiftyFirstChar) > 0).toBe(true);
+        });
+    });
+
+    describe('lexicographicCompare', () => {
+        it('should work', () => {
+            // Simple tests
+            expect(lexicographicCompare('a', 'b') < 0).toBe(true);
+            expect(lexicographicCompare('ab', 'b') < 0).toBe(true);
+            expect(lexicographicCompare('cat', 'dog') < 0).toBe(true);
+
+            // Simple tests (reversed)
+            expect(lexicographicCompare('b', 'a') > 0).toBe(true);
+            expect(lexicographicCompare('b', 'ab') > 0).toBe(true);
+            expect(lexicographicCompare('dog', 'cat') > 0).toBe(true);
+
+            // Simple equality tests
+            expect(lexicographicCompare('a', 'a') === 0).toBe(true);
+            expect(lexicographicCompare('A', 'A') === 0).toBe(true);
+
+            // ASCII rule testing
+            expect(lexicographicCompare('A', 'a') < 0).toBe(true);
+            expect(lexicographicCompare('a', 'A') > 0).toBe(true);
+        });
+    });
+});

spec/unit/webrtc/call.spec.ts

@@ -0,0 +1,301 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { TestClient } from '../../TestClient';
+import { MatrixCall, CallErrorCode, CallEvent } from '../../../src/webrtc/call';
+
+const DUMMY_SDP = (
+    "v=0\r\n" +
+    "o=- 5022425983810148698 2 IN IP4 127.0.0.1\r\n" +
+    "s=-\r\nt=0 0\r\na=group:BUNDLE 0\r\n" +
+    "a=msid-semantic: WMS h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA\r\n" +
+    "m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126\r\n" +
+    "c=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:hLDR\r\n" +
+    "a=ice-pwd:bMGD9aOldHWiI+6nAq/IIlRw\r\n" +
+    "a=ice-options:trickle\r\n" +
+    "a=fingerprint:sha-256 E4:94:84:F9:4A:98:8A:56:F5:5F:FD:AF:72:B9:32:89:49:5C:4B:9A:" +
+        "4A:15:8E:41:8A:F3:69:E4:39:52:DC:D6\r\n" +
+    "a=setup:active\r\n" +
+    "a=mid:0\r\n" +
+    "a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n" +
+    "a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n" +
+    "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n" +
+    "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n" +
+    "a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n" +
+    "a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n" +
+    "a=sendrecv\r\n" +
+    "a=msid:h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA 4357098f-3795-4131-bff4-9ba9c0348c49\r\n" +
+    "a=rtcp-mux\r\n" +
+    "a=rtpmap:111 opus/48000/2\r\n" +
+    "a=rtcp-fb:111 transport-cc\r\n" +
+    "a=fmtp:111 minptime=10;useinbandfec=1\r\n" +
+    "a=rtpmap:103 ISAC/16000\r\n" +
+    "a=rtpmap:104 ISAC/32000\r\n" +
+    "a=rtpmap:9 G722/8000\r\n" +
+    "a=rtpmap:0 PCMU/8000\r\n" +
+    "a=rtpmap:8 PCMA/8000\r\n" +
+    "a=rtpmap:106 CN/32000\r\n" +
+    "a=rtpmap:105 CN/16000\r\n" +
+    "a=rtpmap:13 CN/8000\r\n" +
+    "a=rtpmap:110 telephone-event/48000\r\n" +
+    "a=rtpmap:112 telephone-event/32000\r\n" +
+    "a=rtpmap:113 telephone-event/16000\r\n" +
+    "a=rtpmap:126 telephone-event/8000\r\n" +
+    "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n"
+);
+
+class MockRTCPeerConnection {
+    localDescription: RTCSessionDescription;
+
+    constructor() {
+        this.localDescription = {
+            sdp: DUMMY_SDP,
+            type: 'offer',
+            toJSON: function() {},
+        };
+    }
+
+    addEventListener() {}
+    createOffer() {
+        return Promise.resolve({});
+    }
+    setRemoteDescription() {
+        return Promise.resolve();
+    }
+    setLocalDescription() {
+        return Promise.resolve();
+    }
+    close() {}
+    getStats() { return []; }
+}
+
+describe('Call', function() {
+    let client;
+    let call;
+    let prevNavigator;
+    let prevDocument;
+    let prevWindow;
+
+    beforeEach(function() {
+        prevNavigator = global.navigator;
+        prevDocument = global.document;
+        prevWindow = global.window;
+
+        global.navigator = {
+            mediaDevices: {
+                // @ts-ignore Mock
+                getUserMedia: () => {
+                    return {
+                        getTracks: () => [],
+                        getAudioTracks: () => [],
+                        getVideoTracks: () => [],
+                    };
+                },
+            },
+        };
+
+        global.window = {
+            // @ts-ignore Mock
+            RTCPeerConnection: MockRTCPeerConnection,
+            // @ts-ignore Mock
+            RTCSessionDescription: {},
+            // @ts-ignore Mock
+            RTCIceCandidate: {},
+            getUserMedia: {},
+        };
+        // @ts-ignore Mock
+        global.document = {};
+
+        client = new TestClient("@alice:foo", "somedevice", "token", undefined, {});
+        // We just stub out sendEvent: we're not interested in testing the client's
+        // event sending code here
+        client.client.sendEvent = () => {};
+        client.httpBackend.when("GET", "/voip/turnServer").respond(200, {});
+        call = new MatrixCall({
+            client: client.client,
+            roomId: '!foo:bar',
+        });
+        // call checks one of these is wired up
+        call.on('error', () => {});
+    });
+
+    afterEach(function() {
+        client.stop();
+        global.navigator = prevNavigator;
+        global.window = prevWindow;
+        global.document = prevDocument;
+    });
+
+    it('should ignore candidate events from non-matching party ID', async function() {
+        const callPromise = call.placeVoiceCall();
+        await client.httpBackend.flush();
+        await callPromise;
+        await call.onAnswerReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    answer: {
+                        sdp: DUMMY_SDP,
+                    },
+                };
+            },
+        });
+
+        call.peerConn.addIceCandidate = jest.fn();
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    candidates: [
+                        {
+                            candidate: '',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1);
+
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'some_other_party_id',
+                    candidates: [
+                        {
+                            candidate: '',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1);
+
+        // Hangup to stop timers
+        call.hangup(CallErrorCode.UserHangup, true);
+    });
+
+    it('should add candidates received before answer if party ID is correct', async function() {
+        const callPromise = call.placeVoiceCall();
+        await client.httpBackend.flush();
+        await callPromise;
+        call.peerConn.addIceCandidate = jest.fn();
+
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    candidates: [
+                        {
+                            candidate: 'the_correct_candidate',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'some_other_party_id',
+                    candidates: [
+                        {
+                            candidate: 'the_wrong_candidate',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(0);
+
+        await call.onAnswerReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    answer: {
+                        sdp: DUMMY_SDP,
+                    },
+                };
+            },
+        });
+
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1);
+        expect(call.peerConn.addIceCandidate).toHaveBeenCalledWith({
+            candidate: 'the_correct_candidate',
+            sdpMid: '',
+        });
+    });
+
+    it('should map asserted identity messages to remoteAssertedIdentity', async function() {
+        const callPromise = call.placeVoiceCall();
+        await client.httpBackend.flush();
+        await callPromise;
+        await call.onAnswerReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'party_id',
+                    answer: {
+                        sdp: DUMMY_SDP,
+                    },
+                };
+            },
+        });
+
+        const identChangedCallback = jest.fn();
+        call.on(CallEvent.AssertedIdentityChanged, identChangedCallback);
+
+        await call.onAssertedIdentityReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'party_id',
+                    asserted_identity: {
+                        id: "@steve:example.com",
+                        display_name: "Steve Gibbons",
+                    },
+                };
+            },
+        });
+
+        expect(identChangedCallback).toHaveBeenCalled();
+
+        const ident = call.getRemoteAssertedIdentity();
+        expect(ident.id).toEqual("@steve:example.com");
+        expect(ident.displayName).toEqual("Steve Gibbons");
+
+        // Hangup to stop timers
+        call.hangup(CallErrorCode.UserHangup, true);
+    });
+});

src/@types/IIdentityServerProvider.ts

@@ -0,0 +1,24 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export interface IIdentityServerProvider {
+    /**
+     * Gets an access token for use against the identity server,
+     * for the associated client.
+     * @returns {Promise<string>} Resolves to the access token.
+     */
+    getAccessToken(): Promise<string>;
+}

src/@types/event.ts

@@ -0,0 +1,155 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { UnstableValue } from "../NamespacedValue";
+
+export enum EventType {
+    // Room state events
+    RoomCanonicalAlias = "m.room.canonical_alias",
+    RoomCreate = "m.room.create",
+    RoomJoinRules = "m.room.join_rules",
+    RoomMember = "m.room.member",
+    RoomThirdPartyInvite = "m.room.third_party_invite",
+    RoomPowerLevels = "m.room.power_levels",
+    RoomName = "m.room.name",
+    RoomTopic = "m.room.topic",
+    RoomAvatar = "m.room.avatar",
+    RoomPinnedEvents = "m.room.pinned_events",
+    RoomEncryption = "m.room.encryption",
+    RoomHistoryVisibility = "m.room.history_visibility",
+    RoomGuestAccess = "m.room.guest_access",
+    RoomServerAcl = "m.room.server_acl",
+    RoomTombstone = "m.room.tombstone",
+    /**
+     * @deprecated Should not be used.
+     */
+    RoomAliases = "m.room.aliases", // deprecated https://matrix.org/docs/spec/client_server/r0.6.1#historical-events
+
+    SpaceChild = "m.space.child",
+    SpaceParent = "m.space.parent",
+
+    // Room timeline events
+    RoomRedaction = "m.room.redaction",
+    RoomMessage = "m.room.message",
+    RoomMessageEncrypted = "m.room.encrypted",
+    Sticker = "m.sticker",
+    CallInvite = "m.call.invite",
+    CallCandidates = "m.call.candidates",
+    CallAnswer = "m.call.answer",
+    CallHangup = "m.call.hangup",
+    CallReject = "m.call.reject",
+    CallSelectAnswer = "m.call.select_answer",
+    CallNegotiate = "m.call.negotiate",
+    CallReplaces = "m.call.replaces",
+    CallAssertedIdentity = "m.call.asserted_identity",
+    CallAssertedIdentityPrefix = "org.matrix.call.asserted_identity",
+    KeyVerificationRequest = "m.key.verification.request",
+    KeyVerificationStart = "m.key.verification.start",
+    KeyVerificationCancel = "m.key.verification.cancel",
+    KeyVerificationMac = "m.key.verification.mac",
+    KeyVerificationDone = "m.key.verification.done",
+    // use of this is discouraged https://matrix.org/docs/spec/client_server/r0.6.1#m-room-message-feedback
+    RoomMessageFeedback = "m.room.message.feedback",
+    Reaction = "m.reaction",
+
+    // Room ephemeral events
+    Typing = "m.typing",
+    Receipt = "m.receipt",
+    Presence = "m.presence",
+
+    // Room account_data events
+    FullyRead = "m.fully_read",
+    Tag = "m.tag",
+    SpaceOrder = "org.matrix.msc3230.space_order", // MSC3230
+
+    // User account_data events
+    PushRules = "m.push_rules",
+    Direct = "m.direct",
+    IgnoredUserList = "m.ignored_user_list",
+
+    // to_device events
+    RoomKey = "m.room_key",
+    RoomKeyRequest = "m.room_key_request",
+    ForwardedRoomKey = "m.forwarded_room_key",
+    Dummy = "m.dummy",
+}
+
+export enum MsgType {
+    Text = "m.text",
+    Emote = "m.emote",
+    Notice = "m.notice",
+    Image = "m.image",
+    File = "m.file",
+    Audio = "m.audio",
+    Location = "m.location",
+    Video = "m.video",
+}
+
+export const RoomCreateTypeField = "type";
+
+export enum RoomType {
+    Space = "m.space",
+}
+
+/**
+ * Identifier for an [MSC3088](https://github.com/matrix-org/matrix-doc/pull/3088)
+ * room purpose. Note that this reference is UNSTABLE and subject to breaking changes,
+ * including its eventual removal.
+ */
+export const UNSTABLE_MSC3088_PURPOSE = new UnstableValue("m.room.purpose", "org.matrix.msc3088.purpose");
+
+/**
+ * Enabled flag for an [MSC3088](https://github.com/matrix-org/matrix-doc/pull/3088)
+ * room purpose. Note that this reference is UNSTABLE and subject to breaking changes,
+ * including its eventual removal.
+ */
+export const UNSTABLE_MSC3088_ENABLED = new UnstableValue("m.enabled", "org.matrix.msc3088.enabled");
+
+/**
+ * Subtype for an [MSC3089](https://github.com/matrix-org/matrix-doc/pull/3089) space-room.
+ * Note that this reference is UNSTABLE and subject to breaking changes, including its
+ * eventual removal.
+ */
+export const UNSTABLE_MSC3089_TREE_SUBTYPE = new UnstableValue("m.data_tree", "org.matrix.msc3089.data_tree");
+
+/**
+ * Leaf type for an event in a [MSC3089](https://github.com/matrix-org/matrix-doc/pull/3089) space-room.
+ * Note that this reference is UNSTABLE and subject to breaking changes, including its
+ * eventual removal.
+ */
+export const UNSTABLE_MSC3089_LEAF = new UnstableValue("m.leaf", "org.matrix.msc3089.leaf");
+
+/**
+ * Branch (Leaf Reference) type for the index approach in a
+ * [MSC3089](https://github.com/matrix-org/matrix-doc/pull/3089) space-room. Note that this reference is
+ * UNSTABLE and subject to breaking changes, including its eventual removal.
+ */
+export const UNSTABLE_MSC3089_BRANCH = new UnstableValue("m.branch", "org.matrix.msc3089.branch");
+
+export interface IEncryptedFile {
+    url: string;
+    mimetype?: string;
+    key: {
+        alg: string;
+        key_ops: string[]; // eslint-disable-line camelcase
+        kty: string;
+        k: string;
+        ext: boolean;
+    };
+    iv: string;
+    hashes: {[alg: string]: string};
+    v: string;
+}

src/@types/global.d.ts

@@ -0,0 +1,87 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// this is needed to tell TS about global.Olm
+import * as Olm from "@matrix-org/olm"; // eslint-disable-line @typescript-eslint/no-unused-vars
+
+export {};
+
+declare global {
+    namespace NodeJS {
+        interface Global {
+            localStorage: Storage;
+        }
+    }
+
+    interface Window {
+        electron?: Electron;
+    }
+
+    interface Electron {
+        getDesktopCapturerSources(options: GetSourcesOptions): Promise<Array<DesktopCapturerSource>>;
+    }
+
+    interface MediaDevices {
+        // This is experimental and types don't know about it yet
+        // https://github.com/microsoft/TypeScript/issues/33232
+        getDisplayMedia(constraints: MediaStreamConstraints | DesktopCapturerConstraints): Promise<MediaStream>;
+        getUserMedia(constraints: MediaStreamConstraints | DesktopCapturerConstraints): Promise<MediaStream>;
+    }
+
+    interface DesktopCapturerConstraints {
+        audio: boolean | {
+            mandatory: {
+                chromeMediaSource: string;
+                chromeMediaSourceId: string;
+            };
+        };
+        video: boolean | {
+            mandatory: {
+                chromeMediaSource: string;
+                chromeMediaSourceId: string;
+            };
+        };
+    }
+
+    interface DesktopCapturerSource {
+        id: string;
+        name: string;
+        thumbnailURL: string;
+    }
+
+    interface GetSourcesOptions {
+        types: Array<string>;
+        thumbnailSize?: {
+            height: number;
+            width: number;
+        };
+        fetchWindowIcons?: boolean;
+    }
+
+    interface HTMLAudioElement {
+        // sinkId & setSinkId are experimental and typescript doesn't know about them
+        sinkId: string;
+        setSinkId(outputId: string);
+    }
+
+    interface DummyInterfaceWeShouldntBeUsingThis {}
+
+    interface Navigator {
+        // We check for the webkit-prefixed getUserMedia to detect if we're
+        // on webkit: we should check if we still need to do this
+        webkitGetUserMedia: DummyInterfaceWeShouldntBeUsingThis;
+    }
+}

src/@types/partials.ts

@@ -0,0 +1,39 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export interface IImageInfo {
+    size?: number;
+    mimetype?: string;
+    thumbnail_info?: { // eslint-disable-line camelcase
+        w?: number;
+        h?: number;
+        size?: number;
+        mimetype?: string;
+    };
+    w?: number;
+    h?: number;
+}
+
+export enum Visibility {
+    Public = "public",
+    Private = "private",
+}
+
+export enum Preset {
+    PrivateChat = "private_chat",
+    TrustedPrivateChat = "trusted_private_chat",
+    PublicChat = "public_chat",
+}

src/@types/requests.ts

@@ -0,0 +1,122 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { Callback } from "../client";
+import { Preset, Visibility } from "./partials";
+
+// allow camelcase as these are things go onto the wire
+/* eslint-disable camelcase */
+
+export interface IJoinRoomOpts {
+    /**
+     * True to do a room initial sync on the resulting
+     * room. If false, the <strong>returned Room object will have no current state.
+     * </strong> Default: true.
+     */
+    syncRoom?: boolean;
+
+    /**
+     * If the caller has a keypair 3pid invite, the signing URL is passed in this parameter.
+     */
+    inviteSignUrl?: string;
+
+    /**
+     * The server names to try and join through in addition to those that are automatically chosen.
+     */
+    viaServers?: string[];
+}
+
+export interface IRedactOpts {
+    reason?: string;
+}
+
+export interface ISendEventResponse {
+    event_id: string;
+}
+
+export interface IPresenceOpts {
+    presence: "online" | "offline" | "unavailable";
+    status_msg?: string;
+}
+
+export interface IPaginateOpts {
+    backwards?: boolean;
+    limit?: number;
+}
+
+export interface IGuestAccessOpts {
+    allowJoin: boolean;
+    allowRead: boolean;
+}
+
+export interface ISearchOpts {
+    keys?: string[];
+    query: string;
+}
+
+export interface IEventSearchOpts {
+    filter: any; // TODO: Types
+    term: string;
+}
+
+export interface IInvite3PID {
+    id_server: string;
+    id_access_token?: string; // this gets injected by the js-sdk
+    medium: string;
+    address: string;
+}
+
+export interface ICreateRoomStateEvent {
+    type: string;
+    state_key?: string; // defaults to an empty string
+    content: object;
+}
+
+export interface ICreateRoomOpts {
+    room_alias_name?: string;
+    visibility?: Visibility;
+    name?: string;
+    topic?: string;
+    preset?: Preset;
+    power_level_content_override?: object;
+    creation_content?: object;
+    initial_state?: ICreateRoomStateEvent[];
+    invite?: string[];
+    invite_3pid?: IInvite3PID[];
+    is_direct?: boolean;
+    room_version?: string;
+}
+
+export interface IRoomDirectoryOptions {
+    server?: string;
+    limit?: number;
+    since?: string;
+
+    // TODO: Proper types
+    filter?: any & {generic_search_term: string};
+}
+
+export interface IUploadOpts {
+    name?: string;
+    includeFilename?: boolean;
+    type?: string;
+    rawResponse?: boolean;
+    onlyContentUri?: boolean;
+    callback?: Callback;
+    progressHandler?: (state: {loaded: number, total: number}) => void;
+}
+
+/* eslint-enable camelcase */

src/@types/signed.ts

@@ -0,0 +1,21 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export interface ISignatures {
+    [entity: string]: {
+        [keyId: string]: string;
+    };
+}

src/NamespacedValue.ts

@@ -0,0 +1,93 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Represents a simple Matrix namespaced value. This will assume that if a stable prefix
+ * is provided that the stable prefix should be used when representing the identifier.
+ */
+export class NamespacedValue<S extends string, U extends string> {
+    // Stable is optional, but one of the two parameters is required, hence the weird-looking types.
+    // Goal is to to have developers explicitly say there is no stable value (if applicable).
+    public constructor(public readonly stable: S | null | undefined, public readonly unstable?: U) {
+        if (!this.unstable && !this.stable) {
+            throw new Error("One of stable or unstable values must be supplied");
+        }
+    }
+
+    public get name(): U | S {
+        if (this.stable) {
+            return this.stable;
+        }
+        return this.unstable;
+    }
+
+    public get altName(): U | S | null {
+        if (!this.stable) {
+            return null;
+        }
+        return this.unstable;
+    }
+
+    public matches(val: string): boolean {
+        return this.name === val || this.altName === val;
+    }
+
+    // this desperately wants https://github.com/microsoft/TypeScript/pull/26349 at the top level of the class
+    // so we can instantiate `NamespacedValue<string, _, _>` as a default type for that namespace.
+    public findIn<T>(obj: any): T {
+        let val: T;
+        if (this.name) {
+            val = obj?.[this.name];
+        }
+        if (!val && this.altName) {
+            val = obj?.[this.altName];
+        }
+        return val;
+    }
+
+    public includedIn(arr: any[]): boolean {
+        let included = false;
+        if (this.name) {
+            included = arr.includes(this.name);
+        }
+        if (!included && this.altName) {
+            included = arr.includes(this.altName);
+        }
+        return included;
+    }
+}
+
+/**
+ * Represents a namespaced value which prioritizes the unstable value over the stable
+ * value.
+ */
+export class UnstableValue<S extends string, U extends string> extends NamespacedValue<S, U> {
+    // Note: Constructor difference is that `unstable` is *required*.
+    public constructor(stable: S, unstable: U) {
+        super(stable, unstable);
+        if (!this.unstable) {
+            throw new Error("Unstable value must be supplied");
+        }
+    }
+
+    public get name(): U {
+        return this.unstable;
+    }
+
+    public get altName(): S {
+        return this.stable;
+    }
+}

src/ReEmitter.js

@@ -1,52 +0,0 @@
-/*
-Copyright 2015, 2016 OpenMarket Ltd
-Copyright 2017 Vector Creations Ltd
-Copyright 2017 New Vector Ltd
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/**
- * @module
- */
-
-export class ReEmitter {
-    constructor(target) {
-        this.target = target;
-
-        // We keep one bound event handler for each event name so we know
-        // what event is arriving
-        this.boundHandlers = {};
-    }
-
-    _handleEvent(eventName, ...args) {
-        this.target.emit(eventName, ...args);
-    }
-
-    reEmit(source, eventNames) {
-        // We include the source as the last argument for event handlers which may need it,
-        // such as read receipt listeners on the client class which won't have the context
-        // of the room.
-        const forSource = (handler, ...args) => {
-            handler(...args, source);
-        };
-        for (const eventName of eventNames) {
-            if (this.boundHandlers[eventName] === undefined) {
-                this.boundHandlers[eventName] = this._handleEvent.bind(this, eventName);
-            }
-
-            const boundHandler = forSource.bind(this, this.boundHandlers[eventName]);
-            source.on(eventName, boundHandler);
-        }
-    }
-}

src/ReEmitter.ts

@@ -0,0 +1,50 @@
+/*
+Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventEmitter } from "events";
+
+export class ReEmitter {
+    private target: EventEmitter;
+
+    constructor(target: EventEmitter) {
+        this.target = target;
+    }
+
+    reEmit(source: EventEmitter, eventNames: string[]) {
+        for (const eventName of eventNames) {
+            // We include the source as the last argument for event handlers which may need it,
+            // such as read receipt listeners on the client class which won't have the context
+            // of the room.
+            const forSource = (...args) => {
+                // EventEmitter special cases 'error' to make the emit function throw if no
+                // handler is attached, which sort of makes sense for making sure that something
+                // handles an error, but for re-emitting, there could be a listener on the original
+                // source object so the test doesn't really work. We *could* try to replicate the
+                // same logic and throw if there is no listener on either the source or the target,
+                // but this behaviour is fairly undesireable for us anyway: the main place we throw
+                // 'error' events is for calls, where error events are usually emitted some time
+                // later by a different part of the code where 'emit' throwing because the app hasn't
+                // added an error handler isn't terribly helpful. (A better fix in retrospect may
+                // have been to just avoid using the event name 'error', but backwards compat...)
+                if (eventName === 'error' && this.target.listenerCount('error') === 0) return;
+                this.target.emit(eventName, ...args, source);
+            };
+            source.on(eventName, forSource);
+        }
+    }
+}

src/autodiscovery.js

@@ -17,8 +17,8 @@ limitations under the License.
 
 /** @module auto-discovery */
 
-import {logger} from './logger';
-import {URL as NodeURL} from "url";
+import { logger } from './logger';
+import { URL as NodeURL } from "url";
 
 // Dev note: Auto discovery is part of the spec.
 // See: https://matrix.org/docs/spec/client_server/r0.4.0.html#server-discovery
@@ -502,19 +502,21 @@ export class AutoDiscovery {
             request(
                 { method: "GET", uri: url, timeout: 5000 },
                 (err, response, body) => {
-                    if (err || response.statusCode < 200 || response.statusCode >= 300) {
+                    if (err || response &&
+                        (response.statusCode < 200 || response.statusCode >= 300)
+                    ) {
                         let action = "FAIL_PROMPT";
                         let reason = (err ? err.message : null) || "General failure";
-                        if (response.statusCode === 404) {
+                        if (response && response.statusCode === 404) {
                             action = "IGNORE";
                             reason = AutoDiscovery.ERROR_MISSING_WELLKNOWN;
                         }
-                        resolve({raw: {}, action: action, reason: reason, error: err});
+                        resolve({ raw: {}, action: action, reason: reason, error: err });
                         return;
                     }
 
                     try {
-                        resolve({raw: JSON.parse(body), action: "SUCCESS"});
+                        resolve({ raw: JSON.parse(body), action: "SUCCESS" });
                     } catch (e) {
                         let reason = AutoDiscovery.ERROR_INVALID;
                         if (e.name === "SyntaxError") {

src/browser-index.js

@@ -34,7 +34,7 @@ matrixcs.request(function(opts, fn) {
 let indexedDB;
 try {
     indexedDB = global.indexedDB;
-} catch(e) {}
+} catch (e) {}
 
 // if our browser (appears to) support indexeddb, use an indexeddb crypto store.
 if (indexedDB) {

src/content-repo.js

@@ -59,7 +59,7 @@ export function getHttpUriForMxc(baseUrl, mxc, width, height,
     if (resizeMethod) {
         params.method = resizeMethod;
     }
-    if (utils.keys(params).length > 0) {
+    if (Object.keys(params).length > 0) {
         // these are thumbnailing params so they probably want the
         // thumbnailing API...
         prefix = "/_matrix/media/r0/thumbnail/";
@@ -72,38 +72,6 @@ export function getHttpUriForMxc(baseUrl, mxc, width, height,
         serverAndMediaId = serverAndMediaId.substr(0, fragmentOffset);
     }
     return baseUrl + prefix + serverAndMediaId +
-        (utils.keys(params).length === 0 ? "" :
+        (Object.keys(params).length === 0 ? "" :
         ("?" + utils.encodeParams(params))) + fragment;
 }
-
-/**
- * Get an identicon URL from an arbitrary string.
- * @param {string} baseUrl The base homeserver url which has a content repo.
- * @param {string} identiconString The string to create an identicon for.
- * @param {Number} width The desired width of the image in pixels. Default: 96.
- * @param {Number} height The desired height of the image in pixels. Default: 96.
- * @return {string} The complete URL to the identicon.
- * @deprecated This is no longer in the specification.
- */
-export function getIdenticonUri(baseUrl, identiconString, width, height) {
-    if (!identiconString) {
-        return null;
-    }
-    if (!width) {
-        width = 96;
-    }
-    if (!height) {
-        height = 96;
-    }
-    const params = {
-        width: width,
-        height: height,
-    };
-
-    const path = utils.encodeUri("/_matrix/media/unstable/identicon/$ident", {
-        $ident: identiconString,
-    });
-    return baseUrl + path +
-        (utils.keys(params).length === 0 ? "" :
-            ("?" + utils.encodeParams(params)));
-}

src/crypto/CrossSigning.js

@@ -20,10 +20,13 @@ limitations under the License.
  * @module crypto/CrossSigning
  */
 
-import {decodeBase64, encodeBase64, pkSign, pkVerify} from './olmlib';
-import {EventEmitter} from 'events';
-import {logger} from '../logger';
-import {IndexedDBCryptoStore} from '../crypto/store/indexeddb-crypto-store';
+import { decodeBase64, encodeBase64, pkSign, pkVerify } from './olmlib';
+import { EventEmitter } from 'events';
+import { logger } from '../logger';
+import { IndexedDBCryptoStore } from '../crypto/store/indexeddb-crypto-store';
+import { decryptAES, encryptAES } from './aes';
+
+const KEY_REQUEST_TIMEOUT_MS = 1000 * 60;
 
 function publicKeyFromKeyInfo(keyInfo) {
     // `keys` is an object with { [`ed25519:${pubKey}`]: pubKey }
@@ -55,17 +58,42 @@ export class CrossSigningInfo extends EventEmitter {
         this._cacheCallbacks = cacheCallbacks || {};
         this.keys = {};
         this.firstUse = true;
+        // This tracks whether we've ever verified this user with any identity.
+        // When you verify a user, any devices online at the time that receive
+        // the verifying signature via the homeserver will latch this to true
+        // and can use it in the future to detect cases where the user has
+        // become unverifed later for any reason.
+        this.crossSigningVerifiedBefore = false;
+    }
+
+    static fromStorage(obj, userId) {
+        const res = new CrossSigningInfo(userId);
+        for (const prop in obj) {
+            if (obj.hasOwnProperty(prop)) {
+                res[prop] = obj[prop];
+            }
+        }
+        return res;
+    }
+
+    toStorage() {
+        return {
+            keys: this.keys,
+            firstUse: this.firstUse,
+            crossSigningVerifiedBefore: this.crossSigningVerifiedBefore,
+        };
     }
 
     /**
      * Calls the app callback to ask for a private key
+     *
      * @param {string} type The key type ("master", "self_signing", or "user_signing")
      * @param {string} expectedPubkey The matching public key or undefined to use
      *     the stored public key for the given key type.
      * @returns {Array} An array with [ public key, Olm.PkSigning ]
      */
     async getCrossSigningKey(type, expectedPubkey) {
-        const shouldCache = ["self_signing", "user_signing"].indexOf(type) >= 0;
+        const shouldCache = ["master", "self_signing", "user_signing"].indexOf(type) >= 0;
 
         if (!this._callbacks.getCrossSigningKey) {
             throw new Error("No getCrossSigningKey callback supplied");
@@ -118,37 +146,34 @@ export class CrossSigningInfo extends EventEmitter {
         );
     }
 
-    static fromStorage(obj, userId) {
-        const res = new CrossSigningInfo(userId);
-        for (const prop in obj) {
-            if (obj.hasOwnProperty(prop)) {
-                res[prop] = obj[prop];
-            }
-        }
-        return res;
-    }
-
-    toStorage() {
-        return {
-            keys: this.keys,
-            firstUse: this.firstUse,
-        };
-    }
-
     /**
      * Check whether the private keys exist in secret storage.
      * XXX: This could be static, be we often seem to have an instance when we
      * want to know this anyway...
      *
      * @param {SecretStorage} secretStorage The secret store using account data
-     * @returns {boolean} Whether all private keys were found in storage
+     * @returns {object} map of key name to key info the secret is encrypted
+     *     with, or null if it is not present or not encrypted with a trusted
+     *     key
      */
     async isStoredInSecretStorage(secretStorage) {
-        let stored = true;
-        for (const type of ["master", "self_signing", "user_signing"]) {
-            stored &= await secretStorage.isStored(`m.cross_signing.${type}`, false);
+        // check what SSSS keys have encrypted the master key (if any)
+        const stored =
+              await secretStorage.isStored("m.cross_signing.master", false) || {};
+        // then check which of those SSSS keys have also encrypted the SSK and USK
+        function intersect(s) {
+            for (const k of Object.keys(stored)) {
+                if (!s[k]) {
+                    delete stored[k];
+                }
+            }
         }
-        return stored;
+        for (const type of ["self_signing", "user_signing"]) {
+            intersect(
+                await secretStorage.isStored(`m.cross_signing.${type}`, false) || {},
+            );
+        }
+        return Object.keys(stored).length ? stored : null;
     }
 
     /**
@@ -156,12 +181,12 @@ export class CrossSigningInfo extends EventEmitter {
      * typically called in conjunction with the creation of new cross-signing
      * keys.
      *
-     * @param {object} keys The keys to store
+     * @param {Map} keys The keys to store
      * @param {SecretStorage} secretStorage The secret store using account data
      */
     static async storeInSecretStorage(keys, secretStorage) {
-        for (const type of Object.keys(keys)) {
-            const encodedKey = encodeBase64(keys[type]);
+        for (const [type, privateKey] of keys) {
+            const encodedKey = encodeBase64(privateKey);
             await secretStorage.store(`m.cross_signing.${type}`, encodedKey);
         }
     }
@@ -177,9 +202,50 @@ export class CrossSigningInfo extends EventEmitter {
      */
     static async getFromSecretStorage(type, secretStorage) {
         const encodedKey = await secretStorage.get(`m.cross_signing.${type}`);
+        if (!encodedKey) {
+            return null;
+        }
         return decodeBase64(encodedKey);
     }
 
+    /**
+     * Check whether the private keys exist in the local key cache.
+     *
+     * @param {string} [type] The type of key to get. One of "master",
+     * "self_signing", or "user_signing". Optional, will check all by default.
+     * @returns {boolean} True if all keys are stored in the local cache.
+     */
+    async isStoredInKeyCache(type) {
+        const cacheCallbacks = this._cacheCallbacks;
+        if (!cacheCallbacks) return false;
+        const types = type ? [type] : ["master", "self_signing", "user_signing"];
+        for (const t of types) {
+            if (!await cacheCallbacks.getCrossSigningKeyCache(t)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Get cross-signing private keys from the local cache.
+     *
+     * @returns {Map} A map from key type (string) to private key (Uint8Array)
+     */
+    async getCrossSigningKeysFromCache() {
+        const keys = new Map();
+        const cacheCallbacks = this._cacheCallbacks;
+        if (!cacheCallbacks) return keys;
+        for (const type of ["master", "self_signing", "user_signing"]) {
+            const privKey = await cacheCallbacks.getCrossSigningKeyCache(type);
+            if (!privKey) {
+                continue;
+            }
+            keys.set(type, privKey);
+        }
+        return keys;
+    }
+
     /**
      * Get the ID used to identify the user. This can also be used to test for
      * the existence of a given key type.
@@ -289,6 +355,13 @@ export class CrossSigningInfo extends EventEmitter {
         }
     }
 
+    /**
+     * unsets the keys, used when another session has reset the keys, to disable cross-signing
+     */
+    clearKeys() {
+        this.keys = {};
+    }
+
     setKeys(keys) {
         const signingKeys = {};
         if (keys.master) {
@@ -361,6 +434,14 @@ export class CrossSigningInfo extends EventEmitter {
         }
     }
 
+    updateCrossSigningVerifiedBefore(isCrossSigningVerified) {
+        // It is critical that this value latches forward from false to true but
+        // never back to false to avoid a downgrade attack.
+        if (!this.crossSigningVerifiedBefore && isCrossSigningVerified) {
+            this.crossSigningVerifiedBefore = true;
+        }
+    }
+
     async signObject(data, type) {
         if (!this.keys[type]) {
             throw new Error(
@@ -419,13 +500,13 @@ export class CrossSigningInfo extends EventEmitter {
             && this.getId("self_signing")
             && this.getId("self_signing") === userCrossSigning.getId("self_signing")
         ) {
-            return new UserTrustLevel(true, this.firstUse);
+            return new UserTrustLevel(true, true, this.firstUse);
         }
 
         if (!this.keys.user_signing) {
             // If there's no user signing key, they can't possibly be verified.
             // They may be TOFU trusted though.
-            return new UserTrustLevel(false, userCrossSigning.firstUse);
+            return new UserTrustLevel(false, false, userCrossSigning.firstUse);
         }
 
         let userTrusted;
@@ -437,7 +518,11 @@ export class CrossSigningInfo extends EventEmitter {
         } catch (e) {
             userTrusted = false;
         }
-        return new UserTrustLevel(userTrusted, userCrossSigning.firstUse);
+        return new UserTrustLevel(
+            userTrusted,
+            userCrossSigning.crossSigningVerifiedBefore,
+            userCrossSigning.firstUse,
+        );
     }
 
     /**
@@ -446,17 +531,20 @@ export class CrossSigningInfo extends EventEmitter {
      * @param {CrossSigningInfo} userCrossSigning Cross signing info for user
      * @param {module:crypto/deviceinfo} device The device to check
      * @param {bool} localTrust Whether the device is trusted locally
+     * @param {bool} trustCrossSignedDevices Whether we trust cross signed devices
      *
      * @returns {DeviceTrustLevel}
      */
-    checkDeviceTrust(userCrossSigning, device, localTrust) {
+    checkDeviceTrust(userCrossSigning, device, localTrust, trustCrossSignedDevices) {
         const userTrust = this.checkUserTrust(userCrossSigning);
 
         const userSSK = userCrossSigning.keys.self_signing;
         if (!userSSK) {
             // if the user has no self-signing key then we cannot make any
             // trust assertions about this device from cross-signing
-            return new DeviceTrustLevel(false, false, localTrust);
+            return new DeviceTrustLevel(
+                false, false, localTrust, trustCrossSignedDevices,
+            );
         }
 
         const deviceObj = deviceToObject(device, userCrossSigning.userId);
@@ -468,9 +556,13 @@ export class CrossSigningInfo extends EventEmitter {
                 deviceObj, publicKeyFromKeyInfo(userSSK), userCrossSigning.userId,
             );
             // ...then we trust this device as much as far as we trust the user
-            return DeviceTrustLevel.fromUserTrustLevel(userTrust, localTrust);
+            return DeviceTrustLevel.fromUserTrustLevel(
+                userTrust, localTrust, trustCrossSignedDevices,
+            );
         } catch (e) {
-            return new DeviceTrustLevel(false, false, localTrust);
+            return new DeviceTrustLevel(
+                false, false, localTrust, trustCrossSignedDevices,
+            );
         }
     }
 
@@ -502,8 +594,9 @@ export const CrossSigningLevel = {
  * Represents the ways in which we trust a user
  */
 export class UserTrustLevel {
-    constructor(crossSigningVerified, tofu) {
+    constructor(crossSigningVerified, crossSigningVerifiedBefore, tofu) {
         this._crossSigningVerified = crossSigningVerified;
+        this._crossSigningVerifiedBefore = crossSigningVerifiedBefore;
         this._tofu = tofu;
     }
 
@@ -521,6 +614,14 @@ export class UserTrustLevel {
         return this._crossSigningVerified;
     }
 
+    /**
+     * @returns {bool} true if we ever verified this user before (at least for
+     * the history of verifications observed by this device).
+     */
+    wasCrossSigningVerified() {
+        return this._crossSigningVerifiedBefore;
+    }
+
     /**
      * @returns {bool} true if this user's key is trusted on first use
      */
@@ -533,17 +634,19 @@ export class UserTrustLevel {
  * Represents the ways in which we trust a device
  */
 export class DeviceTrustLevel {
-    constructor(crossSigningVerified, tofu, localVerified) {
+    constructor(crossSigningVerified, tofu, localVerified, trustCrossSignedDevices) {
         this._crossSigningVerified = crossSigningVerified;
         this._tofu = tofu;
         this._localVerified = localVerified;
+        this._trustCrossSignedDevices = trustCrossSignedDevices;
     }
 
-    static fromUserTrustLevel(userTrustLevel, localVerified) {
+    static fromUserTrustLevel(userTrustLevel, localVerified, trustCrossSignedDevices) {
         return new DeviceTrustLevel(
             userTrustLevel._crossSigningVerified,
             userTrustLevel._tofu,
             localVerified,
+            trustCrossSignedDevices,
         );
     }
 
@@ -551,7 +654,9 @@ export class DeviceTrustLevel {
      * @returns {bool} true if this device is verified via any means
      */
     isVerified() {
-        return this.isCrossSigningVerified() || this.isLocallyVerified();
+        return Boolean(this.isLocallyVerified() || (
+            this._trustCrossSignedDevices && this.isCrossSigningVerified()
+        ));
     }
 
     /**
@@ -577,27 +682,135 @@ export class DeviceTrustLevel {
     }
 }
 
-export function createCryptoStoreCacheCallbacks(store) {
+export function createCryptoStoreCacheCallbacks(store, olmdevice) {
     return {
-        getCrossSigningKeyCache: function(type, _expectedPublicKey) {
-            return new Promise((resolve) => {
+        getCrossSigningKeyCache: async function(type, _expectedPublicKey) {
+            const key = await new Promise((resolve) => {
                 return store.doTxn(
                     'readonly',
                     [IndexedDBCryptoStore.STORE_ACCOUNT],
                     (txn) => {
-                        store.getCrossSigningPrivateKey(txn, resolve, type);
+                        store.getSecretStorePrivateKey(txn, resolve, type);
                     },
                 );
             });
+
+            if (key && key.ciphertext) {
+                const pickleKey = Buffer.from(olmdevice._pickleKey);
+                const decrypted = await decryptAES(key, pickleKey, type);
+                return decodeBase64(decrypted);
+            } else {
+                return key;
+            }
         },
-        storeCrossSigningKeyCache: function(type, key) {
+        storeCrossSigningKeyCache: async function(type, key) {
+            if (!(key instanceof Uint8Array)) {
+                throw new Error(
+                    `storeCrossSigningKeyCache expects Uint8Array, got ${key}`,
+                );
+            }
+            const pickleKey = Buffer.from(olmdevice._pickleKey);
+            key = await encryptAES(encodeBase64(key), pickleKey, type);
             return store.doTxn(
                 'readwrite',
                 [IndexedDBCryptoStore.STORE_ACCOUNT],
                 (txn) => {
-                    store.storeCrossSigningPrivateKey(txn, type, key);
+                    store.storeSecretStorePrivateKey(txn, type, key);
                 },
             );
         },
     };
 }
+
+/**
+ * Request cross-signing keys from another device during verification.
+ *
+ * @param {MatrixClient} baseApis base Matrix API interface
+ * @param {string} userId The user ID being verified
+ * @param {string} deviceId The device ID being verified
+ */
+export async function requestKeysDuringVerification(baseApis, userId, deviceId) {
+    // If this is a self-verification, ask the other party for keys
+    if (baseApis.getUserId() !== userId) {
+        return;
+    }
+    logger.log("Cross-signing: Self-verification done; requesting keys");
+    // This happens asynchronously, and we're not concerned about waiting for
+    // it.  We return here in order to test.
+    return new Promise((resolve, reject) => {
+        const client = baseApis;
+        const original = client.crypto._crossSigningInfo;
+
+        // We already have all of the infrastructure we need to validate and
+        // cache cross-signing keys, so instead of replicating that, here we set
+        // up callbacks that request them from the other device and call
+        // CrossSigningInfo.getCrossSigningKey() to validate/cache
+        const crossSigning = new CrossSigningInfo(
+            original.userId,
+            { getCrossSigningKey: async (type) => {
+                logger.debug("Cross-signing: requesting secret",
+                                type, deviceId);
+                const { promise } = client.requestSecret(
+                    `m.cross_signing.${type}`, [deviceId],
+                );
+                const result = await promise;
+                const decoded = decodeBase64(result);
+                return Uint8Array.from(decoded);
+            } },
+            original._cacheCallbacks,
+        );
+        crossSigning.keys = original.keys;
+
+        // XXX: get all keys out if we get one key out
+        // https://github.com/vector-im/element-web/issues/12604
+        // then change here to reject on the timeout
+        // Requests can be ignored, so don't wait around forever
+        const timeout = new Promise((resolve, reject) => {
+            setTimeout(
+                resolve,
+                KEY_REQUEST_TIMEOUT_MS,
+                new Error("Timeout"),
+            );
+        });
+
+        // also request and cache the key backup key
+        const backupKeyPromise = new Promise(async resolve => {
+            const cachedKey = await client.crypto.getSessionBackupPrivateKey();
+            if (!cachedKey) {
+                logger.info("No cached backup key found. Requesting...");
+                const secretReq = client.requestSecret(
+                    'm.megolm_backup.v1', [deviceId],
+                );
+                const base64Key = await secretReq.promise;
+                logger.info("Got key backup key, decoding...");
+                const decodedKey = decodeBase64(base64Key);
+                logger.info("Decoded backup key, storing...");
+                client.crypto.storeSessionBackupPrivateKey(
+                    Uint8Array.from(decodedKey),
+                );
+                logger.info("Backup key stored. Starting backup restore...");
+                const backupInfo = await client.getKeyBackupVersion();
+                // no need to await for this - just let it go in the bg
+                client.restoreKeyBackupWithCache(
+                    undefined, undefined, backupInfo,
+                ).then(() => {
+                    logger.info("Backup restored.");
+                });
+            }
+            resolve();
+        });
+
+        // We call getCrossSigningKey() for its side-effects
+        return Promise.race([
+            Promise.all([
+                crossSigning.getCrossSigningKey("master"),
+                crossSigning.getCrossSigningKey("self_signing"),
+                crossSigning.getCrossSigningKey("user_signing"),
+                backupKeyPromise,
+            ]),
+            timeout,
+        ]).then(resolve, reject);
+    }).catch((e) => {
+        logger.warn("Cross-signing: failure while requesting keys:", e);
+    });
+}

src/crypto/DeviceList.js

@@ -22,14 +22,13 @@ limitations under the License.
  * Manages the list of other users' devices
  */
 
-import {EventEmitter} from 'events';
-import {logger} from '../logger';
-import {DeviceInfo} from './deviceinfo';
-import {CrossSigningInfo} from './CrossSigning';
+import { EventEmitter } from 'events';
+import { logger } from '../logger';
+import { DeviceInfo } from './deviceinfo';
+import { CrossSigningInfo } from './CrossSigning';
 import * as olmlib from './olmlib';
-import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
-import {defer, sleep} from '../utils';
-
+import { IndexedDBCryptoStore } from './store/indexeddb-crypto-store';
+import { chunkPromises, defer, sleep } from '../utils';
 
 /* State transition diagram for DeviceList._deviceTrackingStatus
  *
@@ -51,7 +50,6 @@ import {defer, sleep} from '../utils';
  *   +----------------------- UP_TO_DATE ------------------------+
  */
 
-
 // constants for DeviceList._deviceTrackingStatus
 const TRACKING_STATUS_NOT_TRACKED = 0;
 const TRACKING_STATUS_PENDING_DOWNLOAD = 1;
@@ -62,7 +60,7 @@ const TRACKING_STATUS_UP_TO_DATE = 3;
  * @alias module:crypto/DeviceList
  */
 export class DeviceList extends EventEmitter {
-    constructor(baseApis, cryptoStore, olmDevice) {
+    constructor(baseApis, cryptoStore, olmDevice, keyDownloadChunkSize = 250) {
         super();
 
         this._cryptoStore = cryptoStore;
@@ -98,6 +96,9 @@ export class DeviceList extends EventEmitter {
         // userId -> promise
         this._keyDownloadsInProgressByUser = {};
 
+        // Maximum number of user IDs per request to prevent server overload (#1619)
+        this._keyDownloadChunkSize = keyDownloadChunkSize;
+
         // Set whenever changes are made other than setting the sync token
         this._dirty = false;
 
@@ -109,6 +110,9 @@ export class DeviceList extends EventEmitter {
         this._savePromiseTime = null;
         // The timer used to delay the save
         this._saveTimer = null;
+        // True if we have fetched data from the server or loaded a non-empty
+        // set of device data from the store
+        this._hasFetched = null;
     }
 
     /**
@@ -118,6 +122,7 @@ export class DeviceList extends EventEmitter {
         await this._cryptoStore.doTxn(
             'readonly', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
                 this._cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
+                    this._hasFetched = Boolean(deviceData && deviceData.devices);
                     this._devices = deviceData ? deviceData.devices : {},
                     this._crossSigningInfo = deviceData ?
                         deviceData.crossSigningInfo || {} : {};
@@ -317,6 +322,15 @@ export class DeviceList extends EventEmitter {
         return stored;
     }
 
+    /**
+     * Returns a list of all user IDs the DeviceList knows about
+     *
+     * @return {array} All known user IDs
+     */
+    getKnownUserIds() {
+        return Object.keys(this._devices);
+    }
+
     /**
      * Get the stored device keys for a user id
      *
@@ -643,6 +657,7 @@ export class DeviceList extends EventEmitter {
         });
 
         const finished = (success) => {
+            this.emit("crypto.willUpdateDevices", users, !this._hasFetched);
             users.forEach((u) => {
                 this._dirty = true;
 
@@ -668,7 +683,8 @@ export class DeviceList extends EventEmitter {
                 }
             });
             this.saveIfDirty();
-            this.emit("crypto.devicesUpdated", users);
+            this.emit("crypto.devicesUpdated", users, !this._hasFetched);
+            this._hasFetched = true;
         };
 
         return prom;
@@ -765,33 +781,40 @@ class DeviceListUpdateSerialiser {
             opts.token = this._syncToken;
         }
 
-        this._baseApis.downloadKeysForUsers(
-            downloadUsers, opts,
-        ).then((res) => {
-            const dk = res.device_keys || {};
-            const masterKeys = res.master_keys || {};
-            const ssks = res.self_signing_keys || {};
-            const usks = res.user_signing_keys || {};
+        const factories = [];
+        for (let i = 0; i < downloadUsers.length; i += this._deviceList._keyDownloadChunkSize) {
+            const userSlice = downloadUsers.slice(i, i + this._deviceList._keyDownloadChunkSize);
+            factories.push(() => this._baseApis.downloadKeysForUsers(userSlice, opts));
+        }
 
-            // do each user in a separate promise, to avoid wedging the CPU
-            // (https://github.com/vector-im/riot-web/issues/3158)
+        chunkPromises(factories, 3).then(async (responses) => {
+            const dk = Object.assign({}, ...(responses.map(res => res.device_keys || {})));
+            const masterKeys = Object.assign({}, ...(responses.map(res => res.master_keys || {})));
+            const ssks = Object.assign({}, ...(responses.map(res => res.self_signing_keys || {})));
+            const usks = Object.assign({}, ...(responses.map(res => res.user_signing_keys || {})));
+
+            // yield to other things that want to execute in between users, to
+            // avoid wedging the CPU
+            // (https://github.com/vector-im/element-web/issues/3158)
             //
             // of course we ought to do this in a web worker or similar, but
             // this serves as an easy solution for now.
-            let prom = Promise.resolve();
             for (const userId of downloadUsers) {
-                prom = prom.then(sleep(5)).then(() => {
-                    return this._processQueryResponseForUser(
+                await sleep(5);
+                try {
+                    await this._processQueryResponseForUser(
                         userId, dk[userId], {
                             master: masterKeys[userId],
                             self_signing: ssks[userId],
                             user_signing: usks[userId],
                         },
                     );
-                });
+                } catch (e) {
+                    // log the error but continue, so that one bad key
+                    // doesn't kill the whole process
+                    logger.error(`Error processing keys for ${userId}:`, e);
+                }
             }
-
-            return prom;
         }).then(() => {
             logger.log('Completed key download for ' + downloadUsers);
 
@@ -812,7 +835,7 @@ class DeviceListUpdateSerialiser {
     }
 
     async _processQueryResponseForUser(
-        userId, dkResponse, crossSigningResponse, sskResponse,
+        userId, dkResponse, crossSigningResponse,
     ) {
         logger.log('got device keys for ' + userId + ':', dkResponse);
         logger.log('got cross-signing keys for ' + userId + ':', crossSigningResponse);
@@ -830,6 +853,7 @@ class DeviceListUpdateSerialiser {
 
             await _updateStoredDeviceKeysForUser(
                 this._olmDevice, userId, userStore, dkResponse || {},
+                this._baseApis.getUserId(), this._baseApis.deviceId,
             );
 
             // put the updates into the object that will be returned as our results
@@ -866,9 +890,9 @@ class DeviceListUpdateSerialiser {
     }
 }
 
-
-async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
-        userResult) {
+async function _updateStoredDeviceKeysForUser(
+    _olmDevice, userId, userStore, userResult, localUserId, localDeviceId,
+) {
     let updated = false;
 
     // remove any devices in the store which aren't in the response
@@ -878,6 +902,13 @@ async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
         }
 
         if (!(deviceId in userResult)) {
+            if (userId === localUserId && deviceId === localDeviceId) {
+                logger.warn(
+                    `Local device ${deviceId} missing from sync, skipping removal`,
+                );
+                continue;
+            }
+
             logger.log("Device " + userId + ":" + deviceId +
                 " has been removed");
             delete userStore[deviceId];

src/crypto/EncryptionSetup.js

@@ -0,0 +1,355 @@
+import { logger } from "../logger";
+import { MatrixEvent } from "../models/event";
+import { EventEmitter } from "events";
+import { createCryptoStoreCacheCallbacks } from "./CrossSigning";
+import { IndexedDBCryptoStore } from './store/indexeddb-crypto-store';
+import {
+    PREFIX_UNSTABLE,
+} from "../http-api";
+
+/**
+ * Builds an EncryptionSetupOperation by calling any of the add.. methods.
+ * Once done, `buildOperation()` can be called which allows to apply to operation.
+ *
+ * This is used as a helper by Crypto to keep track of all the network requests
+ * and other side-effects of bootstrapping, so it can be applied in one go (and retried in the future)
+ * Also keeps track of all the private keys created during bootstrapping, so we don't need to prompt for them
+ * more than once.
+ */
+export class EncryptionSetupBuilder {
+    /**
+     * @param {Object.<String, MatrixEvent>} accountData pre-existing account data, will only be read, not written.
+     * @param {CryptoCallbacks} delegateCryptoCallbacks crypto callbacks to delegate to if the key isn't in cache yet
+     */
+    constructor(accountData, delegateCryptoCallbacks) {
+        this.accountDataClientAdapter = new AccountDataClientAdapter(accountData);
+        this.crossSigningCallbacks = new CrossSigningCallbacks();
+        this.ssssCryptoCallbacks = new SSSSCryptoCallbacks(delegateCryptoCallbacks);
+
+        this._crossSigningKeys = null;
+        this._keySignatures = null;
+        this._keyBackupInfo = null;
+    }
+
+    /**
+     * Adds new cross-signing public keys
+     *
+     * @param {function} authUpload Function called to await an interactive auth
+     * flow when uploading device signing keys.
+     * Args:
+     *     {function} A function that makes the request requiring auth. Receives
+     *     the auth data as an object. Can be called multiple times, first with
+     *     an empty authDict, to obtain the flows.
+     * @param {Object} keys the new keys
+     */
+    addCrossSigningKeys(authUpload, keys) {
+        this._crossSigningKeys = { authUpload, keys };
+    }
+
+    /**
+     * Adds the key backup info to be updated on the server
+     *
+     * Used either to create a new key backup, or add signatures
+     * from the new MSK.
+     *
+     * @param {Object} keyBackupInfo as received from/sent to the server
+     */
+    addSessionBackup(keyBackupInfo) {
+        this._keyBackupInfo = keyBackupInfo;
+    }
+
+    /**
+     * Adds the session backup private key to be updated in the local cache
+     *
+     * Used after fixing the format of the key
+     *
+     * @param {Uint8Array} privateKey
+     */
+    addSessionBackupPrivateKeyToCache(privateKey) {
+        this._sessionBackupPrivateKey = privateKey;
+    }
+
+    /**
+     * Add signatures from a given user and device/x-sign key
+     * Used to sign the new cross-signing key with the device key
+     *
+     * @param {String} userId
+     * @param {String} deviceId
+     * @param {String} signature
+     */
+    addKeySignature(userId, deviceId, signature) {
+        if (!this._keySignatures) {
+            this._keySignatures = {};
+        }
+        const userSignatures = this._keySignatures[userId] || {};
+        this._keySignatures[userId] = userSignatures;
+        userSignatures[deviceId] = signature;
+    }
+
+    /**
+     * @param {String} type
+     * @param {Object} content
+     * @return {Promise}
+     */
+    setAccountData(type, content) {
+        return this.accountDataClientAdapter.setAccountData(type, content);
+    }
+
+    /**
+     * builds the operation containing all the parts that have been added to the builder
+     * @return {EncryptionSetupOperation}
+     */
+    buildOperation() {
+        const accountData = this.accountDataClientAdapter._values;
+        return new EncryptionSetupOperation(
+            accountData,
+            this._crossSigningKeys,
+            this._keyBackupInfo,
+            this._keySignatures,
+        );
+    }
+
+    /**
+     * Stores the created keys locally.
+     *
+     * This does not yet store the operation in a way that it can be restored,
+     * but that is the idea in the future.
+     *
+     * @param  {Crypto} crypto
+     * @return {Promise}
+     */
+    async persist(crypto) {
+        // store private keys in cache
+        if (this._crossSigningKeys) {
+            const cacheCallbacks = createCryptoStoreCacheCallbacks(
+                crypto._cryptoStore, crypto._olmDevice);
+            for (const type of ["master", "self_signing", "user_signing"]) {
+                logger.log(`Cache ${type} cross-signing private key locally`);
+                const privateKey = this.crossSigningCallbacks.privateKeys.get(type);
+                await cacheCallbacks.storeCrossSigningKeyCache(type, privateKey);
+            }
+            // store own cross-sign pubkeys as trusted
+            await crypto._cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    crypto._cryptoStore.storeCrossSigningKeys(
+                        txn, this._crossSigningKeys.keys);
+                },
+            );
+        }
+        // store session backup key in cache
+        if (this._sessionBackupPrivateKey) {
+            await crypto.storeSessionBackupPrivateKey(this._sessionBackupPrivateKey);
+        }
+    }
+}
+
+/**
+ * Can be created from EncryptionSetupBuilder, or
+ * (in a follow-up PR, not implemented yet) restored from storage, to retry.
+ *
+ * It does not have knowledge of any private keys, unlike the builder.
+ */
+export class EncryptionSetupOperation {
+    /**
+     * @param  {Map<String, Object>} accountData
+     * @param  {Object} crossSigningKeys
+     * @param  {Object} keyBackupInfo
+     * @param  {Object} keySignatures
+     */
+    constructor(accountData, crossSigningKeys, keyBackupInfo, keySignatures) {
+        this._accountData = accountData;
+        this._crossSigningKeys = crossSigningKeys;
+        this._keyBackupInfo = keyBackupInfo;
+        this._keySignatures = keySignatures;
+    }
+
+    /**
+     * Runs the (remaining part of, in the future) operation by sending requests to the server.
+     * @param  {Crypto} crypto
+     */
+    async apply(crypto) {
+        const baseApis = crypto._baseApis;
+        // upload cross-signing keys
+        if (this._crossSigningKeys) {
+            const keys = {};
+            for (const [name, key] of Object.entries(this._crossSigningKeys.keys)) {
+                keys[name + "_key"] = key;
+            }
+
+            // We must only call `uploadDeviceSigningKeys` from inside this auth
+            // helper to ensure we properly handle auth errors.
+            await this._crossSigningKeys.authUpload(authDict => {
+                return baseApis.uploadDeviceSigningKeys(authDict, keys);
+            });
+
+            // pass the new keys to the main instance of our own CrossSigningInfo.
+            crypto._crossSigningInfo.setKeys(this._crossSigningKeys.keys);
+        }
+        // set account data
+        if (this._accountData) {
+            for (const [type, content] of this._accountData) {
+                await baseApis.setAccountData(type, content);
+            }
+        }
+        // upload first cross-signing signatures with the new key
+        // (e.g. signing our own device)
+        if (this._keySignatures) {
+            await baseApis.uploadKeySignatures(this._keySignatures);
+        }
+        // need to create/update key backup info
+        if (this._keyBackupInfo) {
+            if (this._keyBackupInfo.version) {
+                // session backup signature
+                // The backup is trusted because the user provided the private key.
+                // Sign the backup with the cross signing key so the key backup can
+                // be trusted via cross-signing.
+                await baseApis.http.authedRequest(
+                    undefined, "PUT", "/room_keys/version/" + this._keyBackupInfo.version,
+                    undefined, {
+                        algorithm: this._keyBackupInfo.algorithm,
+                        auth_data: this._keyBackupInfo.auth_data,
+                    },
+                    { prefix: PREFIX_UNSTABLE },
+                );
+            } else {
+                // add new key backup
+                await baseApis.http.authedRequest(
+                    undefined, "POST", "/room_keys/version",
+                    undefined, this._keyBackupInfo,
+                    { prefix: PREFIX_UNSTABLE },
+                );
+            }
+        }
+    }
+}
+
+/**
+ * Catches account data set by SecretStorage during bootstrapping by
+ * implementing the methods related to account data in MatrixClient
+ */
+class AccountDataClientAdapter extends EventEmitter {
+    /**
+     * @param  {Object.<String, MatrixEvent>} accountData existing account data
+     */
+    constructor(accountData) {
+        super();
+        this._existingValues = accountData;
+        this._values = new Map();
+    }
+
+    /**
+     * @param  {String} type
+     * @return {Promise<Object>} the content of the account data
+     */
+    getAccountDataFromServer(type) {
+        return Promise.resolve(this.getAccountData(type));
+    }
+
+    /**
+     * @param  {String} type
+     * @return {Object} the content of the account data
+     */
+    getAccountData(type) {
+        const modifiedValue = this._values.get(type);
+        if (modifiedValue) {
+            return modifiedValue;
+        }
+        const existingValue = this._existingValues[type];
+        if (existingValue) {
+            return existingValue.getContent();
+        }
+        return null;
+    }
+
+    /**
+     * @param {String} type
+     * @param {Object} content
+     * @return {Promise}
+     */
+    setAccountData(type, content) {
+        const lastEvent = this._values.get(type);
+        this._values.set(type, content);
+        // ensure accountData is emitted on the next tick,
+        // as SecretStorage listens for it while calling this method
+        // and it seems to rely on this.
+        return Promise.resolve().then(() => {
+            const event = new MatrixEvent({ type, content });
+            this.emit("accountData", event, lastEvent);
+        });
+    }
+}
+
+/**
+ * Catches the private cross-signing keys set during bootstrapping
+ * by both cache callbacks (see createCryptoStoreCacheCallbacks) as non-cache callbacks.
+ * See CrossSigningInfo constructor
+ */
+class CrossSigningCallbacks {
+    constructor() {
+        this.privateKeys = new Map();
+    }
+
+    // cache callbacks
+    getCrossSigningKeyCache(type, expectedPublicKey) {
+        return this.getCrossSigningKey(type, expectedPublicKey);
+    }
+
+    storeCrossSigningKeyCache(type, key) {
+        this.privateKeys.set(type, key);
+        return Promise.resolve();
+    }
+
+    // non-cache callbacks
+    getCrossSigningKey(type, _expectedPubkey) {
+        return Promise.resolve(this.privateKeys.get(type));
+    }
+
+    saveCrossSigningKeys(privateKeys) {
+        for (const [type, privateKey] of Object.entries(privateKeys)) {
+            this.privateKeys.set(type, privateKey);
+        }
+    }
+}
+
+/**
+ * Catches the 4S private key set during bootstrapping by implementing
+ * the SecretStorage crypto callbacks
+ */
+class SSSSCryptoCallbacks {
+    constructor(delegateCryptoCallbacks) {
+        this._privateKeys = new Map();
+        this._delegateCryptoCallbacks = delegateCryptoCallbacks;
+    }
+
+    async getSecretStorageKey({ keys }, name) {
+        for (const keyId of Object.keys(keys)) {
+            const privateKey = this._privateKeys.get(keyId);
+            if (privateKey) {
+                return [keyId, privateKey];
+            }
+        }
+        // if we don't have the key cached yet, ask
+        // for it to the general crypto callbacks and cache it
+        if (this._delegateCryptoCallbacks) {
+            const result = await this._delegateCryptoCallbacks.
+                getSecretStorageKey({ keys }, name);
+            if (result) {
+                const [keyId, privateKey] = result;
+                this._privateKeys.set(keyId, privateKey);
+            }
+            return result;
+        }
+    }
+
+    addPrivateKey(keyId, keyInfo, privKey) {
+        this._privateKeys.set(keyId, privKey);
+        // Also pass along to application to cache if it wishes
+        if (
+            this._delegateCryptoCallbacks &&
+            this._delegateCryptoCallbacks.cacheSecretStorageKey
+        ) {
+            this._delegateCryptoCallbacks.cacheSecretStorageKey(keyId, keyInfo, privKey);
+        }
+    }
+}

src/crypto/OlmDevice.js

@@ -16,8 +16,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {logger} from '../logger';
-import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
+import { logger } from '../logger';
+import { IndexedDBCryptoStore } from './store/indexeddb-crypto-store';
 import * as algorithms from './algorithms';
 
 // The maximum size of an event is 65K, and we base64 the content, so this is a
@@ -36,13 +36,18 @@ function checkPayloadLength(payloadString) {
         // Note that even if we manage to do the encryption, the message send may fail,
         // because by the time we've wrapped the ciphertext in the event object, it may
         // exceed 65K. But at least we won't just fail with "abort()" in that case.
-        throw new Error("Message too long (" + payloadString.length + " bytes). " +
+        const err = new Error("Message too long (" + payloadString.length + " bytes). " +
                         "The maximum for an encrypted message is " +
                         MAX_PLAINTEXT_LENGTH + " bytes.");
+        // TODO: [TypeScript] We should have our own error types
+        err.data = {
+            errcode: "M_TOO_LARGE",
+            error: "Payload too large for encrypted message",
+        };
+        throw err;
     }
 }
 
-
 /**
  * The type of object we use for importing and exporting megolm session data.
  *
@@ -56,7 +61,6 @@ function checkPayloadLength(payloadString) {
  * @property {String} session_key Base64'ed key data
  */
 
-
 /**
  * Manages the olm cryptography functions. Each OlmDevice has a single
  * OlmAccount and a number of OlmSessions.
@@ -105,6 +109,9 @@ export function OlmDevice(cryptoStore) {
     // Keep track of sessions that we're starting, so that we don't start
     // multiple sessions for the same device at the same time.
     this._sessionsInProgress = {};
+
+    // Used by olm to serialise prekey message decryptions
+    this._olmPrekeyPromise = Promise.resolve();
 }
 
 /**
@@ -135,7 +142,7 @@ OlmDevice.prototype.init = async function(opts = {}) {
     try {
         if (fromExportedDevice) {
             if (pickleKey) {
-                console.warn(
+                logger.warn(
                     'ignoring opts.pickleKey'
                     + ' because opts.fromExportedDevice is present.',
                 );
@@ -341,7 +348,7 @@ OlmDevice.prototype._unpickleSession = function(sessionInfo, func) {
     const session = new global.Olm.Session();
     try {
         session.unpickle(this._pickleKey, sessionInfo.session);
-        const unpickledSessInfo = Object.assign({}, sessionInfo, {session});
+        const unpickledSessInfo = Object.assign({}, sessionInfo, { session });
 
         func(unpickledSessInfo);
     } finally {
@@ -367,7 +374,6 @@ OlmDevice.prototype._saveSession = function(deviceKey, sessionInfo, txn) {
     );
 };
 
-
 /**
  * get an OlmUtility and call the given function
  *
@@ -384,7 +390,6 @@ OlmDevice.prototype._getUtility = function(func) {
     }
 };
 
-
 /**
  * Signs a message with the ed25519 key for this account.
  *
@@ -425,7 +430,6 @@ OlmDevice.prototype.getOneTimeKeys = async function() {
     return result;
 };
 
-
 /**
  * Get the maximum number of one-time keys we can store.
  *
@@ -468,6 +472,36 @@ OlmDevice.prototype.generateOneTimeKeys = function(numKeys) {
     );
 };
 
+/**
+ * Generate a new fallback keys
+ *
+ * @return {Promise} Resolved once the account is saved back having generated the key
+ */
+OlmDevice.prototype.generateFallbackKey = async function() {
+    await this._cryptoStore.doTxn(
+        'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                account.generate_fallback_key();
+                this._storeAccount(txn, account);
+            });
+        },
+    );
+};
+
+OlmDevice.prototype.getFallbackKey = async function() {
+    let result;
+    await this._cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                result = JSON.parse(account.fallback_key());
+            });
+        },
+    );
+    return result;
+};
+
 /**
  * Generate a new outbound session
  *
@@ -506,11 +540,11 @@ OlmDevice.prototype.createOutboundSession = async function(
                 }
             });
         },
+        logger.withPrefix("[createOutboundSession]"),
     );
     return newSessionId;
 };
 
-
 /**
  * Generate a new inbound session, given an incoming message
  *
@@ -566,12 +600,12 @@ OlmDevice.prototype.createInboundSession = async function(
                 }
             });
         },
+        logger.withPrefix("[createInboundSession]"),
     );
 
     return result;
 };
 
-
 /**
  * Get a list of known session IDs for the given device
  *
@@ -580,8 +614,10 @@ OlmDevice.prototype.createInboundSession = async function(
  * @return {Promise<string[]>}  a list of known session ids for the device
  */
 OlmDevice.prototype.getSessionIdsForDevice = async function(theirDeviceIdentityKey) {
+    const log = logger.withPrefix("[getSessionIdsForDevice]");
+
     if (this._sessionsInProgress[theirDeviceIdentityKey]) {
-        logger.log("waiting for olm session to be created");
+        log.debug(`Waiting for Olm session for ${theirDeviceIdentityKey} to be created`);
         try {
             await this._sessionsInProgress[theirDeviceIdentityKey];
         } catch (e) {
@@ -599,6 +635,7 @@ OlmDevice.prototype.getSessionIdsForDevice = async function(theirDeviceIdentityK
                 },
             );
         },
+        log,
     );
 
     return sessionIds;
@@ -612,13 +649,14 @@ OlmDevice.prototype.getSessionIdsForDevice = async function(theirDeviceIdentityK
  * @param {boolean} nowait Don't wait for an in-progress session to complete.
  *     This should only be set to true of the calling function is the function
  *     that marked the session as being in-progress.
+ * @param {Logger} [log] A possibly customised log
  * @return {Promise<?string>}  session id, or null if no established session
  */
 OlmDevice.prototype.getSessionIdForDevice = async function(
-    theirDeviceIdentityKey, nowait,
+    theirDeviceIdentityKey, nowait, log,
 ) {
     const sessionInfos = await this.getSessionInfoForDevice(
-        theirDeviceIdentityKey, nowait,
+        theirDeviceIdentityKey, nowait, log,
     );
 
     if (sessionInfos.length === 0) {
@@ -658,11 +696,16 @@ OlmDevice.prototype.getSessionIdForDevice = async function(
  * @param {boolean} nowait Don't wait for an in-progress session to complete.
  *     This should only be set to true of the calling function is the function
  *     that marked the session as being in-progress.
+ * @param {Logger} [log] A possibly customised log
  * @return {Array.<{sessionId: string, hasReceivedMessage: Boolean}>}
  */
-OlmDevice.prototype.getSessionInfoForDevice = async function(deviceIdentityKey, nowait) {
+OlmDevice.prototype.getSessionInfoForDevice = async function(
+    deviceIdentityKey, nowait, log = logger,
+) {
+    log = log.withPrefix("[getSessionInfoForDevice]");
+
     if (this._sessionsInProgress[deviceIdentityKey] && !nowait) {
-        logger.log("waiting for olm session to be created");
+        log.debug(`Waiting for Olm session for ${deviceIdentityKey} to be created`);
         try {
             await this._sessionsInProgress[deviceIdentityKey];
         } catch (e) {
@@ -688,6 +731,7 @@ OlmDevice.prototype.getSessionInfoForDevice = async function(deviceIdentityKey,
                 }
             });
         },
+        log,
     );
 
     return info;
@@ -722,6 +766,7 @@ OlmDevice.prototype.encryptMessage = async function(
                 this._saveSession(theirDeviceIdentityKey, sessionInfo, txn);
             });
         },
+        logger.withPrefix("[encryptMessage]"),
     );
     return res;
 };
@@ -755,6 +800,7 @@ OlmDevice.prototype.decryptMessage = async function(
                 this._saveSession(theirDeviceIdentityKey, sessionInfo, txn);
             });
         },
+        logger.withPrefix("[decryptMessage]"),
     );
     return payloadString;
 };
@@ -786,6 +832,7 @@ OlmDevice.prototype.matchesSession = async function(
                 matches = sessionInfo.session.matches_inbound(ciphertext);
             });
         },
+        logger.withPrefix("[matchesSession]"),
     );
     return matches;
 };
@@ -802,7 +849,6 @@ OlmDevice.prototype.filterOutNotifiedErrorDevices = async function(devices) {
     return await this._cryptoStore.filterOutNotifiedErrorDevices(devices);
 };
 
-
 // Outbound group session
 // ======================
 
@@ -817,7 +863,6 @@ OlmDevice.prototype._saveOutboundGroupSession = function(session) {
     this._outboundGroupSessionStore[session.session_id()] = pickledSession;
 };
 
-
 /**
  * extract an OutboundGroupSession from _outboundGroupSessionStore and call the
  * given function
@@ -842,7 +887,6 @@ OlmDevice.prototype._getOutboundGroupSession = function(sessionId, func) {
     }
 };
 
-
 /**
  * Generate a new outbound group session
  *
@@ -859,7 +903,6 @@ OlmDevice.prototype.createOutboundGroupSession = function() {
     }
 };
 
-
 /**
  * Encrypt an outgoing message with an outbound group session
  *
@@ -899,7 +942,6 @@ OlmDevice.prototype.getOutboundGroupSessionKey = function(sessionId) {
     });
 };
 
-
 // Inbound group session
 // =====================
 
@@ -983,16 +1025,18 @@ OlmDevice.prototype._getInboundGroupSession = function(
  * @param {Object<string, string>} keysClaimed Other keys the sender claims.
  * @param {boolean} exportFormat true if the megolm keys are in export format
  *    (ie, they lack an ed25519 signature)
+ * @param {Object} [extraSessionData={}] any other data to be include with the session
  */
 OlmDevice.prototype.addInboundGroupSession = async function(
     roomId, senderKey, forwardingCurve25519KeyChain,
     sessionId, sessionKey, keysClaimed,
-    exportFormat,
+    exportFormat, extraSessionData = {},
 ) {
     await this._cryptoStore.doTxn(
         'readwrite', [
             IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
             IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+            IndexedDBCryptoStore.STORE_SHARED_HISTORY_INBOUND_GROUP_SESSIONS,
         ], (txn) => {
             /* if we already have this session, consider updating it */
             this._getInboundGroupSession(
@@ -1019,9 +1063,14 @@ OlmDevice.prototype.addInboundGroupSession = async function(
                                     + senderKey + "/" + sessionId,
                             );
                             if (existingSession.first_known_index()
-                                <= session.first_known_index()) {
+                                <= session.first_known_index()
+                                && !(existingSession.first_known_index() == session.first_known_index()
+                                    && !extraSessionData.untrusted
+                                    && existingSessionData.untrusted)) {
                                 // existing session has lower index (i.e. can
-                                // decrypt more), so keep it
+                                // decrypt more), or they have the same index and
+                                // the new sessions trust does not win over the old
+                                // sessions trust, so keep it
                                 logger.log(
                                     `Keeping existing megolm session ${sessionId}`,
                                 );
@@ -1029,22 +1078,34 @@ OlmDevice.prototype.addInboundGroupSession = async function(
                             }
                         }
 
-                        const sessionData = {
+                        logger.info(
+                            "Storing megolm session " + senderKey + "/" + sessionId +
+                            " with first index " + session.first_known_index(),
+                        );
+
+                        const sessionData = Object.assign({}, extraSessionData, {
                             room_id: roomId,
                             session: session.pickle(this._pickleKey),
                             keysClaimed: keysClaimed,
                             forwardingCurve25519KeyChain: forwardingCurve25519KeyChain,
-                        };
+                        });
 
                         this._cryptoStore.storeEndToEndInboundGroupSession(
                             senderKey, sessionId, sessionData, txn,
                         );
+
+                        if (!existingSession && extraSessionData.sharedHistory) {
+                            this._cryptoStore.addSharedHistoryInboundGroupSession(
+                                roomId, senderKey, sessionId, txn,
+                            );
+                        }
                     } finally {
                         session.free();
                     }
                 },
             );
         },
+        logger.withPrefix("[addInboundGroupSession]"),
     );
 };
 
@@ -1210,10 +1271,12 @@ OlmDevice.prototype.decryptGroupMessage = async function(
                         forwardingCurve25519KeyChain: (
                             sessionData.forwardingCurve25519KeyChain || []
                         ),
+                        untrusted: sessionData.untrusted,
                     };
                 },
             );
         },
+        logger.withPrefix("[decryptGroupMessage]"),
     );
 
     if (error) {
@@ -1259,6 +1322,7 @@ OlmDevice.prototype.hasInboundSessionKeys = async function(roomId, senderKey, se
                 },
             );
         },
+        logger.withPrefix("[hasInboundSessionKeys]"),
     );
 
     return result;
@@ -1314,10 +1378,12 @@ OlmDevice.prototype.getInboundGroupSessionKey = async function(
                         "forwarding_curve25519_key_chain":
                             sessionData.forwardingCurve25519KeyChain || [],
                         "sender_claimed_ed25519_key": senderEd25519Key,
+                        "shared_history": sessionData.sharedHistory || false,
                     };
                 },
             );
         },
+        logger.withPrefix("[getInboundGroupSessionKey]"),
     );
 
     return result;
@@ -1345,10 +1411,24 @@ OlmDevice.prototype.exportInboundGroupSession = function(
             "session_key": session.export_session(messageIndex),
             "forwarding_curve25519_key_chain": session.forwardingCurve25519KeyChain || [],
             "first_known_index": session.first_known_index(),
+            "org.matrix.msc3061.shared_history": sessionData.sharedHistory || false,
         };
     });
 };
 
+OlmDevice.prototype.getSharedHistoryInboundGroupSessions = async function(roomId) {
+    let result;
+    await this._cryptoStore.doTxn(
+        'readonly', [
+            IndexedDBCryptoStore.STORE_SHARED_HISTORY_INBOUND_GROUP_SESSIONS,
+        ], (txn) => {
+            result = this._cryptoStore.getSharedHistoryInboundGroupSessions(roomId, txn);
+        },
+        logger.withPrefix("[getSharedHistoryInboundGroupSessionsForRoom]"),
+    );
+    return result;
+};
+
 // Utilities
 // =========
 

src/crypto/OutgoingRoomKeyRequestManager.js

@@ -14,8 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {logger} from '../logger';
-import * as utils from '../utils';
+import { logger } from '../logger';
 
 /**
  * Internal module. Management of outgoing room key requests.
@@ -58,7 +57,7 @@ const SEND_KEY_REQUESTS_DELAY_MS = 500;
  *
  * @enum {number}
  */
-const ROOM_KEY_REQUEST_STATES = {
+export const ROOM_KEY_REQUEST_STATES = {
     /** request not yet sent */
     UNSENT: 0,
 
@@ -97,10 +96,6 @@ export class OutgoingRoomKeyRequestManager {
      */
     start() {
         this._clientRunning = true;
-
-        // set the timer going, to handle any requests which didn't get sent
-        // on the previous run of the client.
-        this._startTimer();
     }
 
     /**
@@ -113,7 +108,14 @@ export class OutgoingRoomKeyRequestManager {
     }
 
     /**
-     * Send off a room key request, if we haven't already done so.
+     * Send any requests that have been queued
+     */
+    sendQueuedRequests() {
+        this._startTimer();
+    }
+
+    /**
+     * Queue up a room key request, if we haven't already queued or sent one.
      *
      * The `requestBody` is compared (with a deep-equality check) against
      * previous queued or sent requests and if it matches, no change is made.
@@ -129,7 +131,7 @@ export class OutgoingRoomKeyRequestManager {
      *    pending list (or we have established that a similar request already
      *    exists)
      */
-    async sendRoomKeyRequest(requestBody, recipients, resend=false) {
+    async queueRoomKeyRequest(requestBody, recipients, resend=false) {
         const req = await this._cryptoStore.getOutgoingRoomKeyRequest(
             requestBody,
         );
@@ -184,7 +186,7 @@ export class OutgoingRoomKeyRequestManager {
                         // in state ROOM_KEY_REQUEST_STATES.SENT, so we must have
                         // raced with another tab to mark the request cancelled.
                         // Try again, to make sure the request is resent.
-                        return await this.sendRoomKeyRequest(
+                        return await this.queueRoomKeyRequest(
                             requestBody, recipients, resend,
                         );
                     }
@@ -220,9 +222,6 @@ export class OutgoingRoomKeyRequestManager {
                 throw new Error('unhandled state: ' + req.state);
             }
         }
-        // some of the branches require the timer to be started.  Just start it
-        // all the time, because it doesn't hurt to start it.
-        this._startTimer();
     }
 
     /**
@@ -327,6 +326,21 @@ export class OutgoingRoomKeyRequestManager {
         );
     }
 
+    /**
+     * Find anything in `sent` state, and kick it around the loop again.
+     * This is intended for situations where something substantial has changed, and we
+     * don't really expect the other end to even care about the cancellation.
+     * For example, after initialization or self-verification.
+     * @return {Promise} An array of `queueRoomKeyRequest` outputs.
+     */
+    async cancelAndResendAllOutgoingRequests() {
+        const outgoings = await this._cryptoStore.getAllOutgoingRoomKeyRequestsByState(
+            ROOM_KEY_REQUEST_STATES.SENT,
+        );
+        return Promise.all(outgoings.map(({ requestBody, recipients }) =>
+            this.queueRoomKeyRequest(requestBody, recipients, true)));
+    }
+
     // start the background timer to send queued requests, if the timer isn't
     // already running
     _startTimer() {
@@ -366,15 +380,12 @@ export class OutgoingRoomKeyRequestManager {
             return Promise.resolve();
         }
 
-        logger.log("Looking for queued outgoing room key requests");
-
         return this._cryptoStore.getOutgoingRoomKeyRequestByState([
             ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
             ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
             ROOM_KEY_REQUEST_STATES.UNSENT,
         ]).then((req) => {
             if (!req) {
-                logger.log("No more outgoing room key requests");
                 this._sendOutgoingRoomKeyRequestsTimer = null;
                 return;
             }
@@ -398,7 +409,6 @@ export class OutgoingRoomKeyRequestManager {
             }).catch((e) => {
                 logger.error("Error sending room key request; will retry later.", e);
                 this._sendOutgoingRoomKeyRequestsTimer = null;
-                this._startTimer();
             });
         });
     }
@@ -485,7 +495,7 @@ function stringifyRequestBody(requestBody) {
 
 function stringifyRecipientList(recipients) {
     return '['
-        + utils.map(recipients, (r) => `${r.userId}:${r.deviceId}`).join(",")
+        + recipients.map((r) => `${r.userId}:${r.deviceId}`).join(",")
         + ']';
 }
 

src/crypto/RoomList.js

@@ -20,7 +20,7 @@ limitations under the License.
  * Manages the list of encrypted rooms
  */
 
-import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
+import { IndexedDBCryptoStore } from './store/indexeddb-crypto-store';
 
 /**
  * @alias module:crypto/RoomList

src/crypto/SecretStorage.js

@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,24 +14,27 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {EventEmitter} from 'events';
-import {logger} from '../logger';
+import { EventEmitter } from 'events';
+import { logger } from '../logger';
 import * as olmlib from './olmlib';
-import {pkVerify} from './olmlib';
-import {randomString} from '../randomstring';
+import { randomString } from '../randomstring';
+import { encryptAES, decryptAES } from './aes';
+import { encodeBase64 } from "./olmlib";
 
-export const SECRET_STORAGE_ALGORITHM_V1 = "m.secret_storage.v1.curve25519-aes-sha2";
+export const SECRET_STORAGE_ALGORITHM_V1_AES
+    = "m.secret_storage.v1.aes-hmac-sha2";
+
+const ZERO_STR = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
 
 /**
  * Implements Secure Secret Storage and Sharing (MSC1946)
  * @module crypto/SecretStorage
  */
 export class SecretStorage extends EventEmitter {
-    constructor(baseApis, cryptoCallbacks, crossSigningInfo) {
+    constructor(baseApis, cryptoCallbacks) {
         super();
         this._baseApis = baseApis;
         this._cryptoCallbacks = cryptoCallbacks;
-        this._crossSigningInfo = crossSigningInfo;
         this._requests = {};
         this._incomingRequests = {};
     }
@@ -45,7 +48,7 @@ export class SecretStorage extends EventEmitter {
     }
 
     setDefaultKeyId(keyId) {
-        return new Promise((resolve) => {
+        return new Promise(async (resolve, reject) => {
             const listener = (ev) => {
                 if (
                     ev.getType() === 'm.secret_storage.default_key' &&
@@ -57,10 +60,15 @@ export class SecretStorage extends EventEmitter {
             };
             this._baseApis.on('accountData', listener);
 
-            this._baseApis.setAccountData(
-                'm.secret_storage.default_key',
-                { key: keyId },
-            );
+            try {
+                await this._baseApis.setAccountData(
+                    'm.secret_storage.default_key',
+                    { key: keyId },
+                );
+            } catch (e) {
+                this._baseApis.removeListener('accountData', listener);
+                reject(e);
+            }
         });
     }
 
@@ -73,39 +81,29 @@ export class SecretStorage extends EventEmitter {
      * @param {string} [keyId] the ID of the key.  If not given, a random
      *     ID will be generated.
      *
-     * @return {string} the ID of the key
+     * @return {object} An object with:
+     *     keyId: {string} the ID of the key
+     *     keyInfo: {object} details about the key (iv, mac, passphrase)
      */
     async addKey(algorithm, opts, keyId) {
-        const keyData = {algorithm};
+        const keyInfo = { algorithm };
 
         if (!opts) opts = {};
 
         if (opts.name) {
-            keyData.name = opts.name;
+            keyInfo.name = opts.name;
         }
 
-        switch (algorithm) {
-        case SECRET_STORAGE_ALGORITHM_V1:
-        {
-            const decryption = new global.Olm.PkDecryption();
-            try {
-                const { passphrase, pubkey } = opts;
-                // Copies in public key details of the form generated by
-                // the Crypto module's `createRecoveryKeyFromPassphrase`.
-                if (passphrase && pubkey) {
-                    keyData.passphrase = passphrase;
-                    keyData.pubkey = pubkey;
-                } else if (pubkey) {
-                    keyData.pubkey = pubkey;
-                } else {
-                    keyData.pubkey = decryption.generate_key();
-                }
-            } finally {
-                decryption.free();
+        if (algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            if (opts.passphrase) {
+                keyInfo.passphrase = opts.passphrase;
             }
-            break;
-        }
-        default:
+            if (opts.key) {
+                const { iv, mac } = await SecretStorage._calculateKeyCheck(opts.key);
+                keyInfo.iv = iv;
+                keyInfo.mac = mac;
+            }
+        } else {
             throw new Error(`Unknown key algorithm ${opts.algorithm}`);
         }
 
@@ -119,40 +117,36 @@ export class SecretStorage extends EventEmitter {
             );
         }
 
-        await this._crossSigningInfo.signObject(keyData, 'master');
-
         await this._baseApis.setAccountData(
-            `m.secret_storage.key.${keyId}`, keyData,
+            `m.secret_storage.key.${keyId}`, keyInfo,
         );
 
-        return keyId;
+        return {
+            keyId,
+            keyInfo,
+        };
     }
 
     /**
-     * Signs a given secret storage key with the cross-signing master key.
+     * Get the key information for a given ID.
      *
-     * @param {string} [keyId = default key's ID] The ID of the key to sign.
-     *     Defaults to the default key ID if not provided.
+     * @param {string} [keyId = default key's ID] The ID of the key to check
+     *     for. Defaults to the default key ID if not provided.
+     * @returns {Array?} If the key was found, the return value is an array of
+     *     the form [keyId, keyInfo].  Otherwise, null is returned.
      */
-    async signKey(keyId) {
+    async getKey(keyId) {
         if (!keyId) {
             keyId = await this.getDefaultKeyId();
         }
         if (!keyId) {
-            throw new Error("signKey requires a key ID");
+            return null;
         }
 
         const keyInfo = await this._baseApis.getAccountDataFromServer(
-            `m.secret_storage.key.${keyId}`,
-        );
-        if (!keyInfo) {
-            throw new Error(`Key ${keyId} does not exist in account data`);
-        }
-
-        await this._crossSigningInfo.signObject(keyInfo, 'master');
-        await this._baseApis.setAccountData(
-            `m.secret_storage.key.${keyId}`, keyInfo,
+            "m.secret_storage.key." + keyId,
         );
+        return keyInfo ? [keyId, keyInfo] : null;
     }
 
     /**
@@ -163,16 +157,33 @@ export class SecretStorage extends EventEmitter {
      * @return {boolean} Whether we have the key.
      */
     async hasKey(keyId) {
-        if (!keyId) {
-            keyId = await this.getDefaultKeyId();
-        }
-        if (!keyId) {
-            return false;
-        }
+        return !!(await this.getKey(keyId));
+    }
 
-        return !!this._baseApis.getAccountDataFromServer(
-            "m.secret_storage.key." + keyId,
-        );
+    /**
+     * Check whether a key matches what we expect based on the key info
+     *
+     * @param {Uint8Array} key the key to check
+     * @param {object} info the key info
+     *
+     * @return {boolean} whether or not the key matches
+     */
+    async checkKey(key, info) {
+        if (info.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            if (info.mac) {
+                const { mac } = await SecretStorage._calculateKeyCheck(key, info.iv);
+                return info.mac.replace(/=+$/g, '') === mac.replace(/=+$/g, '');
+            } else {
+                // if we have no information, we have to assume the key is right
+                return true;
+            }
+        } else {
+            throw new Error("Unknown algorithm");
+        }
+    }
+
+    static async _calculateKeyCheck(key, iv) {
+        return await encryptAES(ZERO_STR, key, "", iv);
     }
 
     /**
@@ -207,27 +218,12 @@ export class SecretStorage extends EventEmitter {
                 throw new Error("Unknown key: " + keyId);
             }
 
-            // check signature of key info
-            pkVerify(
-                keyInfo,
-                this._crossSigningInfo.getId('master'),
-                this._crossSigningInfo.userId,
-            );
-
             // encrypt secret, based on the algorithm
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-            {
-                const encryption = new global.Olm.PkEncryption();
-                try {
-                    encryption.set_recipient_key(keyInfo.pubkey);
-                    encrypted[keyId] = encryption.encrypt(secret);
-                } finally {
-                    encryption.free();
-                }
-                break;
-            }
-            default:
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                const keys = { [keyId]: keyInfo };
+                const [, encryption] = await this._getSecretStorageKey(keys, name);
+                encrypted[keyId] = await encryption.encrypt(secret);
+            } else {
                 logger.warn("unknown algorithm for secret storage key " + keyId
                             + ": " + keyInfo.algorithm);
                 // do nothing if we don't understand the encryption algorithm
@@ -235,30 +231,7 @@ export class SecretStorage extends EventEmitter {
         }
 
         // save encrypted secret
-        await this._baseApis.setAccountData(name, {encrypted});
-    }
-
-    /**
-     * Store a secret defined to be the same as the given key.
-     * No secret information will be stored, instead the secret will
-     * be stored with a marker to say that the contents of the secret is
-     * the value of the given key.
-     * This is useful for migration from systems that predate SSSS such as
-     * key backup.
-     *
-     * @param {string} name The name of the secret
-     * @param {string} keyId The ID of the key whose value will be the
-     *     value of the secret
-     * @returns {Promise} resolved when account data is saved
-     */
-    storePassthrough(name, keyId) {
-        return this._baseApis.setAccountData(name, {
-            encrypted: {
-                [keyId]: {
-                    passthrough: true,
-                },
-            },
-        });
+        await this._baseApis.setAccountData(name, { encrypted });
     }
 
     /**
@@ -279,7 +252,7 @@ export class SecretStorage extends EventEmitter {
         ) {
             const hasKey = await this.hasKey(keys[0]);
             if (hasKey) {
-                console.log("Fixing up passthrough secret: " + name);
+                logger.log("Fixing up passthrough secret: " + name);
                 await this.storePassthrough(name, keys[0]);
                 const newData = await this._baseApis.getAccountDataFromServer(name);
                 return newData;
@@ -316,22 +289,19 @@ export class SecretStorage extends EventEmitter {
                 "m.secret_storage.key." + keyId,
             );
             const encInfo = secretInfo.encrypted[keyId];
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-                if (
-                    keyInfo.pubkey && (
-                        (encInfo.ciphertext && encInfo.mac && encInfo.ephemeral) ||
-                        encInfo.passthrough
-                    )
-                ) {
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
                     keys[keyId] = keyInfo;
                 }
-                break;
-            default:
-                // do nothing if we don't understand the encryption algorithm
             }
         }
 
+        if (Object.keys(keys).length === 0) {
+            throw new Error(`Could not decrypt ${name} because none of ` +
+                `the keys it is encrypted with are for a supported algorithm`);
+        }
+
         let keyId;
         let decryption;
         try {
@@ -341,18 +311,13 @@ export class SecretStorage extends EventEmitter {
             const encInfo = secretInfo.encrypted[keyId];
 
             // We don't actually need the decryption object if it's a passthrough
-            // since we just want to return the key itself.
-            if (encInfo.passthrough) return decryption.get_private_key();
+            // since we just want to return the key itself. It must be base64
+            // encoded, since this is how a key would normally be stored.
+            if (encInfo.passthrough) return encodeBase64(decryption.get_private_key());
 
-            // decrypt secret
-            switch (keys[keyId].algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-                return decryption.decrypt(
-                    encInfo.ephemeral, encInfo.mac, encInfo.ciphertext,
-                );
-            }
+            return await decryption.decrypt(encInfo);
         } finally {
-            if (decryption) decryption.free();
+            if (decryption && decryption.free) decryption.free();
         }
     }
 
@@ -362,55 +327,43 @@ export class SecretStorage extends EventEmitter {
      * @param {string} name the name of the secret
      * @param {boolean} checkKey check if the secret is encrypted by a trusted key
      *
-     * @return {boolean} whether or not the secret is stored
+     * @return {object?} map of key name to key info the secret is encrypted
+     *     with, or null if it is not present or not encrypted with a trusted
+     *     key
      */
     async isStored(name, checkKey) {
         // check if secret exists
         let secretInfo = await this._baseApis.getAccountDataFromServer(name);
-        if (!secretInfo) return false;
+        if (!secretInfo) return null;
         if (!secretInfo.encrypted) {
             // try to fix it up
             secretInfo = await this._fixupStoredSecret(name, secretInfo);
             if (!secretInfo || !secretInfo.encrypted) {
-                return false;
+                return null;
             }
         }
 
         if (checkKey === undefined) checkKey = true;
 
-        // check if secret is encrypted by a known/trusted secret and
-        // encryption looks sane
+        const ret = {};
+
+        // filter secret encryption keys with supported algorithm
         for (const keyId of Object.keys(secretInfo.encrypted)) {
             // get key information from key storage
             const keyInfo = await this._baseApis.getAccountDataFromServer(
                 "m.secret_storage.key." + keyId,
             );
-            if (!keyInfo) return false;
+            if (!keyInfo) continue;
             const encInfo = secretInfo.encrypted[keyId];
-            if (checkKey) {
-                pkVerify(
-                    keyInfo,
-                    this._crossSigningInfo.getId('master'),
-                    this._crossSigningInfo.userId,
-                );
-            }
 
-            // We don't actually need the decryption object if it's a passthrough
-            // since we just want to return the key itself.
-            if (encInfo.passthrough) return true;
-
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-                if (keyInfo.pubkey && encInfo.ciphertext && encInfo.mac
-                    && encInfo.ephemeral) {
-                    return true;
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
+                    ret[keyId] = keyInfo;
                 }
-                break;
-            default:
-                // do nothing if we don't understand the encryption algorithm
             }
         }
-        return false;
+        return Object.keys(ret).length ? ret : null;
     }
 
     /**
@@ -425,6 +378,7 @@ export class SecretStorage extends EventEmitter {
         const requestId = this._baseApis.makeTxnId();
 
         const requestControl = this._requests[requestId] = {
+            name,
             devices,
         };
         const promise = new Promise((resolve, reject) => {
@@ -462,6 +416,7 @@ export class SecretStorage extends EventEmitter {
         for (const device of devices) {
             toDevice[device] = requestData;
         }
+        logger.info(`Request secret ${name} from ${devices}, id ${requestId}`);
         this._baseApis.sendToDevice("m.secret.request", {
             [this._baseApis.getUserId()]: toDevice,
         });
@@ -489,7 +444,7 @@ export class SecretStorage extends EventEmitter {
                 && this._incomingRequests[deviceId][content.request_id]) {
                 logger.info("received request cancellation for secret (" + sender
                             + ", " + deviceId + ", " + content.request_id + ")");
-                this.baseApis.emit("crypto.secrets.requestCancelled", {
+                this._baseApis.emit("crypto.secrets.requestCancelled", {
                     user_id: sender,
                     device_id: deviceId,
                     request_id: content.request_id,
@@ -507,14 +462,15 @@ export class SecretStorage extends EventEmitter {
             if (!this._cryptoCallbacks.onSecretRequested) {
                 return;
             }
-            const secret = await this._cryptoCallbacks.onSecretRequested({
-                user_id: sender,
-                device_id: deviceId,
-                request_id: content.request_id,
-                name: content.name,
-                device_trust: this._baseApis.checkDeviceTrust(sender, deviceId),
-            });
+            const secret = await this._cryptoCallbacks.onSecretRequested(
+                sender,
+                deviceId,
+                content.request_id,
+                content.name,
+                this._baseApis.checkDeviceTrust(sender, deviceId),
+            );
             if (secret) {
+                logger.info(`Preparing ${content.name} secret for ${deviceId}`);
                 const payload = {
                     type: "m.secret.send",
                     content: {
@@ -524,15 +480,15 @@ export class SecretStorage extends EventEmitter {
                 };
                 const encryptedContent = {
                     algorithm: olmlib.OLM_ALGORITHM,
-                    sender_key: this._baseApis._crypto._olmDevice.deviceCurve25519Key,
+                    sender_key: this._baseApis.crypto._olmDevice.deviceCurve25519Key,
                     ciphertext: {},
                 };
                 await olmlib.ensureOlmSessionsForDevices(
-                    this._baseApis._crypto._olmDevice,
+                    this._baseApis.crypto._olmDevice,
                     this._baseApis,
                     {
                         [sender]: [
-                            await this._baseApis.getStoredDevice(sender, deviceId),
+                            this._baseApis.getStoredDevice(sender, deviceId),
                         ],
                     },
                 );
@@ -540,9 +496,9 @@ export class SecretStorage extends EventEmitter {
                     encryptedContent.ciphertext,
                     this._baseApis.getUserId(),
                     this._baseApis.deviceId,
-                    this._baseApis._crypto._olmDevice,
+                    this._baseApis.crypto._olmDevice,
                     sender,
-                    this._baseApis._crypto.getStoredDevice(sender, deviceId),
+                    this._baseApis.getStoredDevice(sender, deviceId),
                     payload,
                 );
                 const contentMap = {
@@ -551,7 +507,10 @@ export class SecretStorage extends EventEmitter {
                     },
                 };
 
+                logger.info(`Sending ${content.name} secret for ${deviceId}`);
                 this._baseApis.sendToDevice("m.room.encrypted", contentMap);
+            } else {
+                logger.info(`Request denied for ${content.name} secret for ${deviceId}`);
             }
         }
     }
@@ -563,12 +522,12 @@ export class SecretStorage extends EventEmitter {
             return;
         }
         const content = event.getContent();
-        logger.log("got secret share for request ", content.request_id);
+        logger.log("got secret share for request", content.request_id);
         const requestControl = this._requests[content.request_id];
         if (requestControl) {
             // make sure that the device that sent it is one of the devices that
             // we requested from
-            const deviceInfo = this._baseApis._crypto._deviceList.getDeviceByIdentityKey(
+            const deviceInfo = this._baseApis.crypto._deviceList.getDeviceByIdentityKey(
                 olmlib.OLM_ALGORITHM,
                 event.getSenderKey(),
             );
@@ -583,6 +542,10 @@ export class SecretStorage extends EventEmitter {
                 return;
             }
 
+            logger.log(
+                `Successfully received secret ${requestControl.name} ` +
+                `from ${deviceInfo.deviceId}`,
+            );
             requestControl.resolve(content.secret);
         }
     }
@@ -606,27 +569,18 @@ export class SecretStorage extends EventEmitter {
             throw new Error("App returned unknown key from getSecretStorageKey!");
         }
 
-        switch (keys[keyId].algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-            {
-                const decryption = new global.Olm.PkDecryption();
-                let pubkey;
-                try {
-                    pubkey = decryption.init_with_private_key(privateKey);
-                } catch (e) {
-                    decryption.free();
-                    throw new Error("getSecretStorageKey callback returned invalid key");
-                }
-                if (pubkey !== keys[keyId].pubkey) {
-                    decryption.free();
-                    throw new Error(
-                        "getSecretStorageKey callback returned incorrect key",
-                    );
-                }
-                return [keyId, decryption];
-            }
-            default:
-                throw new Error("Unknown key type: " + keys[keyId].algorithm);
+        if (keys[keyId].algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            const decryption = {
+                encrypt: async function(secret) {
+                    return await encryptAES(secret, privateKey, name);
+                },
+                decrypt: async function(encInfo) {
+                    return await decryptAES(encInfo, privateKey, name);
+                },
+            };
+            return [keyId, decryption];
+        } else {
+            throw new Error("Unknown key type: " + keys[keyId].algorithm);
         }
     }
 }

src/crypto/aes.js

@@ -0,0 +1,251 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { getCrypto } from '../utils';
+import { decodeBase64, encodeBase64 } from './olmlib';
+
+const subtleCrypto = (typeof window !== "undefined" && window.crypto) ?
+    (window.crypto.subtle || window.crypto.webkitSubtle) : null;
+
+// salt for HKDF, with 8 bytes of zeros
+const zerosalt = new Uint8Array(8);
+
+/**
+ * encrypt a string in Node.js
+ *
+ * @param {string} data the plaintext to encrypt
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
+ */
+async function encryptNode(data, key, name, ivStr) {
+    const crypto = getCrypto();
+    if (!crypto) {
+        throw new Error("No usable crypto implementation");
+    }
+
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = crypto.randomBytes(16);
+    }
+
+    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
+    // (which would mean we wouldn't be able to decrypt on Android). The loss
+    // of a single bit of iv is a price we have to pay.
+    iv[8] &= 0x7f;
+
+    const [aesKey, hmacKey] = deriveKeysNode(key, name);
+
+    const cipher = crypto.createCipheriv("aes-256-ctr", aesKey, iv);
+    const ciphertext = cipher.update(data, "utf-8", "base64")
+          + cipher.final("base64");
+
+    const hmac = crypto.createHmac("sha256", hmacKey)
+        .update(ciphertext, "base64").digest("base64");
+
+    return {
+        iv: encodeBase64(iv),
+        ciphertext: ciphertext,
+        mac: hmac,
+    };
+}
+
+/**
+ * decrypt a string in Node.js
+ *
+ * @param {object} data the encrypted data
+ * @param {string} data.ciphertext the ciphertext in base64
+ * @param {string} data.iv the initialization vector in base64
+ * @param {string} data.mac the HMAC in base64
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function decryptNode(data, key, name) {
+    const crypto = getCrypto();
+    if (!crypto) {
+        throw new Error("No usable crypto implementation");
+    }
+
+    const [aesKey, hmacKey] = deriveKeysNode(key, name);
+
+    const hmac = crypto.createHmac("sha256", hmacKey)
+        .update(data.ciphertext, "base64").digest("base64").replace(/=+$/g, '');
+
+    if (hmac !== data.mac.replace(/=+$/g, '')) {
+        throw new Error(`Error decrypting secret ${name}: bad MAC`);
+    }
+
+    const decipher = crypto.createDecipheriv(
+        "aes-256-ctr", aesKey, decodeBase64(data.iv),
+    );
+    return decipher.update(data.ciphertext, "base64", "utf-8")
+          + decipher.final("utf-8");
+}
+
+function deriveKeysNode(key, name) {
+    const crypto = getCrypto();
+    const prk = crypto.createHmac("sha256", zerosalt)
+        .update(key).digest();
+
+    const b = Buffer.alloc(1, 1);
+    const aesKey = crypto.createHmac("sha256", prk)
+        .update(name, "utf-8").update(b).digest();
+    b[0] = 2;
+    const hmacKey = crypto.createHmac("sha256", prk)
+        .update(aesKey).update(name, "utf-8").update(b).digest();
+
+    return [aesKey, hmacKey];
+}
+
+/**
+ * encrypt a string in Node.js
+ *
+ * @param {string} data the plaintext to encrypt
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
+ */
+async function encryptBrowser(data, key, name, ivStr) {
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = new Uint8Array(16);
+        window.crypto.getRandomValues(iv);
+    }
+
+    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
+    // (which would mean we wouldn't be able to decrypt on Android). The loss
+    // of a single bit of iv is a price we have to pay.
+    iv[8] &= 0x7f;
+
+    const [aesKey, hmacKey] = await deriveKeysBrowser(key, name);
+    const encodedData = new TextEncoder().encode(data);
+
+    const ciphertext = await subtleCrypto.encrypt(
+        {
+            name: "AES-CTR",
+            counter: iv,
+            length: 64,
+        },
+        aesKey,
+        encodedData,
+    );
+
+    const hmac = await subtleCrypto.sign(
+        { name: 'HMAC' },
+        hmacKey,
+        ciphertext,
+    );
+
+    return {
+        iv: encodeBase64(iv),
+        ciphertext: encodeBase64(ciphertext),
+        mac: encodeBase64(hmac),
+    };
+}
+
+/**
+ * decrypt a string in the browser
+ *
+ * @param {object} data the encrypted data
+ * @param {string} data.ciphertext the ciphertext in base64
+ * @param {string} data.iv the initialization vector in base64
+ * @param {string} data.mac the HMAC in base64
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function decryptBrowser(data, key, name) {
+    const [aesKey, hmacKey] = await deriveKeysBrowser(key, name);
+
+    const ciphertext = decodeBase64(data.ciphertext);
+
+    if (!await subtleCrypto.verify(
+        { name: "HMAC" },
+        hmacKey,
+        decodeBase64(data.mac),
+        ciphertext,
+    )) {
+        throw new Error(`Error decrypting secret ${name}: bad MAC`);
+    }
+
+    const plaintext = await subtleCrypto.decrypt(
+        {
+            name: "AES-CTR",
+            counter: decodeBase64(data.iv),
+            length: 64,
+        },
+        aesKey,
+        ciphertext,
+    );
+
+    return new TextDecoder().decode(new Uint8Array(plaintext));
+}
+
+async function deriveKeysBrowser(key, name) {
+    const hkdfkey = await subtleCrypto.importKey(
+        'raw',
+        key,
+        { name: "HKDF" },
+        false,
+        ["deriveBits"],
+    );
+    const keybits = await subtleCrypto.deriveBits(
+        {
+            name: "HKDF",
+            salt: zerosalt,
+            info: (new TextEncoder().encode(name)),
+            hash: "SHA-256",
+        },
+        hkdfkey,
+        512,
+    );
+
+    const aesKey = keybits.slice(0, 32);
+    const hmacKey = keybits.slice(32);
+
+    const aesProm = subtleCrypto.importKey(
+        'raw',
+        aesKey,
+        { name: 'AES-CTR' },
+        false,
+        ['encrypt', 'decrypt'],
+    );
+
+    const hmacProm = subtleCrypto.importKey(
+        'raw',
+        hmacKey,
+        {
+            name: 'HMAC',
+            hash: { name: 'SHA-256' },
+        },
+        false,
+        ['sign', 'verify'],
+    );
+
+    return await Promise.all([aesProm, hmacProm]);
+}
+
+export function encryptAES(...args) {
+    return subtleCrypto ? encryptBrowser(...args) : encryptNode(...args);
+}
+
+export function decryptAES(...args) {
+    return subtleCrypto ? decryptBrowser(...args) : decryptNode(...args);
+}
+

src/crypto/algorithms/base.js

@@ -46,7 +46,7 @@ export const DECRYPTION_CLASSES = {};
  * @param {string} params.deviceId The identifier for this device.
  * @param {module:crypto} params.crypto crypto core
  * @param {module:crypto/OlmDevice} params.olmDevice olm.js wrapper
- * @param {module:base-apis~MatrixBaseApis} baseApis base matrix api interface
+ * @param {MatrixClient} baseApis base matrix api interface
  * @param {string} params.roomId  The ID of the room we will be sending to
  * @param {object} params.config  The body of the m.room.encryption event
  */
@@ -102,7 +102,7 @@ export class EncryptionAlgorithm {
  * @param {string} params.userId  The UserID for the local user
  * @param {module:crypto} params.crypto crypto core
  * @param {module:crypto/OlmDevice} params.olmDevice olm.js wrapper
- * @param {module:base-apis~MatrixBaseApis} baseApis base matrix api interface
+ * @param {MatrixClient} baseApis base matrix api interface
  * @param {string=} params.roomId The ID of the room we will be receiving
  *     from. Null for to-device events.
  */

src/crypto/algorithms/megolm.js

@@ -22,9 +22,9 @@ limitations under the License.
  * @module crypto/algorithms/megolm
  */
 
-import {logger} from '../../logger';
+import { logger } from '../../logger';
 import * as utils from "../../utils";
-import {polyfillSuper} from "../../utils";
+import { polyfillSuper } from "../../utils";
 import * as olmlib from "../olmlib";
 import {
     DecryptionAlgorithm,
@@ -34,13 +34,29 @@ import {
     UnknownDeviceError,
 } from "./base";
 
-import {WITHHELD_MESSAGES} from '../OlmDevice';
+import { WITHHELD_MESSAGES } from '../OlmDevice';
+
+// determine whether the key can be shared with invitees
+function isRoomSharedHistory(room) {
+    const visibilityEvent = room.currentState &&
+          room.currentState.getStateEvents("m.room.history_visibility", "");
+    // NOTE: if the room visibility is unset, it would normally default to
+    // "world_readable".
+    // (https://spec.matrix.org/unstable/client-server-api/#server-behaviour-5)
+    // But we will be paranoid here, and treat it as a situation where the room
+    // is not shared-history
+    const visibility = visibilityEvent && visibilityEvent.getContent() &&
+          visibilityEvent.getContent().history_visibility;
+    return ["world_readable", "shared"].includes(visibility);
+}
 
 /**
  * @private
  * @constructor
  *
  * @param {string} sessionId
+ * @param {boolean} sharedHistory whether the session can be freely shared with
+ *    other group members, according to the room history visibility settings
  *
  * @property {string} sessionId
  * @property {Number} useCount     number of times this session has been used
@@ -50,15 +66,15 @@ import {WITHHELD_MESSAGES} from '../OlmDevice';
  *    devices with which we have shared the session key
  *        userId -> {deviceId -> msgindex}
  */
-function OutboundSessionInfo(sessionId) {
+function OutboundSessionInfo(sessionId, sharedHistory = false) {
     this.sessionId = sessionId;
     this.useCount = 0;
     this.creationTime = new Date().getTime();
     this.sharedWithDevices = {};
     this.blockedDevicesNotified = {};
+    this.sharedHistory = sharedHistory;
 }
 
-
 /**
  * Check if it's time to rotate the session
  *
@@ -141,7 +157,6 @@ OutboundSessionInfo.prototype.sharedWithTooManyDevices = function(
     }
 };
 
-
 /**
  * Megolm encryption implementation
  *
@@ -183,14 +198,17 @@ utils.inherits(MegolmEncryption, EncryptionAlgorithm);
 /**
  * @private
  *
+ * @param {module:models/room} room
  * @param {Object} devicesInRoom The devices in this room, indexed by user ID
  * @param {Object} blocked The devices that are blocked, indexed by user ID
+ * @param {boolean} [singleOlmCreationPhase] Only perform one round of olm
+ *     session creation
  *
  * @return {Promise} Promise which resolves to the
  *    OutboundSessionInfo when setup is complete.
  */
 MegolmEncryption.prototype._ensureOutboundSession = async function(
-    devicesInRoom, blocked,
+    room, devicesInRoom, blocked, singleOlmCreationPhase,
 ) {
     let session;
 
@@ -202,6 +220,13 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
     const prepareSession = async (oldSession) => {
         session = oldSession;
 
+        const sharedHistory = isRoomSharedHistory(room);
+
+        // history visibility changed
+        if (session && sharedHistory !== session.sharedHistory) {
+            session = null;
+        }
+
         // need to make a brand new session?
         if (session && session.needsRotation(this._sessionRotationPeriodMsgs,
                                              this._sessionRotationPeriodMs)
@@ -217,7 +242,7 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
 
         if (!session) {
             logger.log(`Starting new megolm session for room ${this._roomId}`);
-            session = await this._prepareNewSession();
+            session = await this._prepareNewSession(sharedHistory);
             logger.log(`Started new megolm session ${session.sessionId} ` +
                        `for room ${this._roomId}`);
             this._outboundSessions[session.sessionId] = session;
@@ -248,11 +273,12 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
         const payload = {
             type: "m.room_key",
             content: {
-                algorithm: olmlib.MEGOLM_ALGORITHM,
-                room_id: this._roomId,
-                session_id: session.sessionId,
-                session_key: key.key,
-                chain_index: key.chain_index,
+                "algorithm": olmlib.MEGOLM_ALGORITHM,
+                "room_id": this._roomId,
+                "session_id": session.sessionId,
+                "session_key": key.key,
+                "chain_index": key.chain_index,
+                "org.matrix.msc3061.shared_history": sharedHistory,
             },
         };
         const [devicesWithoutSession, olmSessions] = await olmlib.getExistingOlmSessions(
@@ -262,68 +288,74 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
         await Promise.all([
             (async () => {
                 // share keys with devices that we already have a session for
+                logger.debug(`Sharing keys with existing Olm sessions in ${this._roomId}`);
                 await this._shareKeyWithOlmSessions(
                     session, key, payload, olmSessions,
                 );
+                logger.debug(`Shared keys with existing Olm sessions in ${this._roomId}`);
             })(),
             (async () => {
+                logger.debug(`Sharing keys (start phase 1) with new Olm sessions in ${this._roomId}`);
                 const errorDevices = [];
 
                 // meanwhile, establish olm sessions for devices that we don't
-                // already have a session for, and share keys with them.  Use a
-                // shorter timeout when fetching one-time keys.
+                // already have a session for, and share keys with them.  If
+                // we're doing two phases of olm session creation, use a
+                // shorter timeout when fetching one-time keys for the first
+                // phase.
+                const start = Date.now();
+                const failedServers = [];
                 await this._shareKeyWithDevices(
-                    session, key, payload, devicesWithoutSession, errorDevices, 2000,
+                    session, key, payload, devicesWithoutSession, errorDevices,
+                    singleOlmCreationPhase ? 10000 : 2000, failedServers,
                 );
+                logger.debug(`Shared keys (end phase 1) with new Olm sessions in ${this._roomId}`);
 
-                (async () => {
-                    // Retry sending keys to devices that we were unable to establish
-                    // an olm session for.  This time, we use a longer timeout, but we
-                    // do this in the background and don't block anything else while we
-                    // do this.
-                    const retryDevices = {};
-                    for (const {userId, deviceInfo} of errorDevices) {
-                        retryDevices[userId] = retryDevices[userId] || [];
-                        retryDevices[userId].push(deviceInfo);
-                    }
+                if (!singleOlmCreationPhase && (Date.now() - start < 10000)) {
+                    // perform the second phase of olm session creation if requested,
+                    // and if the first phase didn't take too long
+                    (async () => {
+                        // Retry sending keys to devices that we were unable to establish
+                        // an olm session for.  This time, we use a longer timeout, but we
+                        // do this in the background and don't block anything else while we
+                        // do this.  We only need to retry users from servers that didn't
+                        // respond the first time.
+                        const retryDevices = {};
+                        const failedServerMap = new Set;
+                        for (const server of failedServers) {
+                            failedServerMap.add(server);
+                        }
+                        const failedDevices = [];
+                        for (const { userId, deviceInfo } of errorDevices) {
+                            const userHS = userId.slice(userId.indexOf(":") + 1);
+                            if (failedServerMap.has(userHS)) {
+                                retryDevices[userId] = retryDevices[userId] || [];
+                                retryDevices[userId].push(deviceInfo);
+                            } else {
+                                // if we aren't going to retry, then handle it
+                                // as a failed device
+                                failedDevices.push({ userId, deviceInfo });
+                            }
+                        }
 
-                    const failedDevices = [];
-                    await this._shareKeyWithDevices(
-                        session, key, payload, retryDevices, failedDevices,
-                    );
-
-                    const blockedMap = {};
-                    const filteredFailedDevices =
-                        await this._olmDevice.filterOutNotifiedErrorDevices(
-                            failedDevices,
+                        logger.debug(`Sharing keys (start phase 2) with new Olm sessions in ${this._roomId}`);
+                        await this._shareKeyWithDevices(
+                            session, key, payload, retryDevices, failedDevices, 30000,
                         );
-                    for (const {userId, deviceInfo} of filteredFailedDevices) {
-                        blockedMap[userId] = blockedMap[userId] || {};
-                        // we use a similar format to what
-                        // olmlib.ensureOlmSessionsForDevices returns, so that
-                        // we can use the same function to split
-                        blockedMap[userId][deviceInfo.deviceId] = {
-                            device: {
-                                code: "m.no_olm",
-                                reason: WITHHELD_MESSAGES["m.no_olm"],
-                                deviceInfo,
-                            },
-                        };
+                        logger.debug(`Shared keys (end phase 2) with new Olm sessions in ${this._roomId}`);
 
-                        const deviceId = deviceInfo.deviceId;
-
-                        // mark this device as "handled" because we don't want to try
-                        // to claim a one-time-key for dead devices on every message.
-                        session.markSharedWithDevice(userId, deviceId, key.chain_index);
-                    }
-
-                    // notify devices that we couldn't get an olm session
-                    await this._notifyBlockedDevices(session, blockedMap);
-                })();
+                        await this._notifyFailedOlmDevices(session, key, failedDevices);
+                    })();
+                } else {
+                    await this._notifyFailedOlmDevices(session, key, errorDevices);
+                }
+                logger.debug(`Shared keys (all phases done) with new Olm sessions in ${this._roomId}`);
             })(),
             (async () => {
+                logger.debug(`Notifying blocked devices in ${this._roomId}`);
                 // also, notify blocked devices that they're blocked
                 const blockedMap = {};
+                let blockedCount = 0;
                 for (const [userId, userBlockedDevices] of Object.entries(blocked)) {
                     for (const [deviceId, device] of Object.entries(userBlockedDevices)) {
                         if (
@@ -331,12 +363,14 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
                             session.blockedDevicesNotified[userId][deviceId] === undefined
                         ) {
                             blockedMap[userId] = blockedMap[userId] || {};
-                            blockedMap[userId][deviceId] = {device};
+                            blockedMap[userId][deviceId] = { device };
+                            blockedCount++;
                         }
                     }
                 }
 
                 await this._notifyBlockedDevices(session, blockedMap);
+                logger.debug(`Notified ${blockedCount} blocked devices in ${this._roomId}`);
             })(),
         ]);
     };
@@ -349,6 +383,11 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
     // first wait for the previous share to complete
     const prom = this._setupPromise.then(prepareSession);
 
+    // Ensure any failures are logged for debugging
+    prom.catch(e => {
+        logger.error(`Failed to ensure outbound session in ${this._roomId}`, e);
+    });
+
     // _setupPromise resolves to `session` whether or not the share succeeds
     this._setupPromise = prom.then(returnSession, returnSession);
 
@@ -359,30 +398,26 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
 /**
  * @private
  *
+ * @param {boolean} sharedHistory
+ *
  * @return {module:crypto/algorithms/megolm.OutboundSessionInfo} session
  */
-MegolmEncryption.prototype._prepareNewSession = async function() {
+MegolmEncryption.prototype._prepareNewSession = async function(sharedHistory) {
     const sessionId = this._olmDevice.createOutboundGroupSession();
     const key = this._olmDevice.getOutboundGroupSessionKey(sessionId);
 
     await this._olmDevice.addInboundGroupSession(
         this._roomId, this._olmDevice.deviceCurve25519Key, [], sessionId,
-        key.key, {ed25519: this._olmDevice.deviceEd25519Key},
+        key.key, { ed25519: this._olmDevice.deviceEd25519Key }, false,
+        { sharedHistory: sharedHistory },
     );
 
-    if (this._crypto.backupInfo) {
-        // don't wait for it to complete
-        this._crypto.backupGroupSession(
-            this._roomId, this._olmDevice.deviceCurve25519Key, [],
-            sessionId, key.key,
-        ).catch((e) => {
-            // This throws if the upload failed, but this is fine
-            // since it will have written it to the db and will retry.
-            logger.log("Failed to back up megolm session", e);
-        });
-    }
+    // don't wait for it to complete
+    this._crypto._backupManager.backupGroupSession(
+        this._olmDevice.deviceCurve25519Key, sessionId,
+    );
 
-    return new OutboundSessionInfo(sessionId);
+    return new OutboundSessionInfo(sessionId, sharedHistory);
 };
 
 /**
@@ -416,7 +451,7 @@ MegolmEncryption.prototype._getDevicesWithoutSessions = function(
                 // no session with this device, probably because there
                 // were no one-time keys.
 
-                noOlmDevices.push({userId, deviceInfo});
+                noOlmDevices.push({ userId, deviceInfo });
                 delete sessionResults[deviceId];
 
                 // ensureOlmSessionsForUsers has already done the logging,
@@ -522,6 +557,33 @@ MegolmEncryption.prototype._encryptAndSendKeysToDevices = function(
     }
 
     return Promise.all(promises).then(() => {
+        // prune out any devices that encryptMessageForDevice could not encrypt for,
+        // in which case it will have just not added anything to the ciphertext object.
+        // There's no point sending messages to devices if we couldn't encrypt to them,
+        // since that's effectively a blank message.
+        for (const userId of Object.keys(contentMap)) {
+            for (const deviceId of Object.keys(contentMap[userId])) {
+                if (Object.keys(contentMap[userId][deviceId].ciphertext).length === 0) {
+                    logger.log(
+                        "No ciphertext for device " +
+                        userId + ":" + deviceId + ": pruning",
+                    );
+                    delete contentMap[userId][deviceId];
+                }
+            }
+            // No devices left for that user? Strip that too.
+            if (Object.keys(contentMap[userId]).length === 0) {
+                logger.log("Pruned all devices for user " + userId);
+                delete contentMap[userId];
+            }
+        }
+
+        // Is there anything left?
+        if (Object.keys(contentMap).length === 0) {
+            logger.log("No users left to send to: aborting");
+            return;
+        }
+
         return this._baseApis.sendToDevice("m.room.encrypted", contentMap).then(() => {
             // store that we successfully uploaded the keys of the current slice
             for (const userId of Object.keys(contentMap)) {
@@ -636,14 +698,15 @@ MegolmEncryption.prototype.reshareKeyWithDevice = async function(
     const payload = {
         type: "m.forwarded_room_key",
         content: {
-            algorithm: olmlib.MEGOLM_ALGORITHM,
-            room_id: this._roomId,
-            session_id: sessionId,
-            session_key: key.key,
-            chain_index: key.chain_index,
-            sender_key: senderKey,
-            sender_claimed_ed25519_key: key.sender_claimed_ed25519_key,
-            forwarding_curve25519_key_chain: key.forwarding_curve25519_key_chain,
+            "algorithm": olmlib.MEGOLM_ALGORITHM,
+            "room_id": this._roomId,
+            "session_id": sessionId,
+            "session_key": key.key,
+            "chain_index": key.chain_index,
+            "sender_key": senderKey,
+            "sender_claimed_ed25519_key": key.sender_claimed_ed25519_key,
+            "forwarding_curve25519_key_chain": key.forwarding_curve25519_key_chain,
+            "org.matrix.msc3061.shared_history": key.shared_history || false,
         },
     };
 
@@ -690,17 +753,25 @@ MegolmEncryption.prototype.reshareKeyWithDevice = async function(
  *
  * @param {Number} [otkTimeout] The timeout in milliseconds when requesting
  *     one-time keys for establishing new olm sessions.
+ *
+ * @param {Array} [failedServers] An array to fill with remote servers that
+ *     failed to respond to one-time-key requests.
  */
 MegolmEncryption.prototype._shareKeyWithDevices = async function(
-    session, key, payload, devicesByUser, errorDevices, otkTimeout,
+    session, key, payload, devicesByUser, errorDevices, otkTimeout, failedServers,
 ) {
+    logger.debug(`Ensuring Olm sessions for devices in ${this._roomId}`);
     const devicemap = await olmlib.ensureOlmSessionsForDevices(
-        this._olmDevice, this._baseApis, devicesByUser, otkTimeout,
+        this._olmDevice, this._baseApis, devicesByUser, otkTimeout, failedServers,
+        logger.withPrefix(`[${this._roomId}]`),
     );
+    logger.debug(`Ensured Olm sessions for devices in ${this._roomId}`);
 
     this._getDevicesWithoutSessions(devicemap, devicesByUser, errorDevices);
 
+    logger.debug(`Sharing keys with Olm sessions in ${this._roomId}`);
     await this._shareKeyWithOlmSessions(session, key, payload, devicemap);
+    logger.debug(`Shared keys with Olm sessions in ${this._roomId}`);
 };
 
 MegolmEncryption.prototype._shareKeyWithOlmSessions = async function(
@@ -709,21 +780,81 @@ MegolmEncryption.prototype._shareKeyWithOlmSessions = async function(
     const userDeviceMaps = this._splitDevices(devicemap);
 
     for (let i = 0; i < userDeviceMaps.length; i++) {
+        const taskDetail =
+            `megolm keys for ${session.sessionId} ` +
+            `in ${this._roomId} (slice ${i + 1}/${userDeviceMaps.length})`;
         try {
+            logger.debug(`Sharing ${taskDetail}`);
             await this._encryptAndSendKeysToDevices(
                 session, key.chain_index, userDeviceMaps[i], payload,
             );
-            logger.log(`Completed megolm keyshare for ${session.sessionId} `
-                + `in ${this._roomId} (slice ${i + 1}/${userDeviceMaps.length})`);
+            logger.debug(`Shared ${taskDetail}`);
         } catch (e) {
-            logger.log(`megolm keyshare for ${session.sessionId} in ${this._roomId} `
-                + `(slice ${i + 1}/${userDeviceMaps.length}) failed`);
-
+            logger.error(`Failed to share ${taskDetail}`);
             throw e;
         }
     }
 };
 
+/**
+ * Notify devices that we weren't able to create olm sessions.
+ *
+ * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ *
+ * @param {object} key
+ *
+ * @param {Array<object>} failedDevices the devices that we were unable to
+ *     create olm sessions for, as returned by _shareKeyWithDevices
+ */
+MegolmEncryption.prototype._notifyFailedOlmDevices = async function(
+    session, key, failedDevices,
+) {
+    logger.debug(
+        `Notifying ${failedDevices.length} devices we failed to ` +
+        `create Olm sessions in ${this._roomId}`,
+    );
+
+    // mark the devices that failed as "handled" because we don't want to try
+    // to claim a one-time-key for dead devices on every message.
+    for (const { userId, deviceInfo } of failedDevices) {
+        const deviceId = deviceInfo.deviceId;
+
+        session.markSharedWithDevice(
+            userId, deviceId, key.chain_index,
+        );
+    }
+
+    const filteredFailedDevices =
+          await this._olmDevice.filterOutNotifiedErrorDevices(
+              failedDevices,
+          );
+    logger.debug(
+        `Filtered down to ${filteredFailedDevices.length} error devices ` +
+        `in ${this._roomId}`,
+    );
+    const blockedMap = {};
+    for (const { userId, deviceInfo } of filteredFailedDevices) {
+        blockedMap[userId] = blockedMap[userId] || {};
+        // we use a similar format to what
+        // olmlib.ensureOlmSessionsForDevices returns, so that
+        // we can use the same function to split
+        blockedMap[userId][deviceInfo.deviceId] = {
+            device: {
+                code: "m.no_olm",
+                reason: WITHHELD_MESSAGES["m.no_olm"],
+                deviceInfo,
+            },
+        };
+    }
+
+    // send the notifications
+    await this._notifyBlockedDevices(session, blockedMap);
+    logger.debug(
+        `Notified ${filteredFailedDevices.length} devices we failed to ` +
+        `create Olm sessions in ${this._roomId}`,
+    );
+};
+
 /**
  * Notify blocked devices that they have been blocked.
  *
@@ -767,28 +898,45 @@ MegolmEncryption.prototype._notifyBlockedDevices = async function(
  * @param {module:models/room} room the room the event is in
  */
 MegolmEncryption.prototype.prepareToEncrypt = function(room) {
-    logger.debug(`Preparing to encrypt events for ${this._roomId}`);
-
     if (this.encryptionPreparation) {
         // We're already preparing something, so don't do anything else.
         // FIXME: check if we need to restart
         // (https://github.com/matrix-org/matrix-js-sdk/issues/1255)
+        const elapsedTime = Date.now() - this.encryptionPreparationMetadata.startTime;
+        logger.debug(
+            `Already started preparing to encrypt for ${this._roomId} ` +
+            `${elapsedTime} ms ago, skipping`,
+        );
         return;
     }
 
+    logger.debug(`Preparing to encrypt events for ${this._roomId}`);
+
+    this.encryptionPreparationMetadata = {
+        startTime: Date.now(),
+    };
     this.encryptionPreparation = (async () => {
-        const [devicesInRoom, blocked] = await this._getDevicesInRoom(room);
+        try {
+            logger.debug(`Getting devices in ${this._roomId}`);
+            const [devicesInRoom, blocked] = await this._getDevicesInRoom(room);
 
-        if (this._crypto.getGlobalErrorOnUnknownDevices()) {
-            // Drop unknown devices for now.  When the message gets sent, we'll
-            // throw an error, but we'll still be prepared to send to the known
-            // devices.
-            this._removeUnknownDevices(devicesInRoom);
+            if (this._crypto.getGlobalErrorOnUnknownDevices()) {
+                // Drop unknown devices for now.  When the message gets sent, we'll
+                // throw an error, but we'll still be prepared to send to the known
+                // devices.
+                this._removeUnknownDevices(devicesInRoom);
+            }
+
+            logger.debug(`Ensuring outbound session in ${this._roomId}`);
+            await this._ensureOutboundSession(room, devicesInRoom, blocked, true);
+
+            logger.debug(`Ready to encrypt events for ${this._roomId}`);
+        } catch (e) {
+            logger.error(`Failed to prepare to encrypt events for ${this._roomId}`, e);
+        } finally {
+            delete this.encryptionPreparationMetadata;
+            delete this.encryptionPreparation;
         }
-
-        await this._ensureOutboundSession(devicesInRoom, blocked);
-
-        delete this.encryptionPreparation;
     })();
 };
 
@@ -824,7 +972,7 @@ MegolmEncryption.prototype.encryptMessage = async function(room, eventType, cont
         this._checkForUnknownDevices(devicesInRoom);
     }
 
-    const session = await this._ensureOutboundSession(devicesInRoom, blocked);
+    const session = await this._ensureOutboundSession(room, devicesInRoom, blocked);
     const payloadJson = {
         room_id: this._roomId,
         type: eventType,
@@ -925,7 +1073,7 @@ MegolmEncryption.prototype._removeUnknownDevices = function(devicesInRoom) {
  */
 MegolmEncryption.prototype._getDevicesInRoom = async function(room) {
     const members = await room.getEncryptionTargetMembers();
-    const roomMembers = utils.map(members, function(u) {
+    const roomMembers = members.map(function(u) {
         return u.userId;
     });
 
@@ -940,7 +1088,7 @@ MegolmEncryption.prototype._getDevicesInRoom = async function(room) {
     // with them, which means that they will have announced any new devices via
     // device_lists in their /sync response.  This cache should then be maintained
     // using all the device_lists changes and left fields.
-    // See https://github.com/vector-im/riot-web/issues/2305 for details.
+    // See https://github.com/vector-im/element-web/issues/2305 for details.
     const devices = await this._crypto.downloadKeys(roomMembers, false);
     const blocked = {};
     // remove any blocked devices
@@ -955,8 +1103,10 @@ MegolmEncryption.prototype._getDevicesInRoom = async function(room) {
                 continue;
             }
 
+            const deviceTrust = this._crypto.checkDeviceTrust(userId, deviceId);
+
             if (userDevices[deviceId].isBlocked() ||
-                (userDevices[deviceId].isUnverified() && isBlacklisting)
+                (!deviceTrust.isVerified() && isBlacklisting)
             ) {
                 if (!blocked[userId]) {
                     blocked[userId] = {};
@@ -1032,7 +1182,7 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {
     //
     // then, if the key turns up while decryption is in progress (and
     // decryption fails), we will schedule a retry.
-    // (fixes https://github.com/vector-im/riot-web/issues/5001)
+    // (fixes https://github.com/vector-im/element-web/issues/5001)
     this._addEventToPendingList(event);
 
     let res;
@@ -1124,6 +1274,7 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {
         senderCurve25519Key: res.senderKey,
         claimedEd25519Key: res.keysClaimed.ed25519,
         forwardingCurve25519KeyChain: res.forwardingCurve25519KeyChain,
+        untrusted: res.untrusted,
     };
 };
 
@@ -1187,7 +1338,6 @@ MegolmDecryption.prototype._removeEventFromPendingList = function(event) {
     }
 };
 
-
 /**
  * @inheritdoc
  *
@@ -1217,7 +1367,7 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
     if (event.getType() == "m.forwarded_room_key") {
         exportFormat = true;
         forwardingKeyChain = content.forwarding_curve25519_key_chain;
-        if (!utils.isArray(forwardingKeyChain)) {
+        if (!Array.isArray(forwardingKeyChain)) {
             forwardingKeyChain = [];
         }
 
@@ -1246,11 +1396,14 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
         keysClaimed = event.getKeysClaimed();
     }
 
-    logger.log(`Received and adding key for megolm session ${senderKey}|${sessionId}`);
+    const extraSessionData = {};
+    if (content["org.matrix.msc3061.shared_history"]) {
+        extraSessionData.sharedHistory = true;
+    }
     return this._olmDevice.addInboundGroupSession(
         content.room_id, senderKey, forwardingKeyChain, sessionId,
         content.session_key, keysClaimed,
-        exportFormat,
+        exportFormat, extraSessionData,
     ).then(() => {
         // have another go at decrypting events sent with this session.
         this._retryDecryption(senderKey, sessionId)
@@ -1270,18 +1423,8 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
                 }
             });
     }).then(() => {
-        if (this._crypto.backupInfo) {
-            // don't wait for the keys to be backed up for the server
-            this._crypto.backupGroupSession(
-                content.room_id, senderKey, forwardingKeyChain,
-                content.session_id, content.session_key, keysClaimed,
-                exportFormat,
-            ).catch((e) => {
-                // This throws if the upload failed, but this is fine
-                // since it will have written it to the db and will retry.
-                logger.log("Failed to back up megolm session", e);
-            });
-        }
+        // don't wait for the keys to be backed up for the server
+        this._crypto._backupManager.backupGroupSession(senderKey, content.session_id);
     }).catch((e) => {
         logger.error(`Error handling m.room_key_event: ${e}`);
     });
@@ -1338,7 +1481,7 @@ MegolmDecryption.prototype.onRoomKeyWithheldEvent = async function(event) {
             }
         }
         await olmlib.ensureOlmSessionsForDevices(
-            this._olmDevice, this._baseApis, {[sender]: [device]}, false,
+            this._olmDevice, this._baseApis, { [sender]: [device] }, false,
         );
         const encryptedContent = {
             algorithm: olmlib.OLM_ALGORITHM,
@@ -1352,7 +1495,7 @@ MegolmDecryption.prototype.onRoomKeyWithheldEvent = async function(event) {
             this._olmDevice,
             sender,
             device,
-            {type: "m.dummy"},
+            { type: "m.dummy" },
         );
 
         await this._olmDevice.recordSessionProblem(senderKey, "no_olm", true);
@@ -1456,14 +1599,15 @@ MegolmDecryption.prototype._buildKeyForwardingMessage = async function(
     return {
         type: "m.forwarded_room_key",
         content: {
-            algorithm: olmlib.MEGOLM_ALGORITHM,
-            room_id: roomId,
-            sender_key: senderKey,
-            sender_claimed_ed25519_key: key.sender_claimed_ed25519_key,
-            session_id: sessionId,
-            session_key: key.key,
-            chain_index: key.chain_index,
-            forwarding_curve25519_key_chain: key.forwarding_curve25519_key_chain,
+            "algorithm": olmlib.MEGOLM_ALGORITHM,
+            "room_id": roomId,
+            "sender_key": senderKey,
+            "sender_claimed_ed25519_key": key.sender_claimed_ed25519_key,
+            "session_id": sessionId,
+            "session_key": key.key,
+            "chain_index": key.chain_index,
+            "forwarding_curve25519_key_chain": key.forwarding_curve25519_key_chain,
+            "org.matrix.msc3061.shared_history": key.shared_history || false,
         },
     };
 };
@@ -1472,8 +1616,18 @@ MegolmDecryption.prototype._buildKeyForwardingMessage = async function(
  * @inheritdoc
  *
  * @param {module:crypto/OlmDevice.MegolmSessionData} session
+ * @param {object} [opts={}] options for the import
+ * @param {boolean} [opts.untrusted] whether the key should be considered as untrusted
+ * @param {string} [opts.source] where the key came from
  */
-MegolmDecryption.prototype.importRoomKey = function(session) {
+MegolmDecryption.prototype.importRoomKey = function(session, opts = {}) {
+    const extraSessionData = {};
+    if (opts.untrusted) {
+        extraSessionData.untrusted = true;
+    }
+    if (session["org.matrix.msc3061.shared_history"]) {
+        extraSessionData.sharedHistory = true;
+    }
     return this._olmDevice.addInboundGroupSession(
         session.room_id,
         session.sender_key,
@@ -1482,17 +1636,12 @@ MegolmDecryption.prototype.importRoomKey = function(session) {
         session.session_key,
         session.sender_claimed_keys,
         true,
+        extraSessionData,
     ).then(() => {
-        if (this._crypto.backupInfo) {
+        if (opts.source !== "backup") {
             // don't wait for it to complete
-            this._crypto.backupGroupSession(
-                session.room_id,
-                session.sender_key,
-                session.forwarding_curve25519_key_chain,
-                session.session_id,
-                session.session_key,
-                session.sender_claimed_keys,
-                true,
+            this._crypto._backupManager.backupGroupSession(
+                session.sender_key, session.session_id,
             ).catch((e) => {
                 // This throws if the upload failed, but this is fine
                 // since it will have written it to the db and will retry.
@@ -1505,7 +1654,8 @@ MegolmDecryption.prototype.importRoomKey = function(session) {
 };
 
 /**
- * Have another go at decrypting events after we receive a key
+ * Have another go at decrypting events after we receive a key. Resolves once
+ * decryption has been re-attempted on all events.
  *
  * @private
  * @param {String} senderKey
@@ -1524,21 +1674,17 @@ MegolmDecryption.prototype._retryDecryption = async function(senderKey, sessionI
         return true;
     }
 
-    pending.delete(sessionId);
-    if (pending.size === 0) {
-        this._pendingEvents[senderKey];
-    }
+    logger.debug("Retrying decryption on events", [...pending]);
 
     await Promise.all([...pending].map(async (ev) => {
         try {
-            await ev.attemptDecryption(this._crypto);
+            await ev.attemptDecryption(this._crypto, { isRetry: true });
         } catch (e) {
             // don't die if something goes wrong
         }
     }));
 
-    // ev.attemptDecryption will re-add to this._pendingEvents if an event
-    // couldn't be decrypted
+    // If decrypted successfully, they'll have been removed from _pendingEvents
     return !((this._pendingEvents[senderKey] || {})[sessionId]);
 };
 
@@ -1563,6 +1709,80 @@ MegolmDecryption.prototype.retryDecryptionFromSender = async function(senderKey)
     return !this._pendingEvents[senderKey];
 };
 
+MegolmDecryption.prototype.sendSharedHistoryInboundSessions = async function(devicesByUser) {
+    await olmlib.ensureOlmSessionsForDevices(
+        this._olmDevice, this._baseApis, devicesByUser,
+    );
+
+    logger.log("sendSharedHistoryInboundSessions to users", Object.keys(devicesByUser));
+
+    const sharedHistorySessions =
+          await this._olmDevice.getSharedHistoryInboundGroupSessions(
+              this._roomId,
+          );
+    logger.log("shared-history sessions", sharedHistorySessions);
+    for (const [senderKey, sessionId] of sharedHistorySessions) {
+        const payload = await this._buildKeyForwardingMessage(
+            this._roomId, senderKey, sessionId,
+        );
+
+        const promises = [];
+        const contentMap = {};
+        for (const [userId, devices] of Object.entries(devicesByUser)) {
+            contentMap[userId] = {};
+            for (const deviceInfo of devices) {
+                const encryptedContent = {
+                    algorithm: olmlib.OLM_ALGORITHM,
+                    sender_key: this._olmDevice.deviceCurve25519Key,
+                    ciphertext: {},
+                };
+                contentMap[userId][deviceInfo.deviceId] = encryptedContent;
+                promises.push(
+                    olmlib.encryptMessageForDevice(
+                        encryptedContent.ciphertext,
+                        this._userId,
+                        this._deviceId,
+                        this._olmDevice,
+                        userId,
+                        deviceInfo,
+                        payload,
+                    ),
+                );
+            }
+        }
+        await Promise.all(promises);
+
+        // prune out any devices that encryptMessageForDevice could not encrypt for,
+        // in which case it will have just not added anything to the ciphertext object.
+        // There's no point sending messages to devices if we couldn't encrypt to them,
+        // since that's effectively a blank message.
+        for (const userId of Object.keys(contentMap)) {
+            for (const deviceId of Object.keys(contentMap[userId])) {
+                if (Object.keys(contentMap[userId][deviceId].ciphertext).length === 0) {
+                    logger.log(
+                        "No ciphertext for device " +
+                            userId + ":" + deviceId + ": pruning",
+                    );
+                    delete contentMap[userId][deviceId];
+                }
+            }
+            // No devices left for that user? Strip that too.
+            if (Object.keys(contentMap[userId]).length === 0) {
+                logger.log("Pruned all devices for user " + userId);
+                delete contentMap[userId];
+            }
+        }
+
+        // Is there anything left?
+        if (Object.keys(contentMap).length === 0) {
+            logger.log("No users left to send to: aborting");
+            return;
+        }
+
+        await this._baseApis.sendToDevice("m.room.encrypted", contentMap);
+    }
+};
+
 registerAlgorithm(
     olmlib.MEGOLM_ALGORITHM, MegolmEncryption, MegolmDecryption,
 );

src/crypto/algorithms/olm.js

@@ -20,11 +20,11 @@ limitations under the License.
  * @module crypto/algorithms/olm
  */
 
-import {logger} from '../../logger';
+import { logger } from '../../logger';
 import * as utils from "../../utils";
-import {polyfillSuper} from "../../utils";
+import { polyfillSuper } from "../../utils";
 import * as olmlib from "../olmlib";
-import {DeviceInfo} from "../deviceinfo";
+import { DeviceInfo } from "../deviceinfo";
 import {
     DecryptionAlgorithm,
     DecryptionError,
@@ -95,7 +95,7 @@ OlmEncryption.prototype.encryptMessage = async function(room, eventType, content
 
     const members = await room.getEncryptionTargetMembers();
 
-    const users = utils.map(members, function(u) {
+    const users = members.map(function(u) {
         return u.userId;
     });
 
@@ -264,6 +264,25 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
  */
 OlmDecryption.prototype._decryptMessage = async function(
     theirDeviceIdentityKey, message,
+) {
+    // This is a wrapper that serialises decryptions of prekey messages, because
+    // otherwise we race between deciding we have no active sessions for the message
+    // and creating a new one, which we can only do once because it removes the OTK.
+    if (message.type !== 0) {
+        // not a prekey message: we can safely just try & decrypt it
+        return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+    } else {
+        const myPromise = this._olmDevice._olmPrekeyPromise.then(() => {
+            return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+        });
+        // we want the error, but don't propagate it to the next decryption
+        this._olmDevice._olmPrekeyPromise = myPromise.catch(() => {});
+        return await myPromise;
+    }
+};
+
+OlmDecryption.prototype._reallyDecryptMessage = async function(
+    theirDeviceIdentityKey, message,
 ) {
     const sessionIds = await this._olmDevice.getSessionIdsForDevice(
         theirDeviceIdentityKey,
@@ -339,5 +358,4 @@ OlmDecryption.prototype._decryptMessage = async function(
     return res.payload;
 };
 
-
 registerAlgorithm(olmlib.OLM_ALGORITHM, OlmEncryption, OlmDecryption);

src/crypto/api.ts

@@ -0,0 +1,131 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { DeviceInfo } from "./deviceinfo";
+import { IKeyBackupVersion } from "./keybackup";
+import { ISecretStorageKeyInfo } from "../matrix";
+
+// TODO: Merge this with crypto.js once converted
+
+export enum CrossSigningKey {
+    Master = "master",
+    SelfSigning = "self_signing",
+    UserSigning = "user_signing",
+}
+
+export interface IEncryptedEventInfo {
+    /**
+     * whether the event is encrypted (if not encrypted, some of the other properties may not be set)
+     */
+    encrypted: boolean;
+
+    /**
+     * the sender's key
+     */
+    senderKey: string;
+
+    /**
+     * the algorithm used to encrypt the event
+     */
+    algorithm: string;
+
+    /**
+     * whether we can be sure that the owner of the senderKey sent the event
+     */
+    authenticated: boolean;
+
+    /**
+     * the sender's device information, if available
+     */
+    sender?: DeviceInfo;
+
+    /**
+     * if the event's ed25519 and curve25519 keys don't match (only meaningful if `sender` is set)
+     */
+    mismatchedSender: boolean;
+}
+
+export interface IRecoveryKey {
+    keyInfo: {
+        pubkey: Uint8Array;
+        passphrase?: {
+            algorithm: string;
+            iterations: number;
+            salt: string;
+        };
+    };
+    privateKey: Uint8Array;
+    encodedPrivateKey: string;
+}
+
+export interface ICreateSecretStorageOpts {
+    /**
+     * Function called to await a secret storage key creation flow.
+     * Returns:
+     *     {Promise<Object>} Object with public key metadata, encoded private
+     *     recovery key which should be disposed of after displaying to the user,
+     *     and raw private key to avoid round tripping if needed.
+     */
+    createSecretStorageKey?: () => Promise<IRecoveryKey>;
+
+    /**
+     * The current key backup object. If passed,
+     * the passphrase and recovery key from this backup will be used.
+     */
+    keyBackupInfo?: IKeyBackupVersion;
+
+    /**
+     * If true, a new key backup version will be
+     * created and the private key stored in the new SSSS store. Ignored if keyBackupInfo
+     * is supplied.
+     */
+    setupNewKeyBackup?: boolean;
+
+    /**
+     * Reset even if keys already exist.
+     */
+    setupNewSecretStorage?: boolean;
+
+    /**
+     * Function called to get the user's
+     * current key backup passphrase. Should return a promise that resolves with a Uint8Array
+     * containing the key, or rejects if the key cannot be obtained.
+     */
+    getKeyBackupPassphrase?: () => Promise<Uint8Array>;
+}
+
+export interface ISecretStorageKey {
+    keyId: string;
+    keyInfo: ISecretStorageKeyInfo;
+}
+
+export interface IAddSecretStorageKeyOpts {
+    // depends on algorithm
+    // TODO: Types
+}
+
+export interface IImportOpts {
+    stage: string; // TODO: Enum
+    successes: number;
+    failures: number;
+    total: number;
+}
+
+export interface IImportRoomKeysOpts {
+    progressCallback: (stage: IImportOpts) => void;
+    untrusted?: boolean;
+    source?: string; // TODO: Enum
+}

src/crypto/backup.ts

@@ -0,0 +1,651 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module crypto/backup
+ *
+ * Classes for dealing with key backup.
+ */
+
+import { MatrixClient } from "../client";
+import { logger } from "../logger";
+import { MEGOLM_ALGORITHM, verifySignature } from "./olmlib";
+import { DeviceInfo } from "./deviceinfo";
+import { DeviceTrustLevel } from './CrossSigning';
+import { keyFromPassphrase } from './key_passphrase';
+import { sleep } from "../utils";
+import { IndexedDBCryptoStore } from './store/indexeddb-crypto-store';
+import { encodeRecoveryKey } from './recoverykey';
+
+const KEY_BACKUP_KEYS_PER_REQUEST = 200;
+
+type AuthData = Record<string, any>;
+
+type BackupInfo = {
+    algorithm: string,
+    auth_data: AuthData, // eslint-disable-line camelcase
+    [properties: string]: any,
+};
+
+type SigInfo = {
+    deviceId: string,
+    valid?: boolean | null, // true: valid, false: invalid, null: cannot attempt validation
+    device?: DeviceInfo | null,
+    crossSigningId?: boolean,
+    deviceTrust?: DeviceTrustLevel,
+};
+
+type TrustInfo = {
+    usable: boolean, // is the backup trusted, true iff there is a sig that is valid & from a trusted device
+    sigs: SigInfo[],
+};
+
+/** A function used to get the secret key for a backup.
+ */
+type GetKey = () => Promise<Uint8Array>;
+
+interface BackupAlgorithmClass {
+    algorithmName: string;
+    // initialize from an existing backup
+    init(authData: AuthData, getKey: GetKey): Promise<BackupAlgorithm>;
+
+    // prepare a brand new backup
+    prepare(
+        key: string | Uint8Array | null,
+    ): Promise<[Uint8Array, AuthData]>;
+}
+
+interface BackupAlgorithm {
+    encryptSession(data: Record<string, any>): Promise<any>;
+    decryptSessions(ciphertexts: Record<string, any>): Promise<Record<string, any>[]>;
+    authData: AuthData;
+    keyMatches(key: Uint8Array): Promise<boolean>;
+    free(): void;
+}
+
+/**
+ * Manages the key backup.
+ */
+export class BackupManager {
+    private algorithm: BackupAlgorithm | undefined;
+    private backupInfo: BackupInfo | undefined; // The info dict from /room_keys/version
+    public checkedForBackup: boolean; // Have we checked the server for a backup we can use?
+    private sendingBackups: boolean; // Are we currently sending backups?
+    constructor(private readonly baseApis: MatrixClient, public readonly getKey: GetKey) {
+        this.checkedForBackup = false;
+        this.sendingBackups = false;
+    }
+
+    public get version(): string | undefined {
+        return this.backupInfo && this.backupInfo.version;
+    }
+
+    public static async makeAlgorithm(info: BackupInfo, getKey: GetKey): Promise<BackupAlgorithm> {
+        const Algorithm = algorithmsByName[info.algorithm];
+        if (!Algorithm) {
+            throw new Error("Unknown backup algorithm");
+        }
+        return await Algorithm.init(info.auth_data, getKey);
+    }
+
+    public async enableKeyBackup(info: BackupInfo): Promise<void> {
+        this.backupInfo = info;
+        if (this.algorithm) {
+            this.algorithm.free();
+        }
+
+        this.algorithm = await BackupManager.makeAlgorithm(info, this.getKey);
+
+        this.baseApis.emit('crypto.keyBackupStatus', true);
+
+        // There may be keys left over from a partially completed backup, so
+        // schedule a send to check.
+        this.scheduleKeyBackupSend();
+    }
+
+    /**
+     * Disable backing up of keys.
+     */
+    public disableKeyBackup(): void {
+        if (this.algorithm) {
+            this.algorithm.free();
+        }
+        this.algorithm = undefined;
+
+        this.backupInfo = undefined;
+
+        this.baseApis.emit('crypto.keyBackupStatus', false);
+    }
+
+    public getKeyBackupEnabled(): boolean | null {
+        if (!this.checkedForBackup) {
+            return null;
+        }
+        return Boolean(this.algorithm);
+    }
+
+    public async prepareKeyBackupVersion(
+        key?: string | Uint8Array | null,
+        algorithm?: string | undefined,
+    ): Promise<BackupInfo> {
+        const Algorithm = algorithm ? algorithmsByName[algorithm] : DefaultAlgorithm;
+        if (!Algorithm) {
+            throw new Error("Unknown backup algorithm");
+        }
+
+        const [privateKey, authData] = await Algorithm.prepare(key);
+        const recoveryKey = encodeRecoveryKey(privateKey);
+        return {
+            algorithm: Algorithm.algorithmName,
+            auth_data: authData,
+            recovery_key: recoveryKey,
+            privateKey,
+        };
+    }
+
+    public async createKeyBackupVersion(info: BackupInfo): Promise<void> {
+        this.algorithm = await BackupManager.makeAlgorithm(info, this.getKey);
+    }
+
+    /**
+     * Check the server for an active key backup and
+     * if one is present and has a valid signature from
+     * one of the user's verified devices, start backing up
+     * to it.
+     */
+    public async checkAndStart(): Promise<{backupInfo: BackupInfo, trustInfo: TrustInfo}> {
+        logger.log("Checking key backup status...");
+        if (this.baseApis.isGuest()) {
+            logger.log("Skipping key backup check since user is guest");
+            this.checkedForBackup = true;
+            return null;
+        }
+        let backupInfo: BackupInfo;
+        try {
+            backupInfo = await this.baseApis.getKeyBackupVersion();
+        } catch (e) {
+            logger.log("Error checking for active key backup", e);
+            if (e.httpStatus === 404) {
+                // 404 is returned when the key backup does not exist, so that
+                // counts as successfully checking.
+                this.checkedForBackup = true;
+            }
+            return null;
+        }
+        this.checkedForBackup = true;
+
+        const trustInfo = await this.isKeyBackupTrusted(backupInfo);
+
+        if (trustInfo.usable && !this.backupInfo) {
+            logger.log(
+                "Found usable key backup v" + backupInfo.version +
+                    ": enabling key backups",
+            );
+            await this.enableKeyBackup(backupInfo);
+        } else if (!trustInfo.usable && this.backupInfo) {
+            logger.log("No usable key backup: disabling key backup");
+            this.disableKeyBackup();
+        } else if (!trustInfo.usable && !this.backupInfo) {
+            logger.log("No usable key backup: not enabling key backup");
+        } else if (trustInfo.usable && this.backupInfo) {
+            // may not be the same version: if not, we should switch
+            if (backupInfo.version !== this.backupInfo.version) {
+                logger.log(
+                    "On backup version " + this.backupInfo.version + " but found " +
+                        "version " + backupInfo.version + ": switching.",
+                );
+                this.disableKeyBackup();
+                await this.enableKeyBackup(backupInfo);
+                // We're now using a new backup, so schedule all the keys we have to be
+                // uploaded to the new backup. This is a bit of a workaround to upload
+                // keys to a new backup in *most* cases, but it won't cover all cases
+                // because we don't remember what backup version we uploaded keys to:
+                // see https://github.com/vector-im/element-web/issues/14833
+                await this.scheduleAllGroupSessionsForBackup();
+            } else {
+                logger.log("Backup version " + backupInfo.version + " still current");
+            }
+        }
+
+        return { backupInfo, trustInfo };
+    }
+
+    /**
+     * Forces a re-check of the key backup and enables/disables it
+     * as appropriate.
+     *
+     * @return {Object} Object with backup info (as returned by
+     *     getKeyBackupVersion) in backupInfo and
+     *     trust information (as returned by isKeyBackupTrusted)
+     *     in trustInfo.
+     */
+    public async checkKeyBackup(): Promise<{backupInfo: BackupInfo, trustInfo: TrustInfo}> {
+        this.checkedForBackup = false;
+        return this.checkAndStart();
+    }
+
+    /**
+     * Check if the given backup info is trusted.
+     *
+     * @param {object} backupInfo key backup info dict from /room_keys/version
+     * @return {object} {
+     *     usable: [bool], // is the backup trusted, true iff there is a sig that is valid & from a trusted device
+     *     sigs: [
+     *         valid: [bool || null], // true: valid, false: invalid, null: cannot attempt validation
+     *         deviceId: [string],
+     *         device: [DeviceInfo || null],
+     *     ]
+     * }
+     */
+    public async isKeyBackupTrusted(backupInfo: BackupInfo): Promise<TrustInfo> {
+        const ret = {
+            usable: false,
+            trusted_locally: false,
+            sigs: [],
+        };
+
+        if (
+            !backupInfo ||
+                !backupInfo.algorithm ||
+                !backupInfo.auth_data ||
+                !backupInfo.auth_data.public_key ||
+                !backupInfo.auth_data.signatures
+        ) {
+            logger.info("Key backup is absent or missing required data");
+            return ret;
+        }
+
+        const trustedPubkey = this.baseApis.crypto._sessionStore.getLocalTrustedBackupPubKey();
+
+        if (backupInfo.auth_data.public_key === trustedPubkey) {
+            logger.info("Backup public key " + trustedPubkey + " is trusted locally");
+            ret.trusted_locally = true;
+        }
+
+        const mySigs = backupInfo.auth_data.signatures[this.baseApis.getUserId()] || [];
+
+        for (const keyId of Object.keys(mySigs)) {
+            const keyIdParts = keyId.split(':');
+            if (keyIdParts[0] !== 'ed25519') {
+                logger.log("Ignoring unknown signature type: " + keyIdParts[0]);
+                continue;
+            }
+            // Could be a cross-signing master key, but just say this is the device
+            // ID for backwards compat
+            const sigInfo: SigInfo = { deviceId: keyIdParts[1] };
+
+            // first check to see if it's from our cross-signing key
+            const crossSigningId = this.baseApis.crypto._crossSigningInfo.getId();
+            if (crossSigningId === sigInfo.deviceId) {
+                sigInfo.crossSigningId = true;
+                try {
+                    await verifySignature(
+                        this.baseApis.crypto._olmDevice,
+                        backupInfo.auth_data,
+                        this.baseApis.getUserId(),
+                        sigInfo.deviceId,
+                        crossSigningId,
+                    );
+                    sigInfo.valid = true;
+                } catch (e) {
+                    logger.warn(
+                        "Bad signature from cross signing key " + crossSigningId, e,
+                    );
+                    sigInfo.valid = false;
+                }
+                ret.sigs.push(sigInfo);
+                continue;
+            }
+
+            // Now look for a sig from a device
+            // At some point this can probably go away and we'll just support
+            // it being signed by the cross-signing master key
+            const device = this.baseApis.crypto._deviceList.getStoredDevice(
+                this.baseApis.getUserId(), sigInfo.deviceId,
+            );
+            if (device) {
+                sigInfo.device = device;
+                sigInfo.deviceTrust = await this.baseApis.checkDeviceTrust(
+                    this.baseApis.getUserId(), sigInfo.deviceId,
+                );
+                try {
+                    await verifySignature(
+                        this.baseApis.crypto._olmDevice,
+                        backupInfo.auth_data,
+                        this.baseApis.getUserId(),
+                        device.deviceId,
+                        device.getFingerprint(),
+                    );
+                    sigInfo.valid = true;
+                } catch (e) {
+                    logger.info(
+                        "Bad signature from key ID " + keyId + " userID " + this.baseApis.getUserId() +
+                            " device ID " + device.deviceId + " fingerprint: " +
+                            device.getFingerprint(), backupInfo.auth_data, e,
+                    );
+                    sigInfo.valid = false;
+                }
+            } else {
+                sigInfo.valid = null; // Can't determine validity because we don't have the signing device
+                logger.info("Ignoring signature from unknown key " + keyId);
+            }
+            ret.sigs.push(sigInfo);
+        }
+
+        ret.usable = ret.sigs.some((s) => {
+            return (
+                s.valid && (
+                    (s.device && s.deviceTrust.isVerified()) ||
+                        (s.crossSigningId)
+                )
+            );
+        });
+        ret.usable = ret.usable || ret.trusted_locally;
+        return ret;
+    }
+
+    /**
+     * Schedules sending all keys waiting to be sent to the backup, if not already
+     * scheduled. Retries if necessary.
+     *
+     * @param maxDelay Maximum delay to wait in ms. 0 means no delay.
+     */
+    public async scheduleKeyBackupSend(maxDelay = 10000): Promise<void> {
+        if (this.sendingBackups) return;
+
+        this.sendingBackups = true;
+
+        try {
+            // wait between 0 and `maxDelay` seconds, to avoid backup
+            // requests from different clients hitting the server all at
+            // the same time when a new key is sent
+            const delay = Math.random() * maxDelay;
+            await sleep(delay, undefined);
+            let numFailures = 0; // number of consecutive failures
+            for (;;) {
+                if (!this.algorithm) {
+                    return;
+                }
+                try {
+                    const numBackedUp =
+                        await this.backupPendingKeys(KEY_BACKUP_KEYS_PER_REQUEST);
+                    if (numBackedUp === 0) {
+                        // no sessions left needing backup: we're done
+                        return;
+                    }
+                    numFailures = 0;
+                } catch (err) {
+                    numFailures++;
+                    logger.log("Key backup request failed", err);
+                    if (err.data) {
+                        if (
+                            err.data.errcode == 'M_NOT_FOUND' ||
+                                err.data.errcode == 'M_WRONG_ROOM_KEYS_VERSION'
+                        ) {
+                            // Re-check key backup status on error, so we can be
+                            // sure to present the current situation when asked.
+                            await this.checkKeyBackup();
+                            // Backup version has changed or this backup version
+                            // has been deleted
+                            this.baseApis.crypto.emit("crypto.keyBackupFailed", err.data.errcode);
+                            throw err;
+                        }
+                    }
+                }
+                if (numFailures) {
+                    // exponential backoff if we have failures
+                    await sleep(1000 * Math.pow(2, Math.min(numFailures - 1, 4)), undefined);
+                }
+            }
+        } finally {
+            this.sendingBackups = false;
+        }
+    }
+
+    /**
+     * Take some e2e keys waiting to be backed up and send them
+     * to the backup.
+     *
+     * @param {integer} limit Maximum number of keys to back up
+     * @returns {integer} Number of sessions backed up
+     */
+    private async backupPendingKeys(limit: number): Promise<number> {
+        const sessions = await this.baseApis.crypto._cryptoStore.getSessionsNeedingBackup(limit);
+        if (!sessions.length) {
+            return 0;
+        }
+
+        let remaining = await this.baseApis.crypto._cryptoStore.countSessionsNeedingBackup();
+        this.baseApis.crypto.emit("crypto.keyBackupSessionsRemaining", remaining);
+
+        const data = {};
+        for (const session of sessions) {
+            const roomId = session.sessionData.room_id;
+            if (data[roomId] === undefined) {
+                data[roomId] = { sessions: {} };
+            }
+
+            const sessionData = await this.baseApis.crypto._olmDevice.exportInboundGroupSession(
+                session.senderKey, session.sessionId, session.sessionData,
+            );
+            sessionData.algorithm = MEGOLM_ALGORITHM;
+
+            const forwardedCount =
+                (sessionData.forwarding_curve25519_key_chain || []).length;
+
+            const userId = this.baseApis.crypto._deviceList.getUserByIdentityKey(
+                MEGOLM_ALGORITHM, session.senderKey,
+            );
+            const device = this.baseApis.crypto._deviceList.getDeviceByIdentityKey(
+                MEGOLM_ALGORITHM, session.senderKey,
+            );
+            const verified = this.baseApis.crypto._checkDeviceInfoTrust(userId, device).isVerified();
+
+            data[roomId]['sessions'][session.sessionId] = {
+                first_message_index: sessionData.first_known_index,
+                forwarded_count: forwardedCount,
+                is_verified: verified,
+                session_data: await this.algorithm.encryptSession(sessionData),
+            };
+        }
+
+        await this.baseApis.sendKeyBackup(
+            undefined, undefined, this.backupInfo.version,
+            { rooms: data },
+        );
+
+        await this.baseApis.crypto._cryptoStore.unmarkSessionsNeedingBackup(sessions);
+        remaining = await this.baseApis.crypto._cryptoStore.countSessionsNeedingBackup();
+        this.baseApis.crypto.emit("crypto.keyBackupSessionsRemaining", remaining);
+
+        return sessions.length;
+    }
+
+    public async backupGroupSession(
+        senderKey: string, sessionId: string,
+    ): Promise<void> {
+        await this.baseApis.crypto._cryptoStore.markSessionsNeedingBackup([{
+            senderKey: senderKey,
+            sessionId: sessionId,
+        }]);
+
+        if (this.backupInfo) {
+            // don't wait for this to complete: it will delay so
+            // happens in the background
+            this.scheduleKeyBackupSend();
+        }
+        // if this.backupInfo is not set, then the keys will be backed up when
+        // this.enableKeyBackup is called
+    }
+
+    /**
+     * Marks all group sessions as needing to be backed up and schedules them to
+     * upload in the background as soon as possible.
+     */
+    public async scheduleAllGroupSessionsForBackup(): Promise<void> {
+        await this.flagAllGroupSessionsForBackup();
+
+        // Schedule keys to upload in the background as soon as possible.
+        this.scheduleKeyBackupSend(0 /* maxDelay */);
+    }
+
+    /**
+     * Marks all group sessions as needing to be backed up without scheduling
+     * them to upload in the background.
+     * @returns {Promise<int>} Resolves to the number of sessions now requiring a backup
+     *     (which will be equal to the number of sessions in the store).
+     */
+    public async flagAllGroupSessionsForBackup(): Promise<number> {
+        await this.baseApis.crypto._cryptoStore.doTxn(
+            'readwrite',
+            [
+                IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+                IndexedDBCryptoStore.STORE_BACKUP,
+            ],
+            (txn) => {
+                this.baseApis.crypto._cryptoStore.getAllEndToEndInboundGroupSessions(txn, (session) => {
+                    if (session !== null) {
+                        this.baseApis.crypto._cryptoStore.markSessionsNeedingBackup([session], txn);
+                    }
+                });
+            },
+        );
+
+        const remaining = await this.baseApis.crypto._cryptoStore.countSessionsNeedingBackup();
+        this.baseApis.emit("crypto.keyBackupSessionsRemaining", remaining);
+        return remaining;
+    }
+
+    /**
+     * Counts the number of end to end session keys that are waiting to be backed up
+     * @returns {Promise<int>} Resolves to the number of sessions requiring backup
+     */
+    public countSessionsNeedingBackup(): Promise<number> {
+        return this.baseApis.crypto._cryptoStore.countSessionsNeedingBackup();
+    }
+}
+
+export class Curve25519 implements BackupAlgorithm {
+    public static algorithmName = "m.megolm_backup.v1.curve25519-aes-sha2";
+
+    constructor(
+        public authData: AuthData,
+        private publicKey: any, // FIXME: PkEncryption
+        private getKey: () => Promise<Uint8Array>,
+    ) {}
+
+    public static async init(
+        authData: AuthData,
+        getKey: () => Promise<Uint8Array>,
+    ): Promise<Curve25519> {
+        if (!authData || !authData.public_key) {
+            throw new Error("auth_data missing required information");
+        }
+        const publicKey = new global.Olm.PkEncryption();
+        publicKey.set_recipient_key(authData.public_key);
+        return new Curve25519(authData, publicKey, getKey);
+    }
+
+    public static async prepare(
+        key: string | Uint8Array | null,
+    ): Promise<[Uint8Array, AuthData]> {
+        const decryption = new global.Olm.PkDecryption();
+        try {
+            const authData: AuthData = {};
+            if (!key) {
+                authData.public_key = decryption.generate_key();
+            } else if (key instanceof Uint8Array) {
+                authData.public_key = decryption.init_with_private_key(key);
+            } else {
+                const derivation = await keyFromPassphrase(key);
+                authData.private_key_salt = derivation.salt;
+                authData.private_key_iterations = derivation.iterations;
+                authData.public_key = decryption.init_with_private_key(derivation.key);
+            }
+            const publicKey = new global.Olm.PkEncryption();
+            publicKey.set_recipient_key(authData.public_key);
+
+            return [
+                decryption.get_private_key(),
+                authData,
+            ];
+        } finally {
+            decryption.free();
+        }
+    }
+
+    public async encryptSession(data: Record<string, any>): Promise<any> {
+        const plainText: Record<string, any> = Object.assign({}, data);
+        delete plainText.session_id;
+        delete plainText.room_id;
+        delete plainText.first_known_index;
+        return this.publicKey.encrypt(JSON.stringify(plainText));
+    }
+
+    public async decryptSessions(sessions: Record<string, Record<string, any>>): Promise<Record<string, any>[]> {
+        const privKey = await this.getKey();
+        const decryption = new global.Olm.PkDecryption();
+        try {
+            const backupPubKey = decryption.init_with_private_key(privKey);
+
+            if (backupPubKey !== this.authData.public_key) {
+                // eslint-disable-next-line no-throw-literal
+                throw { errcode: MatrixClient.RESTORE_BACKUP_ERROR_BAD_KEY };
+            }
+
+            const keys = [];
+
+            for (const [sessionId, sessionData] of Object.entries(sessions)) {
+                try {
+                    const decrypted = JSON.parse(decryption.decrypt(
+                        sessionData.session_data.ephemeral,
+                        sessionData.session_data.mac,
+                        sessionData.session_data.ciphertext,
+                    ));
+                    decrypted.session_id = sessionId;
+                    keys.push(decrypted);
+                } catch (e) {
+                    logger.log("Failed to decrypt megolm session from backup", e, sessionData);
+                }
+            }
+            return keys;
+        } finally {
+            decryption.free();
+        }
+    }
+
+    public async keyMatches(key: Uint8Array): Promise<boolean> {
+        const decryption = new global.Olm.PkDecryption();
+        let pubKey;
+        try {
+            pubKey = decryption.init_with_private_key(key);
+        } finally {
+            decryption.free();
+        }
+
+        return pubKey === this.authData.public_key;
+    }
+
+    public free(): void {
+        this.publicKey.free();
+    }
+}
+
+export const algorithmsByName: Record<string, BackupAlgorithmClass> = {
+    [Curve25519.algorithmName]: Curve25519,
+};
+
+export const DefaultAlgorithm: BackupAlgorithmClass = Curve25519;

src/crypto/dehydration.ts

@@ -0,0 +1,301 @@
+/*
+Copyright 2020-2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { decodeBase64, encodeBase64 } from './olmlib';
+import { IndexedDBCryptoStore } from '../crypto/store/indexeddb-crypto-store';
+import { decryptAES, encryptAES } from './aes';
+import anotherjson from "another-json";
+import { logger } from '../logger';
+import { ISecretStorageKeyInfo } from "../matrix";
+
+// FIXME: these types should eventually go in a different file
+type Signatures = Record<string, Record<string, string>>;
+
+export interface IDehydratedDevice {
+    device_id: string; // eslint-disable-line camelcase
+    device_data: ISecretStorageKeyInfo & { // eslint-disable-line camelcase
+        algorithm: string;
+        account: string; // pickle
+    };
+}
+
+export interface IDehydratedDeviceKeyInfo {
+    passphrase?: string;
+}
+
+interface DeviceKeys {
+    algorithms: Array<string>;
+    device_id: string; // eslint-disable-line camelcase
+    user_id: string; // eslint-disable-line camelcase
+    keys: Record<string, string>;
+    signatures?: Signatures;
+}
+
+interface OneTimeKey {
+    key: string;
+    fallback?: boolean;
+    signatures?: Signatures;
+}
+
+export const DEHYDRATION_ALGORITHM = "org.matrix.msc2697.v1.olm.libolm_pickle";
+
+const oneweek = 7 * 24 * 60 * 60 * 1000;
+
+export class DehydrationManager {
+    private inProgress = false;
+    private timeoutId: any;
+    private key: Uint8Array;
+    private keyInfo: {[props: string]: any};
+    private deviceDisplayName: string;
+    constructor(private crypto) {
+        this.getDehydrationKeyFromCache();
+    }
+    async getDehydrationKeyFromCache(): Promise<void> {
+        return await this.crypto._cryptoStore.doTxn(
+            'readonly',
+            [IndexedDBCryptoStore.STORE_ACCOUNT],
+            (txn) => {
+                this.crypto._cryptoStore.getSecretStorePrivateKey(
+                    txn,
+                    async (result) => {
+                        if (result) {
+                            const { key, keyInfo, deviceDisplayName, time } = result;
+                            const pickleKey = Buffer.from(this.crypto._olmDevice._pickleKey);
+                            const decrypted = await decryptAES(key, pickleKey, DEHYDRATION_ALGORITHM);
+                            this.key = decodeBase64(decrypted);
+                            this.keyInfo = keyInfo;
+                            this.deviceDisplayName = deviceDisplayName;
+                            const now = Date.now();
+                            const delay = Math.max(1, time + oneweek - now);
+                            this.timeoutId = global.setTimeout(
+                                this.dehydrateDevice.bind(this), delay,
+                            );
+                        }
+                    },
+                    "dehydration",
+                );
+            },
+        );
+    }
+
+    /** set the key, and queue periodic dehydration to the server in the background */
+    async setKeyAndQueueDehydration(
+        key: Uint8Array, keyInfo: {[props: string]: any} = {},
+        deviceDisplayName: string = undefined,
+    ): Promise<void> {
+        const matches = await this.setKey(key, keyInfo, deviceDisplayName);
+        if (!matches) {
+            // start dehydration in the background
+            this.dehydrateDevice();
+        }
+    }
+
+    async setKey(
+        key: Uint8Array, keyInfo: {[props: string]: any} = {},
+        deviceDisplayName: string = undefined,
+    ): Promise<boolean> {
+        if (!key) {
+            // unsetting the key -- cancel any pending dehydration task
+            if (this.timeoutId) {
+                global.clearTimeout(this.timeoutId);
+                this.timeoutId = undefined;
+            }
+            // clear storage
+            await this.crypto._cryptoStore.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    this.crypto._cryptoStore.storeSecretStorePrivateKey(
+                        txn, "dehydration", null,
+                    );
+                },
+            );
+            this.key = undefined;
+            this.keyInfo = undefined;
+            return;
+        }
+
+        // Check to see if it's the same key as before.  If it's different,
+        // dehydrate a new device.  If it's the same, we can keep the same
+        // device.  (Assume that keyInfo and deviceDisplayName will be the
+        // same if the key is the same.)
+        let matches: boolean = this.key && key.length == this.key.length;
+        for (let i = 0; matches && i < key.length; i++) {
+            if (key[i] != this.key[i]) {
+                matches = false;
+            }
+        }
+        if (!matches) {
+            this.key = key;
+            this.keyInfo = keyInfo;
+            this.deviceDisplayName = deviceDisplayName;
+        }
+        return matches;
+    }
+
+    /** returns the device id of the newly created dehydrated device */
+    async dehydrateDevice(): Promise<string> {
+        if (this.inProgress) {
+            logger.log("Dehydration already in progress -- not starting new dehydration");
+            return;
+        }
+        this.inProgress = true;
+        if (this.timeoutId) {
+            global.clearTimeout(this.timeoutId);
+            this.timeoutId = undefined;
+        }
+        try {
+            const pickleKey = Buffer.from(this.crypto._olmDevice._pickleKey);
+
+            // update the crypto store with the timestamp
+            const key = await encryptAES(encodeBase64(this.key), pickleKey, DEHYDRATION_ALGORITHM);
+            await this.crypto._cryptoStore.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    this.crypto._cryptoStore.storeSecretStorePrivateKey(
+                        txn, "dehydration",
+                        {
+                            keyInfo: this.keyInfo,
+                            key,
+                            deviceDisplayName: this.deviceDisplayName,
+                            time: Date.now(),
+                        },
+                    );
+                },
+            );
+            logger.log("Attempting to dehydrate device");
+
+            logger.log("Creating account");
+            // create the account and all the necessary keys
+            const account = new global.Olm.Account();
+            account.create();
+            const e2eKeys = JSON.parse(account.identity_keys());
+
+            const maxKeys = account.max_number_of_one_time_keys();
+            // FIXME: generate in small batches?
+            account.generate_one_time_keys(maxKeys / 2);
+            account.generate_fallback_key();
+            const otks: Record<string, string> = JSON.parse(account.one_time_keys());
+            const fallbacks: Record<string, string> = JSON.parse(account.fallback_key());
+            account.mark_keys_as_published();
+
+            // dehydrate the account and store it on the server
+            const pickledAccount = account.pickle(new Uint8Array(this.key));
+
+            const deviceData: {[props: string]: any} = {
+                algorithm: DEHYDRATION_ALGORITHM,
+                account: pickledAccount,
+            };
+            if (this.keyInfo.passphrase) {
+                deviceData.passphrase = this.keyInfo.passphrase;
+            }
+
+            logger.log("Uploading account to server");
+            const dehydrateResult = await this.crypto._baseApis.http.authedRequest(
+                undefined,
+                "PUT",
+                "/dehydrated_device",
+                undefined,
+                {
+                    device_data: deviceData,
+                    initial_device_display_name: this.deviceDisplayName,
+                },
+                {
+                    prefix: "/_matrix/client/unstable/org.matrix.msc2697.v2",
+                },
+            );
+
+            // send the keys to the server
+            const deviceId = dehydrateResult.device_id;
+            logger.log("Preparing device keys", deviceId);
+            const deviceKeys: DeviceKeys = {
+                algorithms: this.crypto._supportedAlgorithms,
+                device_id: deviceId,
+                user_id: this.crypto._userId,
+                keys: {
+                    [`ed25519:${deviceId}`]: e2eKeys.ed25519,
+                    [`curve25519:${deviceId}`]: e2eKeys.curve25519,
+                },
+            };
+            const deviceSignature = account.sign(anotherjson.stringify(deviceKeys));
+            deviceKeys.signatures = {
+                [this.crypto._userId]: {
+                    [`ed25519:${deviceId}`]: deviceSignature,
+                },
+            };
+            if (this.crypto._crossSigningInfo.getId("self_signing")) {
+                await this.crypto._crossSigningInfo.signObject(deviceKeys, "self_signing");
+            }
+
+            logger.log("Preparing one-time keys");
+            const oneTimeKeys = {};
+            for (const [keyId, key] of Object.entries(otks.curve25519)) {
+                const k: OneTimeKey = { key };
+                const signature = account.sign(anotherjson.stringify(k));
+                k.signatures = {
+                    [this.crypto._userId]: {
+                        [`ed25519:${deviceId}`]: signature,
+                    },
+                };
+                oneTimeKeys[`signed_curve25519:${keyId}`] = k;
+            }
+
+            logger.log("Preparing fallback keys");
+            const fallbackKeys = {};
+            for (const [keyId, key] of Object.entries(fallbacks.curve25519)) {
+                const k: OneTimeKey = { key, fallback: true };
+                const signature = account.sign(anotherjson.stringify(k));
+                k.signatures = {
+                    [this.crypto._userId]: {
+                        [`ed25519:${deviceId}`]: signature,
+                    },
+                };
+                fallbackKeys[`signed_curve25519:${keyId}`] = k;
+            }
+
+            logger.log("Uploading keys to server");
+            await this.crypto._baseApis.http.authedRequest(
+                undefined,
+                "POST",
+                "/keys/upload/" + encodeURI(deviceId),
+                undefined,
+                {
+                    "device_keys": deviceKeys,
+                    "one_time_keys": oneTimeKeys,
+                    "org.matrix.msc2732.fallback_keys": fallbackKeys,
+                },
+            );
+            logger.log("Done dehydrating");
+
+            // dehydrate again in a week
+            this.timeoutId = global.setTimeout(
+                this.dehydrateDevice.bind(this), oneweek,
+            );
+
+            return deviceId;
+        } finally {
+            this.inProgress = false;
+        }
+    }
+
+    private stop() {
+        if (this.timeoutId) {
+            global.clearTimeout(this.timeoutId);
+            this.timeoutId = undefined;
+        }
+    }
+}