v9.6.0-rc.1

[Release] Fix compatibility with v0 calls

.eslintrc.js

@@ -1,71 +1,15 @@
 module.exports = {
-    parser: "babel-eslint", // now needed for class properties
-    parserOptions: {
-        sourceType: "module",
-        ecmaFeatures: {
-        }
-    },
+    extends: ["matrix-org"],
+    plugins: [
+        "babel",
+    ],
     env: {
         browser: true,
         node: true,
-
-        // babel's transform-runtime converts references to ES6 globals such as
-        // Promise and Map to core-js polyfills, so we can use ES6 globals.
-        es6: true,
-        jest: true,
     },
-    extends: ["eslint:recommended", "google"],
-    plugins: [
-        "babel",
-        "jest",
-    ],
+
     rules: {
-        // rules we've always adhered to or now do
-        "max-len": ["error", {
-            code: 90,
-            ignoreComments: true,
-        }],
-        curly: ["error", "multi-line"],
-        "prefer-const": ["error"],
-        "comma-dangle": ["error", {
-            arrays: "always-multiline",
-            objects: "always-multiline",
-            imports: "always-multiline",
-            exports: "always-multiline",
-            functions: "always-multiline",
-        }],
-
-        // loosen jsdoc requirements a little
-        "require-jsdoc": ["error", {
-            require: {
-                FunctionDeclaration: false,
-            }
-        }],
-        "valid-jsdoc": ["error", {
-            requireParamDescription: false,
-            requireReturn: false,
-            requireReturnDescription: false,
-        }],
-
-        // rules we do not want from eslint-recommended
-        "no-console": ["off"],
-        "no-constant-condition": ["off"],
-        "no-empty": ["error", { "allowEmptyCatch": true }],
-
-        // rules we do not want from the google styleguide
-        "object-curly-spacing": ["off"],
-        "spaced-comment": ["off"],
-        "guard-for-in": ["off"],
-
-        // in principle we prefer single quotes, but life is too short
-        quotes: ["off"],
-
-        // rules we'd ideally like to adhere to, but the current
-        // code does not (in most cases because it's still ES5)
-        // we set these to warnings, and assert that the number
-        // of warnings doesn't exceed a given threshold
         "no-var": ["warn"],
-        "brace-style": ["warn", "1tbs", {"allowSingleLine": true}],
         "prefer-rest-params": ["warn"],
         "prefer-spread": ["warn"],
         "one-var": ["warn"],
@@ -79,10 +23,23 @@ module.exports = {
             "asyncArrow": "always",
         }],
         "arrow-parens": "off",
-
-        // eslint's built in no-invalid-this rule breaks with class properties
-        "no-invalid-this": "off",
-        // so we replace it with a version that is class property aware
-        "babel/no-invalid-this": "error",
-    }
-}
+        "prefer-promise-reject-errors": "off",
+        "quotes": "off",
+        "indent": "off",
+        "no-constant-condition": "off",
+        "no-async-promise-executor": "off",
+        // We use a `logger` intermediary module
+        "no-console": "error",
+    },
+    overrides: [{
+        "files": ["src/**/*.ts"],
+        "extends": ["matrix-org/ts"],
+        "rules": {
+            // We're okay being explicit at the moment
+            "@typescript-eslint/no-empty-interface": "off",
+            // While we're converting to ts we make heavy use of this
+            "@typescript-eslint/no-explicit-any": "off",
+            "quotes": "off",
+        },
+    }],
+};

.gitignore

@@ -12,7 +12,6 @@ lib-cov
 out
 /dist
 /lib
-/specbuild
 
 # version file and tarball created by `npm pack` / `yarn pack`
 /git-revision.txt

CHANGELOG.md

@@ -1,3 +1,631 @@
+Changes in [9.6.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.6.0-rc.1) (2021-01-29)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.5.1...v9.6.0-rc.1)
+
+ * Add support for getting call stats
+   [\#1584](https://github.com/matrix-org/matrix-js-sdk/pull/1584)
+ * Fix compatibility with v0 calls
+   [\#1583](https://github.com/matrix-org/matrix-js-sdk/pull/1583)
+ * Upgrade deps 2021-01
+   [\#1582](https://github.com/matrix-org/matrix-js-sdk/pull/1582)
+ * Log the call ID when logging that we've received VoIP events
+   [\#1581](https://github.com/matrix-org/matrix-js-sdk/pull/1581)
+ * Fix extra negotiate message in Firefox
+   [\#1579](https://github.com/matrix-org/matrix-js-sdk/pull/1579)
+ * Add debug logs to encryption prep
+   [\#1580](https://github.com/matrix-org/matrix-js-sdk/pull/1580)
+ * Expose getPresence endpoint
+   [\#1578](https://github.com/matrix-org/matrix-js-sdk/pull/1578)
+ * Queue keys for backup even if backup isn't enabled yet
+   [\#1577](https://github.com/matrix-org/matrix-js-sdk/pull/1577)
+ * Stop retrying TURN access when forbidden
+   [\#1576](https://github.com/matrix-org/matrix-js-sdk/pull/1576)
+ * Add DTMF sending support
+   [\#1573](https://github.com/matrix-org/matrix-js-sdk/pull/1573)
+
+Changes in [9.5.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.5.1) (2021-01-26)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.5.0...v9.5.1)
+
+ * [Release] Fix compatibility with v0 calls
+   [\#1585](https://github.com/matrix-org/matrix-js-sdk/pull/1585)
+
+Changes in [9.5.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.5.0) (2021-01-18)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.5.0-rc.1...v9.5.0)
+
+ * No changes since rc.1
+
+Changes in [9.5.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.5.0-rc.1) (2021-01-13)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.4.1...v9.5.0-rc.1)
+
+ * Don't log if no WebRTC
+   [\#1574](https://github.com/matrix-org/matrix-js-sdk/pull/1574)
+ * Add _unstable_getSharedRooms
+   [\#1417](https://github.com/matrix-org/matrix-js-sdk/pull/1417)
+ * Bump node-notifier from 8.0.0 to 8.0.1
+   [\#1568](https://github.com/matrix-org/matrix-js-sdk/pull/1568)
+ * Ignore party ID if opponent is v0
+   [\#1567](https://github.com/matrix-org/matrix-js-sdk/pull/1567)
+ * Basic call transfer initiation support
+   [\#1566](https://github.com/matrix-org/matrix-js-sdk/pull/1566)
+ * Room version 6 is now a thing
+   [\#1572](https://github.com/matrix-org/matrix-js-sdk/pull/1572)
+ * Store keys with same index but better trust level
+   [\#1571](https://github.com/matrix-org/matrix-js-sdk/pull/1571)
+ * Use TypeScript source for development, swap to build during release
+   [\#1561](https://github.com/matrix-org/matrix-js-sdk/pull/1561)
+ * Revert "Ignore party ID if opponent is v0"
+   [\#1565](https://github.com/matrix-org/matrix-js-sdk/pull/1565)
+ * Basic call transfer initiation support
+   [\#1558](https://github.com/matrix-org/matrix-js-sdk/pull/1558)
+ * Ignore party ID if opponent is v0
+   [\#1559](https://github.com/matrix-org/matrix-js-sdk/pull/1559)
+ * Honour a call reject event from another of our own devices
+   [\#1562](https://github.com/matrix-org/matrix-js-sdk/pull/1562)
+
+Changes in [9.4.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.4.1) (2020-12-21)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.4.0...v9.4.1)
+
+ * Further script tweaks to get all layers building again
+
+Changes in [9.4.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.4.0) (2020-12-21)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.4.0-rc.2...v9.4.0)
+
+ * Revert `postinstall` script change, causes issues for other layers
+
+Changes in [9.4.0-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.4.0-rc.2) (2020-12-16)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.4.0-rc.1...v9.4.0-rc.2)
+
+ * Remove `postinstall` script which also runs as a dependency
+   [\#1560](https://github.com/matrix-org/matrix-js-sdk/pull/1560)
+
+Changes in [9.4.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.4.0-rc.1) (2020-12-16)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.3.0...v9.4.0-rc.1)
+
+ * Fixes to support line 1 / 2
+   [\#1553](https://github.com/matrix-org/matrix-js-sdk/pull/1553)
+ * Add API for listening to remote hold status, advertise VoIP V1
+   [\#1549](https://github.com/matrix-org/matrix-js-sdk/pull/1549)
+ * A hangup from another client is still valid
+   [\#1555](https://github.com/matrix-org/matrix-js-sdk/pull/1555)
+ * Remove temporary build step for tests
+   [\#1554](https://github.com/matrix-org/matrix-js-sdk/pull/1554)
+ * Move browser build steps to prepublish only
+   [\#1552](https://github.com/matrix-org/matrix-js-sdk/pull/1552)
+ * Extend getSsoLoginUrl for MSC2858
+   [\#1541](https://github.com/matrix-org/matrix-js-sdk/pull/1541)
+
+Changes in [9.3.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.3.0) (2020-12-07)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.3.0-rc.1...v9.3.0)
+
+* No changes since rc.1
+
+Changes in [9.3.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.3.0-rc.1) (2020-12-02)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.2.0...v9.3.0-rc.1)
+
+ * Export CallError
+   [\#1551](https://github.com/matrix-org/matrix-js-sdk/pull/1551)
+ * Upgrade dependencies
+   [\#1550](https://github.com/matrix-org/matrix-js-sdk/pull/1550)
+ * Don't log error when environment does not support WebRTC
+   [\#1547](https://github.com/matrix-org/matrix-js-sdk/pull/1547)
+ * Fix dehydration method name
+   [\#1544](https://github.com/matrix-org/matrix-js-sdk/pull/1544)
+
+Changes in [9.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.2.0) (2020-11-23)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.2.0-rc.1...v9.2.0)
+
+ * [Release] Fix dehydration method name
+   [\#1545](https://github.com/matrix-org/matrix-js-sdk/pull/1545)
+
+Changes in [9.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.2.0-rc.1) (2020-11-18)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.1.0...v9.2.0-rc.1)
+
+ * Implement call holding functionality
+   [\#1532](https://github.com/matrix-org/matrix-js-sdk/pull/1532)
+ * Support awaitable one-time dehydration
+   [\#1537](https://github.com/matrix-org/matrix-js-sdk/pull/1537)
+ * Client set profile methods update own user
+   [\#1534](https://github.com/matrix-org/matrix-js-sdk/pull/1534)
+
+Changes in [9.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.1.0) (2020-11-09)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.1.0-rc.1...v9.1.0)
+
+* No changes since rc.1
+
+Changes in [9.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.1.0-rc.1) (2020-11-04)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.0.1...v9.1.0-rc.1)
+
+ * Fix spelling error in the server ACL event type
+   [\#1535](https://github.com/matrix-org/matrix-js-sdk/pull/1535)
+ * await idb operations from crypto store for dehydration
+   [\#1533](https://github.com/matrix-org/matrix-js-sdk/pull/1533)
+ * Fix stuck never-sending messages
+   [\#1531](https://github.com/matrix-org/matrix-js-sdk/pull/1531)
+ * Await key cache check to avoid prompts
+   [\#1529](https://github.com/matrix-org/matrix-js-sdk/pull/1529)
+ * Improve ICE candidate batching
+   [\#1524](https://github.com/matrix-org/matrix-js-sdk/pull/1524)
+ * Convert logger to typescript
+   [\#1527](https://github.com/matrix-org/matrix-js-sdk/pull/1527)
+ * Fix logger typo
+   [\#1525](https://github.com/matrix-org/matrix-js-sdk/pull/1525)
+ * bind online listener to window instead of document
+   [\#1523](https://github.com/matrix-org/matrix-js-sdk/pull/1523)
+ * Support m.call.select_answer
+   [\#1522](https://github.com/matrix-org/matrix-js-sdk/pull/1522)
+
+Changes in [9.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.0.1) (2020-10-28)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.0.0...v9.0.1)
+
+ * [Release] Await key cache check to avoid prompts
+   [\#1530](https://github.com/matrix-org/matrix-js-sdk/pull/1530)
+
+Changes in [9.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.0.0) (2020-10-26)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.0.0-rc.1...v9.0.0)
+
+ * Fix logger typo
+   [\#1528](https://github.com/matrix-org/matrix-js-sdk/pull/1528)
+
+Changes in [9.0.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.0.0-rc.1) (2020-10-21)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.5.0...v9.0.0-rc.1)
+
+BREAKING CHANGES
+---
+
+ * `hasPendingEvent` now returns false instead of throwing when pending ordering mode is not `detached`
+
+All changes
+---
+
+ * Don't cache failures when fetching /versions
+   [\#1521](https://github.com/matrix-org/matrix-js-sdk/pull/1521)
+ * Install deps first as part of release
+   [\#1518](https://github.com/matrix-org/matrix-js-sdk/pull/1518)
+ * [Breaking] Change hasPendingEvent to return false if pending ordering
+   !detached
+   [\#1517](https://github.com/matrix-org/matrix-js-sdk/pull/1517)
+ * Skip editor prompts for merges
+   [\#1519](https://github.com/matrix-org/matrix-js-sdk/pull/1519)
+ * Convert call test to TypeScript
+   [\#1516](https://github.com/matrix-org/matrix-js-sdk/pull/1516)
+ * Support party_id
+   [\#1512](https://github.com/matrix-org/matrix-js-sdk/pull/1512)
+ * Support m.call.reject
+   [\#1510](https://github.com/matrix-org/matrix-js-sdk/pull/1510)
+ * Remove specbuild from .gitignore
+   [\#1515](https://github.com/matrix-org/matrix-js-sdk/pull/1515)
+ * Log the error when we failed to send candidates
+   [\#1514](https://github.com/matrix-org/matrix-js-sdk/pull/1514)
+ * Fixes for call state machine
+   [\#1503](https://github.com/matrix-org/matrix-js-sdk/pull/1503)
+ * Fix call event handler listener removing
+   [\#1506](https://github.com/matrix-org/matrix-js-sdk/pull/1506)
+ * Set the type of the call based on the tracks
+   [\#1501](https://github.com/matrix-org/matrix-js-sdk/pull/1501)
+ * Use new local timestamp for calls
+   [\#1499](https://github.com/matrix-org/matrix-js-sdk/pull/1499)
+ * Adjust types and APIs to match React SDK
+   [\#1502](https://github.com/matrix-org/matrix-js-sdk/pull/1502)
+ * Make an accurate version of 'age' for events
+   [\#1495](https://github.com/matrix-org/matrix-js-sdk/pull/1495)
+ * Make 'options' parameter optional
+   [\#1498](https://github.com/matrix-org/matrix-js-sdk/pull/1498)
+ * Create a giant event type enum
+   [\#1497](https://github.com/matrix-org/matrix-js-sdk/pull/1497)
+ * Convert call.js to Typescript & update WebRTC APIs (re-apply)
+   [\#1494](https://github.com/matrix-org/matrix-js-sdk/pull/1494)
+
+Changes in [8.5.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.5.0) (2020-10-12)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.5.0-rc.1...v8.5.0)
+
+* No changes since rc.1
+
+Changes in [8.5.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.5.0-rc.1) (2020-10-07)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.4.1...v8.5.0-rc.1)
+
+ * Add support for olm fallback keys
+   [\#1467](https://github.com/matrix-org/matrix-js-sdk/pull/1467)
+ * Fix editing local echoes not updating them in real time
+   [\#1492](https://github.com/matrix-org/matrix-js-sdk/pull/1492)
+ * Fix re-emit of Event.replaced to be on client and not room
+   [\#1491](https://github.com/matrix-org/matrix-js-sdk/pull/1491)
+ * Add space to log line
+   [\#1496](https://github.com/matrix-org/matrix-js-sdk/pull/1496)
+ * Revert "Convert call.js to Typescript & update WebRTC APIs"
+   [\#1493](https://github.com/matrix-org/matrix-js-sdk/pull/1493)
+ * Convert call.js to Typescript & update WebRTC APIs
+   [\#1487](https://github.com/matrix-org/matrix-js-sdk/pull/1487)
+ * Dehydrate and rehydrate devices
+   [\#1436](https://github.com/matrix-org/matrix-js-sdk/pull/1436)
+ * Keep local device after processing device list sync
+   [\#1490](https://github.com/matrix-org/matrix-js-sdk/pull/1490)
+ * Enforce logger module via lint rules
+   [\#1489](https://github.com/matrix-org/matrix-js-sdk/pull/1489)
+ * Extend method redactEvent with reason
+   [\#1462](https://github.com/matrix-org/matrix-js-sdk/pull/1462)
+ * Catch exception from call event handler
+   [\#1484](https://github.com/matrix-org/matrix-js-sdk/pull/1484)
+ * Ignore invalid candidates
+   [\#1483](https://github.com/matrix-org/matrix-js-sdk/pull/1483)
+ * Always push docs if they are generated
+   [\#1478](https://github.com/matrix-org/matrix-js-sdk/pull/1478)
+ * Only sign key backup with cross-signing keys when available
+   [\#1481](https://github.com/matrix-org/matrix-js-sdk/pull/1481)
+ * Upgrade dependencies
+   [\#1479](https://github.com/matrix-org/matrix-js-sdk/pull/1479)
+
+Changes in [8.4.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.4.1) (2020-09-28)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.4.0...v8.4.1)
+
+ * Catch exception from call event handler
+   [\#1486](https://github.com/matrix-org/matrix-js-sdk/pull/1486)
+ * Ignore invalid candidates
+   [\#1485](https://github.com/matrix-org/matrix-js-sdk/pull/1485)
+
+Changes in [8.4.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.4.0) (2020-09-28)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.4.0-rc.1...v8.4.0)
+
+ * Only sign key backup with cross-signing keys when available
+   [\#1482](https://github.com/matrix-org/matrix-js-sdk/pull/1482)
+
+Changes in [8.4.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.4.0-rc.1) (2020-09-23)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.3.0...v8.4.0-rc.1)
+
+ * If there are extraParams set, ensure that queryParams is defined
+   [\#1477](https://github.com/matrix-org/matrix-js-sdk/pull/1477)
+ * Add diagnostics to security bootstrap paths
+   [\#1475](https://github.com/matrix-org/matrix-js-sdk/pull/1475)
+ * Switch to a combination of better-docs and docdash
+   [\#1459](https://github.com/matrix-org/matrix-js-sdk/pull/1459)
+ * Undo attempts to cache private keys aggressively
+   [\#1474](https://github.com/matrix-org/matrix-js-sdk/pull/1474)
+ * Repair secret storage reset, cache keys when missing
+   [\#1472](https://github.com/matrix-org/matrix-js-sdk/pull/1472)
+ * Prevent parallel getVersions calls
+   [\#1471](https://github.com/matrix-org/matrix-js-sdk/pull/1471)
+ * Send end-of-candidates
+   [\#1473](https://github.com/matrix-org/matrix-js-sdk/pull/1473)
+ * Add a function for checking the /versions flag for forced e2ee
+   [\#1470](https://github.com/matrix-org/matrix-js-sdk/pull/1470)
+ * Add option to allow users of pantialaimon to use the SDK
+   [\#1469](https://github.com/matrix-org/matrix-js-sdk/pull/1469)
+ * Fixed Yarn broken link
+   [\#1468](https://github.com/matrix-org/matrix-js-sdk/pull/1468)
+ * some TypeScript and doc fixes
+   [\#1466](https://github.com/matrix-org/matrix-js-sdk/pull/1466)
+ * Remove Travis CI reference
+   [\#1464](https://github.com/matrix-org/matrix-js-sdk/pull/1464)
+ * Inject identity server token for 3pid invites on createRoom
+   [\#1463](https://github.com/matrix-org/matrix-js-sdk/pull/1463)
+
+Changes in [8.3.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.3.0) (2020-09-14)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.3.0-rc.1...v8.3.0)
+
+* No changes since rc.1
+
+Changes in [8.3.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.3.0-rc.1) (2020-09-09)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.2.0...v8.3.0-rc.1)
+
+ * Add missing options in ICreateClientOpts
+   [\#1452](https://github.com/matrix-org/matrix-js-sdk/pull/1452)
+ * Ensure ready functions return boolean values
+   [\#1457](https://github.com/matrix-org/matrix-js-sdk/pull/1457)
+ * Handle missing cross-signing keys gracefully
+   [\#1456](https://github.com/matrix-org/matrix-js-sdk/pull/1456)
+ * Fix eslint ts override tsx matching
+   [\#1451](https://github.com/matrix-org/matrix-js-sdk/pull/1451)
+ * Untangle cross-signing and secret storage
+   [\#1450](https://github.com/matrix-org/matrix-js-sdk/pull/1450)
+
+Changes in [8.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.2.0) (2020-09-01)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.2.0-rc.1...v8.2.0)
+
+## Security notice
+
+JS SDK 8.2.0 fixes an issue where encrypted state events could break incoming call handling.
+Thanks to @awesome-michael from Awesome Technologies for responsibly disclosing this via Matrix's
+Security Disclosure Policy.
+
+## All changes
+
+* No changes since rc.1
+
+Changes in [8.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.2.0-rc.1) (2020-08-26)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.1.0...v8.2.0-rc.1)
+
+ * Add state event check
+   [\#1449](https://github.com/matrix-org/matrix-js-sdk/pull/1449)
+ * Add method to check whether client .well-known has been fetched
+   [\#1444](https://github.com/matrix-org/matrix-js-sdk/pull/1444)
+ * Handle auth errors during cross-signing key upload
+   [\#1443](https://github.com/matrix-org/matrix-js-sdk/pull/1443)
+ * Don't fail if the requested audio output isn't available
+   [\#1448](https://github.com/matrix-org/matrix-js-sdk/pull/1448)
+ * Fix logging failures
+   [\#1447](https://github.com/matrix-org/matrix-js-sdk/pull/1447)
+ * Log the constraints we pass to getUserMedia
+   [\#1446](https://github.com/matrix-org/matrix-js-sdk/pull/1446)
+ * Use SAS emoji data from matrix-doc
+   [\#1440](https://github.com/matrix-org/matrix-js-sdk/pull/1440)
+
+Changes in [8.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.1.0) (2020-08-17)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.1.0-rc.1...v8.1.0)
+
+* No changes since rc.1
+
+Changes in [8.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.1.0-rc.1) (2020-08-13)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.0.1...v8.1.0-rc.1)
+
+ * Update on Promises
+   [\#1438](https://github.com/matrix-org/matrix-js-sdk/pull/1438)
+ * Store and request master cross-signing key
+   [\#1437](https://github.com/matrix-org/matrix-js-sdk/pull/1437)
+ * Filter out non-string display names
+   [\#1433](https://github.com/matrix-org/matrix-js-sdk/pull/1433)
+ * Bump elliptic from 6.5.2 to 6.5.3
+   [\#1427](https://github.com/matrix-org/matrix-js-sdk/pull/1427)
+ * Replace Riot with Element in docs and comments
+   [\#1431](https://github.com/matrix-org/matrix-js-sdk/pull/1431)
+ * Remove leftover bits of TSLint
+   [\#1430](https://github.com/matrix-org/matrix-js-sdk/pull/1430)
+
+Changes in [8.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.0.1) (2020-08-05)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.0.1-rc.1...v8.0.1)
+
+ * Filter out non-string display names
+   [\#1434](https://github.com/matrix-org/matrix-js-sdk/pull/1434)
+
+Changes in [8.0.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.0.1-rc.1) (2020-07-31)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.0.0...v8.0.1-rc.1)
+
+ * Remove redundant lint dependencies
+   [\#1426](https://github.com/matrix-org/matrix-js-sdk/pull/1426)
+ * Upload all keys when we start using a new key backup version
+   [\#1428](https://github.com/matrix-org/matrix-js-sdk/pull/1428)
+ * Expose countSessionsNeedingBackup
+   [\#1429](https://github.com/matrix-org/matrix-js-sdk/pull/1429)
+ * Configure and use new eslint package
+   [\#1422](https://github.com/matrix-org/matrix-js-sdk/pull/1422)
+
+Changes in [8.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.0.0) (2020-07-27)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v7.1.0...v8.0.0)
+
+BREAKING CHANGES
+---
+
+* `RoomState` events changed to use a Map instead of an object, which changes the collection APIs available to access them.
+ 
+All Changes
+---
+
+ * Properly support txnId
+   [\#1424](https://github.com/matrix-org/matrix-js-sdk/pull/1424)
+ * [BREAKING] Remove deprecated getIdenticonUri
+   [\#1423](https://github.com/matrix-org/matrix-js-sdk/pull/1423)
+ * Bump lodash from 4.17.15 to 4.17.19
+   [\#1421](https://github.com/matrix-org/matrix-js-sdk/pull/1421)
+ * [BREAKING] Convert RoomState's stored state map to a real map
+   [\#1419](https://github.com/matrix-org/matrix-js-sdk/pull/1419)
+
+Changes in [7.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v7.1.0) (2020-07-03)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v7.1.0-rc.1...v7.1.0)
+
+* No changes since rc.1
+
+Changes in [7.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v7.1.0-rc.1) (2020-07-01)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v7.0.0...v7.1.0-rc.1)
+
+ * Ask general crypto callbacks for 4S privkey if operation adapter doesn't
+   have it yet
+   [\#1414](https://github.com/matrix-org/matrix-js-sdk/pull/1414)
+ * Fix ICreateClientOpts missing idBaseUrl
+   [\#1413](https://github.com/matrix-org/matrix-js-sdk/pull/1413)
+ * Increase max event listeners for rooms
+   [\#1411](https://github.com/matrix-org/matrix-js-sdk/pull/1411)
+ * Don't trust keys megolm received from backup for verifying the sender
+   [\#1406](https://github.com/matrix-org/matrix-js-sdk/pull/1406)
+ * Raise the last known account data / state event for an update
+   [\#1410](https://github.com/matrix-org/matrix-js-sdk/pull/1410)
+ * Isolate encryption bootstrap side-effects
+   [\#1380](https://github.com/matrix-org/matrix-js-sdk/pull/1380)
+ * Add method to get current in-flight to-device requests
+   [\#1405](https://github.com/matrix-org/matrix-js-sdk/pull/1405)
+
+Changes in [7.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v7.0.0) (2020-06-23)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v7.0.0-rc.1...v7.0.0)
+
+* No changes since rc.1
+
+Changes in [7.0.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v7.0.0-rc.1) (2020-06-17)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.2...v7.0.0-rc.1)
+
+BREAKING CHANGES
+---
+
+ * Presence lists were removed from the spec in r0.5.0, and the corresponding methods have now been removed here as well:
+   * `getPresenceList`
+   * `inviteToPresenceList`
+   * `dropFromPresenceList`
+
+All changes
+---
+
+ * Remove support for unspecced device-specific push rules
+   [\#1404](https://github.com/matrix-org/matrix-js-sdk/pull/1404)
+ * Use existing session id for fetching flows as to not get a new session
+   [\#1403](https://github.com/matrix-org/matrix-js-sdk/pull/1403)
+ * Upgrade deps
+   [\#1400](https://github.com/matrix-org/matrix-js-sdk/pull/1400)
+ * Bring back backup key format migration
+   [\#1398](https://github.com/matrix-org/matrix-js-sdk/pull/1398)
+ * Fix: more informative error message when we cant find a key to decrypt with
+   [\#1313](https://github.com/matrix-org/matrix-js-sdk/pull/1313)
+ * Add js-sdk mechanism for polling client well-known for config
+   [\#1394](https://github.com/matrix-org/matrix-js-sdk/pull/1394)
+ * Fix verification request timeouts to match spec
+   [\#1388](https://github.com/matrix-org/matrix-js-sdk/pull/1388)
+ * Drop presence list methods
+   [\#1391](https://github.com/matrix-org/matrix-js-sdk/pull/1391)
+ * Batch up URL previews to prevent excessive requests
+   [\#1395](https://github.com/matrix-org/matrix-js-sdk/pull/1395)
+
+Changes in [6.2.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.2) (2020-06-16)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.1...v6.2.2)
+
+ * Use existing session id for fetching flows as to not get a new session
+   [\#1407](https://github.com/matrix-org/matrix-js-sdk/pull/1407)
+
+Changes in [6.2.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.1) (2020-06-05)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.0...v6.2.1)
+
+ * Bring back backup key format migration
+   [\#1399](https://github.com/matrix-org/matrix-js-sdk/pull/1399)
+
+Changes in [6.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.0) (2020-06-04)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.0-rc.1...v6.2.0)
+
+ * No changes since rc.1
+
+Changes in [6.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.0-rc.1) (2020-06-02)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.1.0...v6.2.0-rc.1)
+
+ * Make auth argument in the register request compliant with r0.6.0
+   [\#1304](https://github.com/matrix-org/matrix-js-sdk/pull/1304)
+ * Send the wrong auth params with the right auth params
+   [\#1393](https://github.com/matrix-org/matrix-js-sdk/pull/1393)
+ * encrypt cached keys with pickle key
+   [\#1387](https://github.com/matrix-org/matrix-js-sdk/pull/1387)
+ * Fix replying to key share requests
+   [\#1385](https://github.com/matrix-org/matrix-js-sdk/pull/1385)
+ * Add dist to package.json files so CDNs can serve it
+   [\#1384](https://github.com/matrix-org/matrix-js-sdk/pull/1384)
+ * Fix getVersion warning saying undefined room
+   [\#1382](https://github.com/matrix-org/matrix-js-sdk/pull/1382)
+ * Combine the two places we processed client-level default push rules
+   [\#1379](https://github.com/matrix-org/matrix-js-sdk/pull/1379)
+ * make MAC check robust against unpadded vs padded base64 differences
+   [\#1378](https://github.com/matrix-org/matrix-js-sdk/pull/1378)
+ * Remove key backup format migration
+   [\#1375](https://github.com/matrix-org/matrix-js-sdk/pull/1375)
+ * Add simple browserify browser-matrix.js tests
+   [\#1241](https://github.com/matrix-org/matrix-js-sdk/pull/1241)
+ * support new key agreement method for SAS
+   [\#1376](https://github.com/matrix-org/matrix-js-sdk/pull/1376)
+
+Changes in [6.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.1.0) (2020-05-19)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.1.0-rc.1...v6.1.0)
+
+ * No changes since rc.1
+
+Changes in [6.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.1.0-rc.1) (2020-05-14)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.0.0...v6.1.0-rc.1)
+
+ * Remove support for asymmetric 4S encryption
+   [\#1373](https://github.com/matrix-org/matrix-js-sdk/pull/1373)
+ * Increase timeout for 2nd phase of Olm session creation
+   [\#1367](https://github.com/matrix-org/matrix-js-sdk/pull/1367)
+ * Add logging on decryption retries
+   [\#1366](https://github.com/matrix-org/matrix-js-sdk/pull/1366)
+ * Emit event when a trusted self-key is stored
+   [\#1364](https://github.com/matrix-org/matrix-js-sdk/pull/1364)
+ * Customize error payload for oversized messages
+   [\#1352](https://github.com/matrix-org/matrix-js-sdk/pull/1352)
+ * Return null for key backup state when we haven't checked yet
+   [\#1363](https://github.com/matrix-org/matrix-js-sdk/pull/1363)
+ * Added a progressCallback for backup key loading
+   [\#1351](https://github.com/matrix-org/matrix-js-sdk/pull/1351)
+ * Add initialFetch param to willUpdateDevices / devicesUpdated
+   [\#1360](https://github.com/matrix-org/matrix-js-sdk/pull/1360)
+ * Fix race between sending .request and receiving .ready over to_device
+   [\#1359](https://github.com/matrix-org/matrix-js-sdk/pull/1359)
+ * Handle race between sending and await next event from other party
+   [\#1357](https://github.com/matrix-org/matrix-js-sdk/pull/1357)
+ * Add crypto.willUpdateDevices event and make
+   getStoredDevices/getStoredDevicesForUser synchronous
+   [\#1354](https://github.com/matrix-org/matrix-js-sdk/pull/1354)
+ * Fix sender of local echo events in unsigned redactions
+   [\#1350](https://github.com/matrix-org/matrix-js-sdk/pull/1350)
+ * Remove redundant key backup setup path
+   [\#1353](https://github.com/matrix-org/matrix-js-sdk/pull/1353)
+ * Remove some dead code from _retryDecryption
+   [\#1349](https://github.com/matrix-org/matrix-js-sdk/pull/1349)
+ * Don't send key requests until after sync processing is finished
+   [\#1348](https://github.com/matrix-org/matrix-js-sdk/pull/1348)
+ * Prevent attempts to send olm messages to ourselves
+   [\#1346](https://github.com/matrix-org/matrix-js-sdk/pull/1346)
+ * Retry account data upload requests
+   [\#1345](https://github.com/matrix-org/matrix-js-sdk/pull/1345)
+ * Log first known index with megolm session updates
+   [\#1344](https://github.com/matrix-org/matrix-js-sdk/pull/1344)
+ * Prune to_device messages to avoid sending empty messages
+   [\#1343](https://github.com/matrix-org/matrix-js-sdk/pull/1343)
+ * Convert bunch of things to TypeScript
+   [\#1335](https://github.com/matrix-org/matrix-js-sdk/pull/1335)
+ * Add logging when making new Olm sessions
+   [\#1342](https://github.com/matrix-org/matrix-js-sdk/pull/1342)
+ * Fix: handle filter not found
+   [\#1340](https://github.com/matrix-org/matrix-js-sdk/pull/1340)
+ * Make getAccountDataFromServer return null if not found
+   [\#1338](https://github.com/matrix-org/matrix-js-sdk/pull/1338)
+ * Fix setDefaultKeyId to fail if the request fails
+   [\#1336](https://github.com/matrix-org/matrix-js-sdk/pull/1336)
+ * Document setRoomEncryption not modifying room state
+   [\#1328](https://github.com/matrix-org/matrix-js-sdk/pull/1328)
+ * Fix: don't do extra /filter request when enabling lazy loading of members
+   [\#1332](https://github.com/matrix-org/matrix-js-sdk/pull/1332)
+ * Reject attemptAuth promise if no auth flow found
+   [\#1329](https://github.com/matrix-org/matrix-js-sdk/pull/1329)
+ * Fix FilterComponent allowed_values check
+   [\#1327](https://github.com/matrix-org/matrix-js-sdk/pull/1327)
+ * Serialise Olm prekey decryptions
+   [\#1326](https://github.com/matrix-org/matrix-js-sdk/pull/1326)
+ * Fix: crash when backup key needs fixing from corruption issue
+   [\#1324](https://github.com/matrix-org/matrix-js-sdk/pull/1324)
+ * Fix cross-signing/SSSS reset
+   [\#1322](https://github.com/matrix-org/matrix-js-sdk/pull/1322)
+ * Implement QR code reciprocate for self-verification with untrusted MSK
+   [\#1320](https://github.com/matrix-org/matrix-js-sdk/pull/1320)
+
 Changes in [6.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0) (2020-05-05)
 ================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.0.0-rc.2...v6.0.0)

CONTRIBUTING.rst

@@ -28,10 +28,9 @@ use GitHub's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use Travis for continuous integration, and all pull requests get
-automatically tested by Travis: if your change breaks the build, then the PR
-will show that there are failed checks, so please check back after a few
-minutes.
+We use continuous integration, and all pull requests get automatically tested:
+if your change breaks the build, then the PR will show that there are failed
+checks, so please check back after a few minutes.
 
 Code style
 ~~~~~~~~~~

README.md

@@ -30,7 +30,7 @@ This SDK targets Node 10 for compatibility, which translates to ES6. If you're u
 a bundler like webpack you'll likely have to transpile dependencies, including this
 SDK, to match your target browsers.
 
-Using `yarn` instead of `npm` is recommended. Please see the Yarn [install guide](https://yarnpkg.com/docs/install/) 
+Using `yarn` instead of `npm` is recommended. Please see the Yarn [install guide](https://classic.yarnpkg.com/en/docs/install) 
 if you do not have it already.
 
 ``yarn add matrix-js-sdk``
@@ -182,10 +182,8 @@ you can pass the result of the promise into it with something like:
   matrixClient.someMethod(arg1, arg2).nodeify(callback);
 ```
 
-The main thing to note is that it is an error to discard the result of a
-promise-returning function, as that will cause exceptions to go unobserved. If
-you have nothing better to do with the result, just call ``.done()`` on it. See
-http://documentup.com/kriskowal/q/#the-end for more information.
+The main thing to note is that it is problematic to discard the result of a
+promise-returning function, as that will cause exceptions to go unobserved.
 
 Methods which return a promise show this in their documentation.
 

examples/node/app.js

@@ -288,7 +288,7 @@ function printMemberList(room) {
 }
 
 function printRoomInfo(room) {
-    var eventDict = room.currentState.events;
+    var eventMap = room.currentState.events;
     var eTypeHeader = "    Event Type(state_key)    ";
     var sendHeader = "        Sender        ";
     // pad content to 100
@@ -300,14 +300,15 @@ function printRoomInfo(room) {
     var contentHeader = padSide + "Content" + padSide;
     print(eTypeHeader+sendHeader+contentHeader);
     print(new Array(100).join("-"));
-    Object.keys(eventDict).forEach(function(eventType) {
+    eventMap.keys().forEach(function(eventType) {
         if (eventType === "m.room.member") { return; } // use /members instead.
-        Object.keys(eventDict[eventType]).forEach(function(stateKey) {
+        var eventEventMap = eventMap.get(eventType);
+        eventEventMap.keys().forEach(function(stateKey) {
             var typeAndKey = eventType + (
                 stateKey.length > 0 ? "("+stateKey+")" : ""
             );
             var typeStr = fixWidth(typeAndKey, eTypeHeader.length);
-            var event = eventDict[eventType][stateKey];
+            var event = eventEventMap.get(stateKey);
             var sendStr = fixWidth(event.getSender(), sendHeader.length);
             var contentStr = fixWidth(
                 JSON.stringify(event.getContent()), contentHeader.length

jsdoc.json

@@ -18,6 +18,13 @@
     "readme": "README.md",
     "recurse": true,
     "verbose": true,
-    "template": "node_modules/better-docs"
+    "template": "node_modules/docdash"
+  },
+  "docdash": {
+    "static": true,
+    "private": false,
+    "search": true,
+    "collapse": true,
+    "typedefs": true
   }
 }

package.json

@@ -1,22 +1,22 @@
 {
   "name": "matrix-js-sdk",
-  "version": "6.0.0",
+  "version": "9.6.0-rc.1",
   "description": "Matrix Client-Server SDK for Javascript",
   "scripts": {
-    "prepare": "yarn build",
+    "prepublishOnly": "yarn build",
     "start": "echo THIS IS FOR LEGACY PURPOSES ONLY. && babel src -w -s -d lib --verbose --extensions \".ts,.js\"",
     "dist": "echo 'This is for the release script so it can make assets (browser bundle).' && yarn build",
     "clean": "rimraf lib dist",
-    "build": "yarn clean && git rev-parse HEAD > git-revision.txt && yarn build:compile && yarn build:compile-browser && yarn build:minify-browser && yarn build:types",
+    "build": "yarn build:dev && yarn build:compile-browser && yarn build:minify-browser",
+    "build:dev": "yarn clean && git rev-parse HEAD > git-revision.txt && yarn build:compile && yarn build:types",
     "build:types": "tsc --emitDeclarationOnly",
     "build:compile": "babel -d lib --verbose --extensions \".ts,.js\" src",
     "build:compile-browser": "mkdirp dist && browserify -d src/browser-index.js -p [ tsify -p ./tsconfig.json ] -t [ babelify --sourceMaps=inline --presets [ @babel/preset-env @babel/preset-typescript ] ] | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js",
     "build:minify-browser": "terser dist/browser-matrix.js --compress --mangle --source-map --output dist/browser-matrix.min.js",
     "gendoc": "jsdoc -c jsdoc.json -P package.json",
-    "lint": "yarn lint:types && yarn lint:ts && yarn lint:js",
-    "lint:js": "eslint --max-warnings 81 src spec",
+    "lint": "yarn lint:types && yarn lint:js",
+    "lint:js": "eslint --max-warnings 73 src spec",
     "lint:types": "tsc --noEmit",
-    "lint:ts": "tslint --project ./tsconfig.json -t stylish",
     "test": "jest spec/ --coverage --testEnvironment node",
     "test:watch": "jest spec/ --coverage --testEnvironment node --watch"
   },
@@ -28,13 +28,15 @@
     "matrix-org"
   ],
   "main": "./lib/index.js",
-  "typings": "./lib/index.d.ts",
   "browser": "./lib/browser-index.js",
   "matrix_src_main": "./src/index.ts",
   "matrix_src_browser": "./src/browser-index.js",
+  "matrix_lib_main": "./lib/index.js",
+  "matrix_lib_typings": "./lib/index.d.ts",
   "author": "matrix.org",
   "license": "Apache-2.0",
   "files": [
+    "dist",
     "lib",
     "src",
     "git-revision.txt",
@@ -46,51 +48,53 @@
     "release.sh"
   ],
   "dependencies": {
-    "@babel/runtime": "^7.8.3",
+    "@babel/runtime": "^7.12.5",
     "another-json": "^0.2.0",
     "browser-request": "^0.3.3",
     "bs58": "^4.0.1",
-    "content-type": "^1.0.2",
-    "loglevel": "^1.6.4",
-    "qs": "^6.5.2",
-    "request": "^2.88.0",
-    "unhomoglyph": "^1.0.2"
+    "content-type": "^1.0.4",
+    "loglevel": "^1.7.1",
+    "qs": "^6.9.6",
+    "request": "^2.88.2",
+    "unhomoglyph": "^1.0.6"
   },
   "devDependencies": {
-    "@babel/cli": "^7.7.5",
-    "@babel/core": "^7.7.5",
-    "@babel/plugin-proposal-class-properties": "^7.7.4",
-    "@babel/plugin-proposal-numeric-separator": "^7.7.4",
-    "@babel/plugin-proposal-object-rest-spread": "^7.7.4",
-    "@babel/plugin-syntax-dynamic-import": "^7.7.4",
-    "@babel/plugin-transform-runtime": "^7.8.3",
-    "@babel/preset-env": "^7.7.6",
-    "@babel/preset-typescript": "^7.7.4",
-    "@babel/register": "^7.7.4",
+    "@babel/cli": "^7.12.10",
+    "@babel/core": "^7.12.10",
+    "@babel/plugin-proposal-class-properties": "^7.12.1",
+    "@babel/plugin-proposal-numeric-separator": "^7.12.7",
+    "@babel/plugin-proposal-object-rest-spread": "^7.12.1",
+    "@babel/plugin-syntax-dynamic-import": "^7.8.3",
+    "@babel/plugin-transform-runtime": "^7.12.10",
+    "@babel/preset-env": "^7.12.11",
+    "@babel/preset-typescript": "^7.12.7",
+    "@babel/register": "^7.12.10",
+    "@types/jest": "^26.0.20",
     "@types/node": "12",
-    "babel-eslint": "^10.0.3",
-    "babel-jest": "^24.9.0",
+    "@types/request": "^2.48.5",
+    "babel-eslint": "^10.1.0",
+    "babel-jest": "^26.6.3",
     "babelify": "^10.0.0",
-    "better-docs": "^1.4.7",
-    "browserify": "^16.5.0",
-    "eslint": "^5.12.0",
-    "eslint-config-google": "^0.7.1",
-    "eslint-plugin-babel": "^5.3.0",
-    "eslint-plugin-jest": "^23.0.4",
+    "better-docs": "^2.3.2",
+    "browserify": "^17.0.0",
+    "docdash": "^1.2.0",
+    "eslint": "7.18.0",
+    "eslint-config-matrix-org": "^0.2.0",
+    "eslint-plugin-babel": "^5.3.1",
     "exorcist": "^1.0.1",
-    "fake-indexeddb": "^3.0.0",
-    "jest": "^24.9.0",
-    "jest-localstorage-mock": "^2.4.0",
-    "jsdoc": "^3.5.5",
+    "fake-indexeddb": "^3.1.2",
+    "jest": "^26.6.3",
+    "jest-localstorage-mock": "^2.4.6",
+    "jsdoc": "^3.6.6",
     "matrix-mock-request": "^1.2.3",
-    "olm": "https://packages.matrix.org/npm/olm/olm-3.1.4.tgz",
-    "rimraf": "^3.0.0",
-    "terser": "^4.4.3",
-    "tsify": "^4.0.1",
-    "tslint": "^5.20.1",
-    "typescript": "^3.7.3"
+    "olm": "https://packages.matrix.org/npm/olm/olm-3.2.1.tgz",
+    "rimraf": "^3.0.2",
+    "terser": "^5.5.1",
+    "tsify": "^5.0.2",
+    "typescript": "^4.1.3"
   },
   "jest": {
     "testEnvironment": "node"
-  }
+  },
+  "typings": "./lib/index.d.ts"
 }

release.sh

@@ -10,7 +10,7 @@
 #   npm; typically installed by Node.js
 #   yarn; install via brew (macOS) or similar (https://yarnpkg.com/docs/install/)
 #
-# Note: this script is also used to release matrix-react-sdk and riot-web.
+# Note: this script is also used to release matrix-react-sdk and element-web.
 
 set -e
 
@@ -94,6 +94,14 @@ if [ $# -ne 1 ]; then
     exit 1
 fi
 
+# We use Git branch / commit dependencies for some packages, and Yarn seems
+# to have a hard time getting that right. See also
+# https://github.com/yarnpkg/yarn/issues/4734. As a workaround, we clean the
+# global cache here to ensure we get the right thing.
+yarn cache clean
+# Ensure all dependencies are updated
+yarn install --ignore-scripts
+
 if [ -z "$skip_changelog" ]; then
     # update_changelog doesn't have a --version flag
     update_changelog -h > /dev/null || (echo "github-changelog-generator is required: please install it"; exit)
@@ -170,6 +178,19 @@ echo "yarn version"
 # manually commit the result.
 yarn version --no-git-tag-version --new-version "$release"
 
+# For the published and dist versions of the package, we copy the
+# `matrix_lib_main` and `matrix_lib_typings` fields to `main` and `typings` (if
+# they exist). This small bit of gymnastics allows us to use the TypeScript
+# source directly for development without needing to build before linting or
+# testing.
+for i in main typings
+do
+    lib_value=$(jq -r ".matrix_lib_$i" package.json)
+    if [ "$lib_value" != "null" ]; then
+        jq ".$i = .matrix_lib_$i" package.json > package.json.new && mv package.json.new package.json
+    fi
+done
+
 # commit yarn.lock if it exists, is versioned, and is modified
 if [[ -f yarn.lock && `git status --porcelain yarn.lock | grep '^ M'` ]];
 then
@@ -204,11 +225,6 @@ if [ $dodist -eq 0 ]; then
     pushd "$builddir"
     git clone "$projdir" .
     git checkout "$rel_branch"
-    # We use Git branch / commit dependencies for some packages, and Yarn seems
-    # to have a hard time getting that right. See also
-    # https://github.com/yarnpkg/yarn/issues/4734. As a workaround, we clean the
-    # global cache here to ensure we get the right thing.
-    yarn cache clean
     yarn install
     # We haven't tagged yet, so tell the dist script what version
     # it's building
@@ -327,6 +343,7 @@ if [ -z "$skip_jsdoc" ]; then
         $release index.html
     git add "$release"
     git commit --no-verify -m "Add jsdoc for $release" index.html "$release"
+    git push origin gh-pages
 fi
 
 # if it is a pre-release, leave it on the release branch for now.
@@ -339,18 +356,40 @@ fi
 echo "updating master branch"
 git checkout master
 git pull
-git merge "$rel_branch"
+git merge "$rel_branch" --no-edit
 
-# push master and docs (if generated) to github
+# push master to github
 git push origin master
-if [ -z "$skip_jsdoc" ]; then
-    git push origin gh-pages
-fi
 
 # finally, merge master back onto develop (if it exists)
 if [ $(git branch -lr | grep origin/develop -c) -ge 1 ]; then
     git checkout develop
     git pull
-    git merge master
+    git merge master --no-edit
+
+    # When merging to develop, we need revert the `main` and `typings` fields if
+    # we adjusted them previously.
+    for i in main typings
+    do
+        # If a `lib` prefixed value is present, it means we adjusted the field
+        # earlier at publish time, so we should revert it now.
+        if [ "$(jq -r ".matrix_lib_$i" package.json)" != "null" ]; then
+            # If there's a `src` prefixed value, use that, otherwise delete.
+            # This is used to delete the `typings` field and reset `main` back
+            # to the TypeScript source.
+            src_value=$(jq -r ".matrix_src_$i" package.json)
+            if [ "$src_value" != "null" ]; then
+                jq ".$i = .matrix_src_$i" package.json > package.json.new && mv package.json.new package.json
+            else
+                jq "del(.$i)" package.json > package.json.new && mv package.json.new package.json
+            fi
+        fi
+    done
+
+    if [ -n "$(git ls-files --modified package.json)" ]; then
+        echo "Committing develop package.json"
+        git commit package.json -m "Resetting package fields for development"
+    fi
+
     git push origin develop
 fi

spec/TestClient.js

@@ -69,6 +69,9 @@ export function TestClient(
 
     this.deviceKeys = null;
     this.oneTimeKeys = {};
+    this._callEventHandler = {
+        calls: new Map(),
+    };
 }
 
 TestClient.prototype.toString = function() {
@@ -230,3 +233,7 @@ TestClient.prototype.flushSync = function() {
         logger.log(`${this}: flushSync completed`);
     });
 };
+
+TestClient.prototype.isFallbackICEServerAllowed = function() {
+    return true;
+};

spec/browserify/setupTests.js

@@ -0,0 +1,23 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// stub for browser-matrix browserify tests
+global.XMLHttpRequest = jest.fn();
+
+afterAll(() => {
+    // clean up XMLHttpRequest mock
+    global.XMLHttpRequest = undefined;
+});

spec/browserify/sync-browserify.spec.js

@@ -0,0 +1,103 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// load XmlHttpRequest mock
+import "./setupTests";
+import "../../dist/browser-matrix"; // uses browser-matrix instead of the src
+import {MockStorageApi} from "../MockStorageApi";
+import {WebStorageSessionStore} from "../../src/store/session/webstorage";
+import MockHttpBackend from "matrix-mock-request";
+import {LocalStorageCryptoStore} from "../../src/crypto/store/localStorage-crypto-store";
+import * as utils from "../test-utils";
+
+const USER_ID = "@user:test.server";
+const DEVICE_ID = "device_id";
+const ACCESS_TOKEN = "access_token";
+const ROOM_ID = "!room_id:server.test";
+
+/* global matrixcs */
+
+describe("Browserify Test", function() {
+    let client;
+    let httpBackend;
+
+    async function createTestClient() {
+        const sessionStoreBackend = new MockStorageApi();
+        const sessionStore = new WebStorageSessionStore(sessionStoreBackend);
+        const httpBackend = new MockHttpBackend();
+
+        const options = {
+            baseUrl: "http://" + USER_ID + ".test.server",
+            userId: USER_ID,
+            accessToken: ACCESS_TOKEN,
+            deviceId: DEVICE_ID,
+            sessionStore: sessionStore,
+            request: httpBackend.requestFn,
+            cryptoStore: new LocalStorageCryptoStore(sessionStoreBackend),
+        };
+
+        const client = matrixcs.createClient(options);
+
+        httpBackend.when("GET", "/pushrules").respond(200, {});
+        httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+
+        return { client, httpBackend };
+    }
+
+    beforeEach(async () => {
+        ({client, httpBackend} = await createTestClient());
+        await client.startClient();
+    });
+
+    afterEach(async () => {
+        client.stopClient();
+        await httpBackend.stop();
+    });
+
+    it("Sync", async function() {
+        const event = utils.mkMembership({
+            room: ROOM_ID,
+            mship: "join",
+            user: "@other_user:server.test",
+            name: "Displayname",
+        });
+
+        const syncData = {
+            next_batch: "batch1",
+            rooms: {
+                join: {},
+            },
+        };
+        syncData.rooms.join[ROOM_ID] = {
+            timeline: {
+                events: [
+                    event,
+                ],
+                limited: false,
+            },
+        };
+
+        httpBackend.when("GET", "/sync").respond(200, syncData);
+        await Promise.race([
+            Promise.all([
+                httpBackend.flushAllExpected(),
+            ]),
+            new Promise((_, reject) => {
+                client.once("sync.unexpectedError", reject);
+            }),
+        ]);
+    }, 20000); // additional timeout as this test can take quite a while
+});

spec/integ/devicelist-integ-spec.js

@@ -140,7 +140,7 @@ describe("DeviceList management:", function() {
 
     it("We should not get confused by out-of-order device query responses",
        () => {
-           // https://github.com/vector-im/riot-web/issues/3126
+           // https://github.com/vector-im/element-web/issues/3126
            aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
            return aliceTestClient.start().then(() => {
                aliceTestClient.httpBackend.when('GET', '/sync').respond(
@@ -271,7 +271,7 @@ describe("DeviceList management:", function() {
            });
        }).timeout(3000);
 
-    // https://github.com/vector-im/riot-web/issues/4983
+    // https://github.com/vector-im/element-web/issues/4983
     describe("Alice should know she has stale device lists", () => {
         beforeEach(async function() {
             await aliceTestClient.start();

spec/integ/matrix-client-retrying.spec.js

@@ -90,7 +90,7 @@ describe("MatrixClient retrying", function() {
         // wait for the localecho of ev1 to be updated
         const p3 = new Promise((resolve, reject) => {
             room.on("Room.localEchoUpdated", (ev0) => {
-                if(ev0 === ev1) {
+                if (ev0 === ev1) {
                     resolve();
                 }
             });

spec/integ/megolm-integ.spec.js

@@ -346,7 +346,7 @@ describe("megolm", function() {
     });
 
     it("Alice receives a megolm message before the session keys", function() {
-        // https://github.com/vector-im/riot-web/issues/2273
+        // https://github.com/vector-im/element-web/issues/2273
         let roomKeyEncrypted;
 
         return aliceTestClient.start().then(() => {
@@ -726,7 +726,7 @@ describe("megolm", function() {
         });
     });
 
-    // https://github.com/vector-im/riot-web/issues/2676
+    // https://github.com/vector-im/element-web/issues/2676
     it("Alice should send to her other devices", function() {
         // for this test, we make the testOlmAccount be another of Alice's devices.
         // it ought to get included in messages Alice sends.
@@ -971,4 +971,60 @@ describe("megolm", function() {
             expect(event.getContent().body).toEqual('42');
         });
     });
+
+    it("Alice receives an untrusted megolm key, only to receive the trusted one shortly after", function() {
+        const testClient = new TestClient(
+            "@alice:localhost", "device2", "access_token2",
+        );
+        const groupSession = new Olm.OutboundGroupSession();
+        groupSession.create();
+        const inboundGroupSession = new Olm.InboundGroupSession();
+        inboundGroupSession.create(groupSession.session_key());
+        const rawEvent = encryptMegolmEvent({
+            senderKey: testSenderKey,
+            groupSession: groupSession,
+            room_id: ROOM_ID,
+        });
+        return testClient.client.initCrypto().then(() => {
+            const keys = [{
+                room_id: ROOM_ID,
+                algorithm: 'm.megolm.v1.aes-sha2',
+                session_id: groupSession.session_id(),
+                session_key: inboundGroupSession.export_session(0),
+                sender_key: testSenderKey,
+            }];
+            return testClient.client.importRoomKeys(keys, { untrusted: true });
+        }).then(() => {
+            const event = testUtils.mkEvent({
+                event: true,
+                ...rawEvent,
+                room: ROOM_ID,
+            });
+            return event.attemptDecryption(testClient.client._crypto, true).then(() => {
+                expect(event.isKeySourceUntrusted()).toBeTruthy();
+            });
+        }).then(() => {
+            const event = testUtils.mkEvent({
+                type: 'm.room_key',
+                content: {
+                    room_id: ROOM_ID,
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    session_id: groupSession.session_id(),
+                    session_key: groupSession.session_key(),
+                },
+                event: true,
+            });
+            event._senderCurve25519Key = testSenderKey;
+            return testClient.client._crypto._onRoomKeyEvent(event);
+        }).then(() => {
+            const event = testUtils.mkEvent({
+                event: true,
+                ...rawEvent,
+                room: ROOM_ID,
+            });
+            return event.attemptDecryption(testClient.client._crypto, true).then(() => {
+                expect(event.isKeySourceUntrusted()).toBeFalsy();
+            });
+        });
+    });
 });

spec/olm-loader.js

@@ -16,6 +16,7 @@ limitations under the License.
 */
 
 import {logger} from '../src/logger';
+import * as utils from "../src/utils";
 
 // try to load the olm library.
 try {
@@ -24,3 +25,11 @@ try {
 } catch (e) {
     logger.warn("unable to run crypto tests: libolm not available");
 }
+
+// also try to set node crypto
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    logger.log('nodejs was compiled without crypto support: some tests will fail');
+}

spec/unit/content-repo.spec.js

@@ -1,4 +1,4 @@
-import {getHttpUriForMxc, getIdenticonUri} from "../../src/content-repo";
+import {getHttpUriForMxc} from "../../src/content-repo";
 
 describe("ContentRepo", function() {
     const baseUrl = "https://my.home.server";
@@ -56,31 +56,4 @@ describe("ContentRepo", function() {
             );
         });
     });
-
-    describe("getIdenticonUri", function() {
-        it("should do nothing for null input", function() {
-            expect(getIdenticonUri(null)).toEqual(null);
-        });
-
-        it("should set w/h by default to 96", function() {
-            expect(getIdenticonUri(baseUrl, "foobar")).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foobar" +
-                "?width=96&height=96",
-            );
-        });
-
-        it("should be able to set custom w/h", function() {
-            expect(getIdenticonUri(baseUrl, "foobar", 32, 64)).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foobar" +
-                "?width=32&height=64",
-            );
-        });
-
-        it("should URL encode the identicon string", function() {
-            expect(getIdenticonUri(baseUrl, "foo#bar", 32, 64)).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foo%23bar" +
-                "?width=32&height=64",
-            );
-        });
-    });
 });

spec/unit/crypto.spec.js

@@ -10,6 +10,7 @@ import * as olmlib from "../../src/crypto/olmlib";
 import {sleep} from "../../src/utils";
 import {EventEmitter} from "events";
 import {CRYPTO_ENABLED} from "../../src/client";
+import {DeviceInfo} from "../../src/crypto/deviceinfo";
 
 const Olm = global.Olm;
 
@@ -26,6 +27,66 @@ describe("Crypto", function() {
         expect(Crypto.getOlmVersion()[0]).toEqual(3);
     });
 
+    describe("encrypted events", function() {
+        it("provides encryption information", async function() {
+            const client = (new TestClient(
+                "@alice:example.com", "deviceid",
+            )).client;
+            await client.initCrypto();
+
+            // unencrypted event
+            const event = {
+                getId: () => "$event_id",
+                getSenderKey: () => null,
+                getWireContent: () => {return {};},
+            };
+
+            let encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeFalsy();
+
+            // unknown sender (e.g. deleted device), forwarded megolm key (untrusted)
+            event.getSenderKey = () => 'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI';
+            event.getWireContent = () => {return {algorithm: olmlib.MEGOLM_ALGORITHM};};
+            event.getForwardingCurve25519KeyChain = () => ["not empty"];
+            event.isKeySourceUntrusted = () => false;
+            event.getClaimedEd25519Key =
+                () => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeFalsy();
+            expect(encryptionInfo.sender).toBeFalsy();
+
+            // known sender, megolm key from backup
+            event.getForwardingCurve25519KeyChain = () => [];
+            event.isKeySourceUntrusted = () => true;
+            const device = new DeviceInfo("FLIBBLE");
+            device.keys["curve25519:FLIBBLE"] =
+                'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI';
+            device.keys["ed25519:FLIBBLE"] =
+                'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
+            client._crypto._deviceList.getDeviceByIdentityKey = () => device;
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeFalsy();
+            expect(encryptionInfo.sender).toBeTruthy();
+            expect(encryptionInfo.mismatchedSender).toBeFalsy();
+
+            // known sender, trusted megolm key, but bad ed25519key
+            event.isKeySourceUntrusted = () => false;
+            device.keys["ed25519:FLIBBLE"] =
+                'BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB';
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeTruthy();
+            expect(encryptionInfo.sender).toBeTruthy();
+            expect(encryptionInfo.mismatchedSender).toBeTruthy();
+
+            client.stopClient();
+        });
+    });
 
     describe('Session management', function() {
         const otkResponse = {
@@ -313,6 +374,10 @@ describe("Crypto", function() {
             // make a room key request, and record the transaction ID for the
             // sendToDevice call
             await aliceClient.cancelAndResendEventRoomKeyRequest(event);
+            // key requests get queued until the sync has finished, but we don't
+            // let the client set up enough for that to happen, so gut-wrench a bit
+            // to force it to send now.
+            aliceClient._crypto._outgoingRoomKeyRequestManager.sendQueuedRequests();
             jest.runAllTimers();
             await Promise.resolve();
             expect(aliceClient.sendToDevice).toBeCalledTimes(1);

spec/unit/crypto/CrossSigningInfo.spec.js

@@ -25,6 +25,8 @@ import {
 import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
 import 'fake-indexeddb/auto';
 import 'jest-localstorage-mock';
+import {OlmDevice} from "../../../src/crypto/OlmDevice";
+import {logger} from '../../../src/logger';
 
 const userId = "@alice:example.com";
 
@@ -37,7 +39,7 @@ const testKey = new Uint8Array([
 ]);
 
 const types = [
-    { type: "master", shouldCache: false },
+    { type: "master", shouldCache: true },
     { type: "self_signing", shouldCache: true },
     { type: "user_signing", shouldCache: true },
     { type: "invalid", shouldCache: false },
@@ -50,7 +52,7 @@ const masterKeyPub = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk";
 
 describe("CrossSigningInfo.getCrossSigningKey", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 
@@ -82,9 +84,16 @@ describe("CrossSigningInfo.getCrossSigningKey", function() {
         const info = new CrossSigningInfo(userId, {
             getCrossSigningKey: () => testKey,
         });
-        const [pubKey, ab] = await info.getCrossSigningKey("master", masterKeyPub);
+        const [pubKey, pkSigning] = await info.getCrossSigningKey("master", masterKeyPub);
         expect(pubKey).toEqual(masterKeyPub);
-        expect(ab).toEqual({a: 106712, b: 106712});
+        // check that the pkSigning object corresponds to the pubKey
+        const signature = pkSigning.sign("message");
+        const util = new global.Olm.Utility();
+        try {
+            util.ed25519_verify(pubKey, "message", signature);
+        } finally {
+            util.free();
+        }
     });
 
     it.each(types)("should request a key from the cache callback (if set)" +
@@ -233,8 +242,9 @@ describe.each([
 
     it("should cache data to the store and retrieve it", async () => {
         await store.startup();
+        const olmDevice = new OlmDevice(store);
         const { getCrossSigningKeyCache, storeCrossSigningKeyCache } =
-          createCryptoStoreCacheCallbacks(store);
+              createCryptoStoreCacheCallbacks(store, olmDevice);
         await storeCrossSigningKeyCache("self_signing", testKey);
 
         // If we've not saved anything, don't expect anything
@@ -243,6 +253,6 @@ describe.each([
         expect(nokey).toBeNull();
 
         const key = await getCrossSigningKeyCache("self_signing", "");
-        expect(key).toEqual(testKey);
+        expect(new Uint8Array(key)).toEqual(testKey);
     });
 });

spec/unit/crypto/DeviceList.spec.js

@@ -72,6 +72,8 @@ describe('DeviceList', function() {
     function createTestDeviceList() {
         const baseApis = {
             downloadKeysForUsers: downloadSpy,
+            getUserId: () => '@test1:sw1v.org',
+            deviceId: 'HGKAWHRVJQ',
         };
         const mockOlm = {
             verifySignature: function(key, message, signature) {},

spec/unit/crypto/backup.spec.js

@@ -27,6 +27,7 @@ import {MockStorageApi} from "../../MockStorageApi";
 import * as testUtils from "../../test-utils";
 import {OlmDevice} from "../../../src/crypto/OlmDevice";
 import {Crypto} from "../../../src/crypto";
+import {resetCrossSigningKeys} from "./crypto-utils";
 
 const Olm = global.Olm;
 
@@ -332,7 +333,7 @@ describe("MegolmBackup", function() {
             client.on("crossSigning.getKey", function(e) {
                 e.done(privateKeys[e.type]);
             });
-            await client.resetCrossSigningKeys();
+            await resetCrossSigningKeys(client);
             let numCalls = 0;
             await new Promise((resolve, reject) => {
                 client._http.authedRequest = function(
@@ -517,6 +518,7 @@ describe("MegolmBackup", function() {
                 return megolmDecryption.decryptEvent(ENCRYPTED_EVENT);
             }).then((res) => {
                 expect(res.clearEvent.content).toEqual('testytest');
+                expect(res.untrusted).toBeTruthy(); // keys from backup are untrusted
             });
         });
 
@@ -546,7 +548,7 @@ describe("MegolmBackup", function() {
             const key = Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]);
             await client._crypto.storeSessionBackupPrivateKey(key);
             const result = await client._crypto.getSessionBackupPrivateKey();
-            expect(result).toEqual(key);
+            expect(new Uint8Array(result)).toEqual(key);
         });
 
         it('caches session backup keys as it encounters them', async function() {

spec/unit/crypto/cross-signing.spec.js

@@ -20,6 +20,9 @@ import anotherjson from 'another-json';
 import * as olmlib from "../../../src/crypto/olmlib";
 import {TestClient} from '../../TestClient';
 import {HttpResponse, setHttpResponses} from '../../test-utils';
+import { resetCrossSigningKeys } from "./crypto-utils";
+import { MatrixError } from '../../../src/http-api';
+import {logger} from '../../../src/logger';
 
 async function makeTestClient(userInfo, options, keys) {
     if (!keys) keys = {};
@@ -47,7 +50,7 @@ async function makeTestClient(userInfo, options, keys) {
 
 describe("Cross Signing", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 
@@ -66,11 +69,66 @@ describe("Cross Signing", function() {
             );
         });
         alice.uploadKeySignatures = async () => {};
+        alice.setAccountData = async () => {};
+        alice.getAccountDataFromServer = async () => {};
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await alice.bootstrapCrossSigning({
+            authUploadDeviceSigningKeys: async func => await func({}),
+        });
         expect(alice.uploadDeviceSigningKeys).toHaveBeenCalled();
     });
 
+    it("should abort bootstrap if device signing auth fails", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = async (auth, keys) => {
+            const errorResponse = {
+                session: "sessionId",
+                flows: [
+                    {
+                        stages: [
+                            "m.login.password",
+                        ],
+                    },
+                ],
+                params: {},
+            };
+
+            // If we're not just polling for flows, add on error rejecting the
+            // auth attempt.
+            if (auth) {
+                Object.assign(errorResponse, {
+                    completed: [],
+                    error: "Invalid password",
+                    errcode: "M_FORBIDDEN",
+                });
+            }
+
+            const error = new MatrixError(errorResponse);
+            error.httpStatus == 401;
+            throw error;
+        };
+        alice.uploadKeySignatures = async () => {};
+        alice.setAccountData = async () => {};
+        alice.getAccountDataFromServer = async () => { };
+        const authUploadDeviceSigningKeys = async func => await func({});
+
+        // Try bootstrap, expecting `authUploadDeviceSigningKeys` to pass
+        // through failure, stopping before actually applying changes.
+        let bootstrapDidThrow = false;
+        try {
+            await alice.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys,
+            });
+        } catch (e) {
+            if (e.errcode === "M_FORBIDDEN") {
+                bootstrapDidThrow = true;
+            }
+        }
+        expect(bootstrapDidThrow).toBeTruthy();
+    });
+
     it("should upload a signature when a user is verified", async function() {
         const alice = await makeTestClient(
             {userId: "@alice:example.com", deviceId: "Osborne2"},
@@ -78,7 +136,7 @@ describe("Cross Signing", function() {
         alice.uploadDeviceSigningKeys = async () => {};
         alice.uploadKeySignatures = async () => {};
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         // Alice downloads Bob's device key
         alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
             keys: {
@@ -273,7 +331,7 @@ describe("Cross Signing", function() {
         alice.uploadDeviceSigningKeys = async () => {};
         alice.uploadKeySignatures = async () => {};
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         // Alice downloads Bob's ssk and device key
         const bobMasterSigning = new global.Olm.PkSigning();
         const bobMasterPrivkey = bobMasterSigning.generate_seed();
@@ -363,7 +421,7 @@ describe("Cross Signing", function() {
         alice.uploadKeySignatures = async () => {};
 
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
 
         const selfSigningKey = new Uint8Array([
             0x1e, 0xf4, 0x01, 0x6d, 0x4f, 0xa1, 0x73, 0x66,
@@ -520,7 +578,7 @@ describe("Cross Signing", function() {
         alice.uploadDeviceSigningKeys = async () => {};
         alice.uploadKeySignatures = async () => {};
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         // Alice downloads Bob's ssk and device key
         // (NOTE: device key is not signed by ssk)
         const bobMasterSigning = new global.Olm.PkSigning();
@@ -588,7 +646,7 @@ describe("Cross Signing", function() {
         );
         alice.uploadDeviceSigningKeys = async () => {};
         alice.uploadKeySignatures = async () => {};
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         // Alice downloads Bob's keys
         const bobMasterSigning = new global.Olm.PkSigning();
         const bobMasterPrivkey = bobMasterSigning.generate_seed();
@@ -740,7 +798,7 @@ describe("Cross Signing", function() {
         bob.uploadDeviceSigningKeys = async () => {};
         bob.uploadKeySignatures = async () => {};
         // set Bob's cross-signing key
-        await bob.resetCrossSigningKeys();
+        await resetCrossSigningKeys(bob);
         alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
             Dynabook: {
                 algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
@@ -766,7 +824,7 @@ describe("Cross Signing", function() {
         let upgradePromise = new Promise((resolve) => {
             upgradeResolveFunc = resolve;
         });
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         await upgradePromise;
 
         const bobTrust = alice.checkUserTrust("@bob:example.com");

spec/unit/crypto/crypto-utils.js

@@ -0,0 +1,44 @@
+import {IndexedDBCryptoStore} from '../../../src/crypto/store/indexeddb-crypto-store';
+
+
+// needs to be phased out and replaced with bootstrapSecretStorage,
+// but that is doing too much extra stuff for it to be an easy transition.
+export async function resetCrossSigningKeys(client, {
+    level,
+    authUploadDeviceSigningKeys = async func => await func(),
+} = {}) {
+    const crypto = client._crypto;
+
+    const oldKeys = Object.assign({}, crypto._crossSigningInfo.keys);
+    try {
+        await crypto._crossSigningInfo.resetKeys(level);
+        await crypto._signObject(crypto._crossSigningInfo.keys.master);
+        // write a copy locally so we know these are trusted keys
+        await crypto._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+            (txn) => {
+                crypto._cryptoStore.storeCrossSigningKeys(
+                    txn, crypto._crossSigningInfo.keys);
+            },
+        );
+    } catch (e) {
+        // If anything failed here, revert the keys so we know to try again from the start
+        // next time.
+        crypto._crossSigningInfo.keys = oldKeys;
+        throw e;
+    }
+    crypto._baseApis.emit("crossSigning.keysChanged", {});
+    await crypto._afterCrossSigningLocalKeyChange();
+}
+
+export async function createSecretStorageKey() {
+    const decryption = new global.Olm.PkDecryption();
+    const storagePublicKey = decryption.generate_key();
+    const storagePrivateKey = decryption.get_private_key();
+    decryption.free();
+    return {
+        // `pubkey` not used anymore with symmetric 4S
+        keyInfo: { pubkey: storagePublicKey },
+        privateKey: storagePrivateKey,
+    };
+}

spec/unit/crypto/secrets.spec.js

@@ -21,6 +21,8 @@ import {MatrixEvent} from "../../../src/models/event";
 import {TestClient} from '../../TestClient';
 import {makeTestClients} from './verification/util';
 import {encryptAES} from "../../../src/crypto/aes";
+import {resetCrossSigningKeys, createSecretStorageKey} from "./crypto-utils";
+import {logger} from '../../../src/logger';
 
 import * as utils from "../../../src/utils";
 
@@ -28,7 +30,7 @@ try {
     const crypto = require('crypto');
     utils.setCrypto(crypto);
 } catch (err) {
-    console.log('nodejs was compiled without crypto support');
+    logger.log('nodejs was compiled without crypto support');
 }
 
 async function makeTestClient(userInfo, options) {
@@ -59,7 +61,7 @@ function sign(obj, key, userId) {
 
 describe("Secrets", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 
@@ -190,9 +192,9 @@ describe("Secrets", function() {
                 }),
             ]);
         };
-        alice.resetCrossSigningKeys();
+        resetCrossSigningKeys(alice);
 
-        const newKeyId = await alice.addSecretStorageKey(
+        const { keyId: newKeyId } = await alice.addSecretStorageKey(
             SECRET_STORAGE_ALGORITHM_V1_AES,
         );
         // we don't await on this because it waits for the event to come down the sync
@@ -224,8 +226,8 @@ describe("Secrets", function() {
             ],
             {
                 cryptoCallbacks: {
-                    onSecretRequested: e => {
-                        expect(e.name).toBe("foo");
+                    onSecretRequested: (userId, deviceId, requestId, secretName, deviceTrust) => {
+                        expect(secretName).toBe("foo");
                         return "bar";
                     },
                 },
@@ -288,17 +290,11 @@ describe("Secrets", function() {
             olmlib.decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="),
         );
         const SSPubKey = "0DfNsRDzEvkCLA0gD3m7VAGJ5VClhjEsewI35xq873Q";
-        const backupKey = new Uint8Array(
-            olmlib.decodeBase64(
-                "OTmOKrPsWP1qxmt0s2qat4vYxULG+U9cmFAWFco3A5U=",
-            ),
-        );
         const SSSSKey = new Uint8Array(
             olmlib.decodeBase64(
                 "XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0=",
             ),
         );
-        const SSSSPubKey = "v3A8HTypbccUm6jaRCRw5l+7lSdzQUACeY9xpQ5BVmE";
 
         it("bootstraps when no storage or cross-signing keys locally", async function() {
             const key = new Uint8Array(16);
@@ -331,7 +327,12 @@ describe("Secrets", function() {
                 this.emit("accountData", event);
             };
 
-            await bob.bootstrapSecretStorage();
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey,
+            });
 
             const crossSigning = bob._crypto._crossSigningInfo;
             const secretStorage = bob._crypto._secretStorage;
@@ -381,6 +382,9 @@ describe("Secrets", function() {
             const secretStorage = bob._crypto._secretStorage;
 
             // Set up cross-signing keys from scratch with specific storage key
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
             await bob.bootstrapSecretStorage({
                 createSecretStorageKey: async () => ({
                     // `pubkey` not used anymore with symmetric 4S
@@ -395,7 +399,9 @@ describe("Secrets", function() {
                 crossSigning.toStorage(),
             );
             crossSigning.keys = {};
-            await bob.bootstrapSecretStorage();
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
 
             expect(crossSigning.getId()).toBeTruthy();
             expect(await crossSigning.isStoredInSecretStorage(secretStorage))
@@ -403,319 +409,6 @@ describe("Secrets", function() {
             expect(await secretStorage.hasKey()).toBeTruthy();
         });
 
-        it("converts asymmetric SSSS to symmetric SSSS", async function() {
-            let crossSigningKeys = {};
-            const secretStorageKeys = {
-                "old_key_id": SSSSKey,
-                "new_key_id": SSSSKey,
-            };
-            const alice = await makeTestClient(
-                {userId: "@alice:example.com", deviceId: "Osborne2"},
-                {
-                    cryptoCallbacks: {
-                        getCrossSigningKey: t => crossSigningKeys[t],
-                        saveCrossSigningKeys: k => crossSigningKeys = k,
-                        getSecretStorageKey: ({keys}, name) => {
-                            for (const keyId of Object.keys(keys)) {
-                                if (secretStorageKeys[keyId]) {
-                                    return [keyId, secretStorageKeys[keyId]];
-                                }
-                            }
-                        },
-                    },
-                },
-            );
-            const encryption = new global.Olm.PkEncryption();
-            encryption.set_recipient_key(SSSSPubKey);
-            alice.store.storeAccountDataEvents([
-                new MatrixEvent({
-                    type: "m.secret_storage.default_key",
-                    content: {
-                        key: "old_key_id",
-                    },
-                }),
-                new MatrixEvent({
-                    type: "m.secret_storage.key.old_key_id",
-                    content: sign({
-                        algorithm: "m.secret_storage.v1.curve25519-aes-sha2",
-                        passphrase: {
-                            algorithm: "m.pbkdf2",
-                            iterations: 500000,
-                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
-                        },
-                        pubkey: SSSSPubKey,
-                    }, XSK, "@alice:example.com"),
-                }),
-                new MatrixEvent({
-                    type: "m.cross_signing.master",
-                    content: {
-                        encrypted: {
-                            old_key_id: encryption.encrypt(olmlib.encodeBase64(XSK)),
-                        },
-                    },
-                }),
-                new MatrixEvent({
-                    type: "m.cross_signing.self_signing",
-                    content: {
-                        encrypted: {
-                            old_key_id: encryption.encrypt(olmlib.encodeBase64(SSK)),
-                        },
-                    },
-                }),
-                new MatrixEvent({
-                    type: "m.cross_signing.user_signing",
-                    content: {
-                        encrypted: {
-                            old_key_id: encryption.encrypt(olmlib.encodeBase64(USK)),
-                        },
-                    },
-                }),
-                new MatrixEvent({
-                    type: "m.megolm_backup.v1",
-                    content: {
-                        encrypted: {
-                            old_key_id: encryption.encrypt(
-                                olmlib.encodeBase64(backupKey),
-                            ),
-                        },
-                    },
-                }),
-            ]);
-            encryption.free();
-            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
-                keys: {
-                    master: {
-                        user_id: "@alice:example.com",
-                        usage: ["master"],
-                        keys: {
-                            [`ed25519:${XSPubKey}`]: XSPubKey,
-                        },
-                    },
-                    self_signing: sign({
-                        user_id: "@alice:example.com",
-                        usage: ["self_signing"],
-                        keys: {
-                            [`ed25519:${SSPubKey}`]: SSPubKey,
-                        },
-                    }, XSK, "@alice:example.com"),
-                    user_signing: sign({
-                        user_id: "@alice:example.com",
-                        usage: ["user_signing"],
-                        keys: {
-                            [`ed25519:${USPubKey}`]: USPubKey,
-                        },
-                    }, XSK, "@alice:example.com"),
-                },
-            });
-            alice.getKeyBackupVersion = async () => {
-                return {
-                    version: "1",
-                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
-                    auth_data: sign({
-                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
-                    }, XSK, "@alice:example.com"),
-                };
-            };
-            const origAddSecretStorageKey = alice._crypto.addSecretStorageKey;
-            alice._crypto.addSecretStorageKey = async function(algorithm, opts, keyId) {
-                return await origAddSecretStorageKey.call(
-                    alice._crypto, algorithm, opts, keyId || "new_key_id",
-                );
-            };
-            alice.setAccountData = async function(name, data) {
-                const event = new MatrixEvent({
-                    type: name,
-                    content: data,
-                });
-                alice.store.storeAccountDataEvents([event]);
-                this.emit("accountData", event);
-            };
-
-            await alice.bootstrapSecretStorage();
-
-            // it should create a new key with the same parameters
-            expect(alice.getAccountData("m.secret_storage.default_key").getContent())
-                .toEqual({key: "new_key_id"});
-            const newKeyInfo = alice.getAccountData("m.secret_storage.key.new_key_id")
-                .getContent();
-            expect(newKeyInfo.algorithm)
-                .toEqual("m.secret_storage.v1.aes-hmac-sha2");
-            expect(newKeyInfo.passphrase).toEqual({
-                algorithm: "m.pbkdf2",
-                iterations: 500000,
-                salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
-            });
-            expect(newKeyInfo).toHaveProperty("iv");
-            expect(newKeyInfo).toHaveProperty("mac");
-            expect(alice.checkSecretStorageKey(secretStorageKeys.new_key_id, newKeyInfo))
-                .toBeTruthy();
-
-            const crossSigningMaster = alice.getAccountData("m.cross_signing.master")
-                .getContent();
-            expect(crossSigningMaster.encrypted).toHaveProperty("new_key_id");
-            expect(crossSigningMaster.encrypted).not.toHaveProperty("old_key_id");
-            expect(await alice.getSecret("m.cross_signing.master"))
-                .toEqual(olmlib.encodeBase64(XSK));
-
-            const USKInfo = alice.getAccountData("m.cross_signing.self_signing")
-                .getContent();
-            expect(USKInfo.encrypted).toHaveProperty("new_key_id");
-            expect(USKInfo.encrypted).not.toHaveProperty("old_key_id");
-            expect(await alice.getSecret("m.cross_signing.user_signing"))
-                .toEqual(olmlib.encodeBase64(USK));
-
-            const SSKInfo = alice.getAccountData("m.cross_signing.self_signing")
-                .getContent();
-            expect(SSKInfo.encrypted).toHaveProperty("new_key_id");
-            expect(SSKInfo.encrypted).not.toHaveProperty("old_key_id");
-            expect(await alice.getSecret("m.cross_signing.self_signing"))
-                .toEqual(olmlib.encodeBase64(SSK));
-
-            const backupKeyInfo = alice.getAccountData("m.megolm_backup.v1")
-                .getContent();
-            expect(backupKeyInfo.encrypted).toHaveProperty("new_key_id");
-            expect(backupKeyInfo.encrypted).not.toHaveProperty("old_key_id");
-            expect(await alice.getSecret("m.megolm_backup.v1"))
-                .toEqual(olmlib.encodeBase64(backupKey));
-        });
-        it("converts asymmetric SSSS with passthrough", async function() {
-            let crossSigningKeys = {};
-            const secretStorageKeys = {
-                "old_key_id": SSSSKey,
-                "new_key_id": SSSSKey,
-            };
-            const alice = await makeTestClient(
-                {userId: "@alice:example.com", deviceId: "Osborne2"},
-                {
-                    cryptoCallbacks: {
-                        getCrossSigningKey: t => crossSigningKeys[t],
-                        saveCrossSigningKeys: k => crossSigningKeys = k,
-                        getSecretStorageKey: ({keys}, name) => {
-                            for (const keyId of Object.keys(keys)) {
-                                if (secretStorageKeys[keyId]) {
-                                    return [keyId, secretStorageKeys[keyId]];
-                                }
-                            }
-                        },
-                    },
-                },
-            );
-            const encryption = new global.Olm.PkEncryption();
-            encryption.set_recipient_key(SSSSPubKey);
-            alice.store.storeAccountDataEvents([
-                new MatrixEvent({
-                    type: "m.secret_storage.default_key",
-                    content: {
-                        key: "old_key_id",
-                    },
-                }),
-                new MatrixEvent({
-                    type: "m.secret_storage.key.old_key_id",
-                    content: sign({
-                        algorithm: "m.secret_storage.v1.curve25519-aes-sha2",
-                        passphrase: {
-                            algorithm: "m.pbkdf2",
-                            iterations: 500000,
-                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
-                        },
-                        pubkey: "v3A8HTypbccUm6jaRCRw5l+7lSdzQUACeY9xpQ5BVmE",
-                    }, XSK, "@alice:example.com"),
-                }),
-                new MatrixEvent({
-                    type: "m.cross_signing.master",
-                    content: {
-                        encrypted: {
-                            old_key_id: encryption.encrypt(olmlib.encodeBase64(XSK)),
-                        },
-                    },
-                }),
-                new MatrixEvent({
-                    type: "m.cross_signing.self_signing",
-                    content: {
-                        encrypted: {
-                            old_key_id: encryption.encrypt(olmlib.encodeBase64(SSK)),
-                        },
-                    },
-                }),
-                new MatrixEvent({
-                    type: "m.cross_signing.user_signing",
-                    content: {
-                        encrypted: {
-                            old_key_id: encryption.encrypt(olmlib.encodeBase64(USK)),
-                        },
-                    },
-                }),
-                new MatrixEvent({
-                    type: "m.megolm_backup.v1",
-                    content: {
-                        encrypted: {
-                            old_key_id: {
-                                passthrough: true,
-                            },
-                        },
-                    },
-                }),
-            ]);
-            encryption.free();
-            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
-                keys: {
-                    master: {
-                        user_id: "@alice:example.com",
-                        usage: ["master"],
-                        keys: {
-                            [`ed25519:${XSPubKey}`]: XSPubKey,
-                        },
-                    },
-                    self_signing: sign({
-                        user_id: "@alice:example.com",
-                        usage: ["self_signing"],
-                        keys: {
-                            [`ed25519:${SSPubKey}`]: SSPubKey,
-                        },
-                    }, XSK, "@alice:example.com"),
-                    user_signing: sign({
-                        user_id: "@alice:example.com",
-                        usage: ["user_signing"],
-                        keys: {
-                            [`ed25519:${USPubKey}`]: USPubKey,
-                        },
-                    }, XSK, "@alice:example.com"),
-                },
-            });
-            alice.getKeyBackupVersion = async () => {
-                return {
-                    version: "1",
-                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
-                    auth_data: sign({
-                        public_key: "v3A8HTypbccUm6jaRCRw5l+7lSdzQUACeY9xpQ5BVmE",
-                    }, XSK, "@alice:example.com"),
-                };
-            };
-            const origAddSecretStorageKey = alice._crypto.addSecretStorageKey;
-            alice._crypto.addSecretStorageKey = async function(algorithm, opts, keyId) {
-                return await origAddSecretStorageKey.call(
-                    alice._crypto, algorithm, opts, keyId || "new_key_id",
-                );
-            };
-            alice.setAccountData = async function(name, data) {
-                const event = new MatrixEvent({
-                    type: name,
-                    content: data,
-                });
-                alice.store.storeAccountDataEvents([event]);
-                this.emit("accountData", event);
-            };
-
-            await alice.bootstrapSecretStorage();
-
-            // the new backup key should be the encoded SSSS key
-            const backupKeyInfo = alice.getAccountData("m.megolm_backup.v1")
-                .getContent();
-            expect(backupKeyInfo.encrypted).toHaveProperty("new_key_id");
-            expect(backupKeyInfo.encrypted).not.toHaveProperty("old_key_id");
-            expect(await alice.getSecret("m.megolm_backup.v1"))
-                .toEqual(olmlib.encodeBase64(SSSSKey));
-        });
         it("adds passphrase checking if it's lacking", async function() {
             let crossSigningKeys = {
                 master: XSK,

spec/unit/crypto/verification/sas.spec.js

@@ -22,6 +22,7 @@ import {DeviceInfo} from "../../../../src/crypto/deviceinfo";
 import {verificationMethods} from "../../../../src/crypto";
 import * as olmlib from "../../../../src/crypto/olmlib";
 import {logger} from "../../../../src/logger";
+import {resetCrossSigningKeys} from "../crypto-utils";
 
 const Olm = global.Olm;
 
@@ -181,11 +182,14 @@ describe("SAS verification", function() {
 
         it("should verify a key", async () => {
             let macMethod;
+            let keyAgreement;
             const origSendToDevice = bob.client.sendToDevice.bind(bob.client);
             bob.client.sendToDevice = function(type, map) {
                 if (type === "m.key.verification.accept") {
                     macMethod = map[alice.client.getUserId()][alice.client.deviceId]
                         .message_authentication_code;
+                    keyAgreement = map[alice.client.getUserId()][alice.client.deviceId]
+                        .key_agreement_protocol;
                 }
                 return origSendToDevice(type, map);
             };
@@ -212,6 +216,7 @@ describe("SAS verification", function() {
 
             // make sure that it uses the preferred method
             expect(macMethod).toBe("hkdf-hmac-sha256");
+            expect(keyAgreement).toBe("curve25519-hkdf-sha256");
 
             // make sure Alice and Bob verified each other
             const bobDevice
@@ -284,12 +289,12 @@ describe("SAS verification", function() {
             );
             alice.httpBackend.when('POST', '/keys/signatures/upload').respond(200, {});
             alice.httpBackend.flush(undefined, 2);
-            await alice.client.resetCrossSigningKeys();
+            await resetCrossSigningKeys(alice.client);
             bob.httpBackend.when('POST', '/keys/device_signing/upload').respond(200, {});
             bob.httpBackend.when('POST', '/keys/signatures/upload').respond(200, {});
             bob.httpBackend.flush(undefined, 2);
 
-            await bob.client.resetCrossSigningKeys();
+            await resetCrossSigningKeys(bob.client);
 
             bob.client._crypto._deviceList.storeCrossSigningForUser(
                 "@alice:example.com", {

spec/unit/crypto/verification/secret_request.spec.js

@@ -54,6 +54,7 @@ describe("self-verifications", () => {
             cacheCallbacks,
         );
         _crossSigningInfo.keys = {
+            master: { keys: { X: testKeyPub } },
             self_signing: { keys: { X: testKeyPub } },
             user_signing: { keys: { X: testKeyPub } },
         };
@@ -96,9 +97,9 @@ describe("self-verifications", () => {
 
         const result = await verification.done();
 
-        /* We should request, and store, two cross signing key and the key backup key */
-        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls.length).toBe(2);
-        expect(_secretStorage.request.mock.calls.length).toBe(3);
+        /* We should request, and store, 3 cross signing keys and the key backup key */
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls.length).toBe(3);
+        expect(_secretStorage.request.mock.calls.length).toBe(4);
 
         expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[0][1])
           .toEqual(testKey);

spec/unit/crypto/verification/util.js

@@ -18,12 +18,13 @@ limitations under the License.
 import {TestClient} from '../../../TestClient';
 import {MatrixEvent} from "../../../../src/models/event";
 import nodeCrypto from "crypto";
+import {logger} from '../../../../src/logger';
 
 export async function makeTestClients(userInfos, options) {
     const clients = [];
     const clientMap = {};
     const sendToDevice = function(type, map) {
-        // console.log(this.getUserId(), "sends", type, map);
+        // logger.log(this.getUserId(), "sends", type, map);
         for (const [userId, devMap] of Object.entries(map)) {
             if (userId in clientMap) {
                 for (const [deviceId, msg] of Object.entries(devMap)) {
@@ -67,7 +68,7 @@ export async function makeTestClients(userInfos, options) {
         setImmediate(() => {
             for (const tc of clients) {
                 if (tc.client === this) { // eslint-disable-line babel/no-invalid-this
-                    console.log("sending remote echo!!");
+                    logger.log("sending remote echo!!");
                     tc.client.emit("Room.timeline", remoteEcho);
                 } else {
                     tc.client.emit("Room.timeline", event);

spec/unit/crypto/verification/verification_request.spec.js

@@ -119,6 +119,8 @@ async function distributeEvent(ownRequest, theirRequest, event) {
     await theirRequest.channel.handleEvent(event, theirRequest, true);
 }
 
+jest.useFakeTimers();
+
 describe("verification request unit tests", function() {
     beforeAll(function() {
         setupWebcrypto();
@@ -246,4 +248,38 @@ describe("verification request unit tests", function() {
         expect(bob1Request.done).toBe(true);
         expect(bob2Request.done).toBe(true);
     });
+
+    it("request times out after 10 minutes", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        await aliceRequest.channel.handleEvent(requestEvent, aliceRequest, true,
+            true, true);
+
+        expect(aliceRequest.cancelled).toBe(false);
+        expect(aliceRequest._cancellingUserId).toBe(undefined);
+        jest.advanceTimersByTime(10 * 60 * 1000);
+        expect(aliceRequest._cancellingUserId).toBe(alice.getUserId());
+    });
+
+    it("request times out 2 minutes after receipt", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        const bobRequest = new VerificationRequest(
+            new InRoomChannel(bob, "!room"), new Map(), bob);
+
+        await bobRequest.channel.handleEvent(requestEvent, bobRequest, true);
+
+        expect(bobRequest.cancelled).toBe(false);
+        expect(bobRequest._cancellingUserId).toBe(undefined);
+        jest.advanceTimersByTime(2 * 60 * 1000);
+        expect(bobRequest._cancellingUserId).toBe(bob.getUserId());
+    });
 });

spec/unit/filter-component.spec.js

@@ -0,0 +1,34 @@
+import {FilterComponent} from "../../src/filter-component";
+import {mkEvent} from '../test-utils';
+
+describe("Filter Component", function() {
+    describe("types", function() {
+        it("should filter out events with other types", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.member',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(false);
+        });
+
+        it("should validate events with the same type", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.message',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(true);
+        });
+    });
+});

spec/unit/interactive-auth.spec.js

@@ -97,7 +97,7 @@ describe("InteractiveAuth", function() {
         // first we expect a call to doRequest
         doRequest.mockImplementation(function(authData) {
             logger.log("request1", authData);
-            expect(authData).toEqual({});
+            expect(authData).toEqual(null); // first request should be null
             const err = new MatrixError({
                 session: "sessionId",
                 flows: [
@@ -156,7 +156,7 @@ describe("InteractiveAuth", function() {
 
         doRequest.mockImplementation(function(authData) {
             logger.log("request1", authData);
-            expect(authData).toEqual({});
+            expect(authData).toEqual(null); // first request should be null
             const err = new MatrixError({
                 session: "sessionId",
                 flows: [],

spec/unit/matrix-client.spec.js

@@ -521,4 +521,19 @@ describe("MatrixClient", function() {
         xit("should be able to peek into a room using peekInRoom", function(done) {
         });
     });
+
+    describe("getPresence", function() {
+        it("should send a presence HTTP GET", function() {
+            httpLookups = [{
+                method: "GET",
+                path: `/presence/${encodeURIComponent(userId)}/status`,
+                data: {
+                    "presence": "unavailable",
+                    "last_active_ago": 420845,
+                },
+            }];
+            client.getPresence(userId);
+            expect(httpLookups.length).toEqual(0);
+        });
+    });
 });

spec/unit/room-member.spec.js

@@ -33,12 +33,6 @@ describe("RoomMember", function() {
             expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
         });
 
-        it("should return an identicon HTTP URL if allowDefault was set and there " +
-        "was no m.room.member event", function() {
-            const url = member.getAvatarUrl(hsUrl, 64, 64, "crop", true);
-            expect(url.indexOf("http")).toEqual(0); // don't care about form
-        });
-
         it("should return nothing if there is no m.room.member and allowDefault=false",
         function() {
             const url = member.getAvatarUrl(hsUrl, 64, 64, "crop", false);

spec/unit/room.spec.js

@@ -45,12 +45,6 @@ describe("Room", function() {
             expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
         });
 
-        it("should return an identicon HTTP URL if allowDefault was set and there " +
-        "was no m.room.avatar event", function() {
-            const url = room.getAvatarUrl(hsUrl, 64, 64, "crop", true);
-            expect(url.indexOf("http")).toEqual(0); // don't care about form
-        });
-
         it("should return nothing if there is no m.room.avatar and allowDefault=false",
         function() {
             const url = room.getAvatarUrl(hsUrl, 64, 64, "crop", false);
@@ -1379,7 +1373,7 @@ describe("Room", function() {
             let hasThrown = false;
             try {
                 await room.loadMembersIfNeeded();
-            } catch(err) {
+            } catch (err) {
                 hasThrown = true;
             }
             expect(hasThrown).toEqual(true);

spec/unit/scheduler.spec.js

@@ -143,7 +143,7 @@ describe("MatrixScheduler", function() {
         deferA.reject({});
         try {
             await globalA;
-        } catch(err) {
+        } catch (err) {
             await Promise.resolve();
             expect(procCount).toEqual(2);
         }

spec/unit/sync-accumulator.spec.js

@@ -17,6 +17,39 @@ limitations under the License.
 
 import {SyncAccumulator} from "../../src/sync-accumulator";
 
+// The event body & unsigned object get frozen to assert that they don't get altered
+// by the impl
+const RES_WITH_AGE = {
+    next_batch: "abc",
+    rooms: {
+        invite: {},
+        leave: {},
+        join: {
+            "!foo:bar": {
+                account_data: { events: [] },
+                ephemeral: { events: [] },
+                unread_notifications: {},
+                timeline: {
+                    events: [
+                        Object.freeze({
+                            content: {
+                                body: "This thing is happening right now!",
+                            },
+                            origin_server_ts: 123456789,
+                            sender: "@alice:localhost",
+                            type: "m.room.message",
+                            unsigned: Object.freeze({
+                                age: 50,
+                            }),
+                        }),
+                    ],
+                    prev_batch: "something",
+                },
+            },
+        },
+    },
+};
+
 describe("SyncAccumulator", function() {
     let sa;
 
@@ -368,6 +401,39 @@ describe("SyncAccumulator", function() {
             expect(summary["m.joined_member_count"]).toEqual(5);
             expect(summary["m.heroes"]).toEqual(["@bob:bar"]);
         });
+
+        it("should return correctly adjusted age attributes", () => {
+            const delta = 1000;
+            const startingTs = 1000;
+
+            const oldDateNow = Date.now;
+            try {
+                Date.now = jest.fn();
+                Date.now.mockReturnValue(startingTs);
+
+                sa.accumulate(RES_WITH_AGE);
+
+                Date.now.mockReturnValue(startingTs + delta);
+
+                const output = sa.getJSON();
+                expect(output.roomsData.join["!foo:bar"].timeline.events[0].unsigned.age).toEqual(
+                    RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0].unsigned.age + delta,
+                );
+                expect(Object.keys(output.roomsData.join["!foo:bar"].timeline.events[0])).toEqual(
+                    Object.keys(RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0]),
+                );
+            } finally {
+                Date.now = oldDateNow;
+            }
+        });
+
+        it("should mangle age without adding extra keys", () => {
+            sa.accumulate(RES_WITH_AGE);
+            const output = sa.getJSON();
+            expect(Object.keys(output.roomsData.join["!foo:bar"].timeline.events[0])).toEqual(
+                Object.keys(RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0]),
+            );
+        });
     });
 });
 

spec/unit/webrtc/call.spec.ts

@@ -0,0 +1,193 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {TestClient} from '../../TestClient';
+import {MatrixCall, CallErrorCode} from '../../../src/webrtc/call';
+
+const DUMMY_SDP = (
+    "v=0\r\n" +
+    "o=- 5022425983810148698 2 IN IP4 127.0.0.1\r\n" +
+    "s=-\r\nt=0 0\r\na=group:BUNDLE 0\r\n" +
+    "a=msid-semantic: WMS h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA\r\n" +
+    "m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126\r\n" +
+    "c=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:hLDR\r\n" +
+    "a=ice-pwd:bMGD9aOldHWiI+6nAq/IIlRw\r\n" +
+    "a=ice-options:trickle\r\n" +
+    "a=fingerprint:sha-256 E4:94:84:F9:4A:98:8A:56:F5:5F:FD:AF:72:B9:32:89:49:5C:4B:9A:" +
+        "4A:15:8E:41:8A:F3:69:E4:39:52:DC:D6\r\n" +
+    "a=setup:active\r\n" +
+    "a=mid:0\r\n" +
+    "a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n" +
+    "a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n" +
+    "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n" +
+    "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n" +
+    "a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n" +
+    "a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n" +
+    "a=sendrecv\r\n" +
+    "a=msid:h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA 4357098f-3795-4131-bff4-9ba9c0348c49\r\n" +
+    "a=rtcp-mux\r\n" +
+    "a=rtpmap:111 opus/48000/2\r\n" +
+    "a=rtcp-fb:111 transport-cc\r\n" +
+    "a=fmtp:111 minptime=10;useinbandfec=1\r\n" +
+    "a=rtpmap:103 ISAC/16000\r\n" +
+    "a=rtpmap:104 ISAC/32000\r\n" +
+    "a=rtpmap:9 G722/8000\r\n" +
+    "a=rtpmap:0 PCMU/8000\r\n" +
+    "a=rtpmap:8 PCMA/8000\r\n" +
+    "a=rtpmap:106 CN/32000\r\n" +
+    "a=rtpmap:105 CN/16000\r\n" +
+    "a=rtpmap:13 CN/8000\r\n" +
+    "a=rtpmap:110 telephone-event/48000\r\n" +
+    "a=rtpmap:112 telephone-event/32000\r\n" +
+    "a=rtpmap:113 telephone-event/16000\r\n" +
+    "a=rtpmap:126 telephone-event/8000\r\n" +
+    "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n"
+);
+
+class MockRTCPeerConnection {
+    localDescription: RTCSessionDescription;
+
+    constructor() {
+        this.localDescription = {
+            sdp: DUMMY_SDP,
+            type: 'offer',
+            toJSON: function() {},
+        };
+    }
+
+    addEventListener() {}
+    createOffer() {
+        return Promise.resolve({});
+    }
+    setRemoteDescription() {
+        return Promise.resolve();
+    }
+    setLocalDescription() {
+        return Promise.resolve();
+    }
+    close() {}
+}
+
+describe('Call', function() {
+    let client;
+    let call;
+    let prevNavigator;
+    let prevDocument;
+    let prevWindow;
+
+    beforeEach(function() {
+        prevNavigator = global.navigator;
+        prevDocument = global.document;
+        prevWindow = global.window;
+
+        global.navigator = {
+            mediaDevices: {
+                // @ts-ignore Mock
+                getUserMedia: () => {
+                    return {
+                        getTracks: () => [],
+                        getAudioTracks: () => [],
+                        getVideoTracks: () => [],
+                    };
+                },
+            },
+        };
+
+        global.window = {
+            // @ts-ignore Mock
+            RTCPeerConnection: MockRTCPeerConnection,
+            // @ts-ignore Mock
+            RTCSessionDescription: {},
+            // @ts-ignore Mock
+            RTCIceCandidate: {},
+            getUserMedia: {},
+        };
+        // @ts-ignore Mock
+        global.document = {};
+
+        client = new TestClient("@alice:foo", "somedevice", "token", undefined, {});
+        // We just stub out sendEvent: we're not interested in testing the client's
+        // event sending code here
+        client.client.sendEvent = () => {};
+        call = new MatrixCall({
+            client: client.client,
+            roomId: '!foo:bar',
+        });
+        // call checks one of these is wired up
+        call.on('error', () => {});
+    });
+
+    afterEach(function() {
+        client.stop();
+        global.navigator = prevNavigator;
+        global.window = prevWindow;
+        global.document = prevDocument;
+    });
+
+    it('should ignore candidate events from non-matching party ID', async function() {
+        await call.placeVoiceCall();
+        await call.onAnswerReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    answer: {
+                        sdp: DUMMY_SDP,
+                    },
+                };
+            },
+        });
+
+        call.peerConn.addIceCandidate = jest.fn();
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    candidates: [
+                        {
+                            candidate: '',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1);
+
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'some_other_party_id',
+                    candidates: [
+                        {
+                            candidate: '',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1);
+
+        // Hangup to stop timers
+        call.hangup(CallErrorCode.UserHangup, true);
+    });
+});

src/@types/event.ts

@@ -0,0 +1,89 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export enum EventType {
+    // Room state events
+    RoomCanonicalAlias = "m.room.canonical_alias",
+    RoomCreate = "m.room.create",
+    RoomJoinRules = "m.room.join_rules",
+    RoomMember = "m.room.member",
+    RoomThirdPartyInvite = "m.room.third_party_invite",
+    RoomPowerLevels = "m.room.power_levels",
+    RoomName = "m.room.name",
+    RoomTopic = "m.room.topic",
+    RoomAvatar = "m.room.avatar",
+    RoomPinnedEvents = "m.room.pinned_events",
+    RoomEncryption = "m.room.encryption",
+    RoomHistoryVisibility = "m.room.history_visibility",
+    RoomGuestAccess = "m.room.guest_access",
+    RoomServerAcl = "m.room.server_acl",
+    RoomTombstone = "m.room.tombstone",
+    /**
+     * @deprecated Should not be used.
+     */
+    RoomAliases = "m.room.aliases", // deprecated https://matrix.org/docs/spec/client_server/r0.6.1#historical-events
+
+    // Room timeline events
+    RoomRedaction = "m.room.redaction",
+    RoomMessage = "m.room.message",
+    RoomMessageEncrypted = "m.room.encrypted",
+    Sticker = "m.sticker",
+    CallInvite = "m.call.invite",
+    CallCandidates = "m.call.candidates",
+    CallAnswer = "m.call.answer",
+    CallHangup = "m.call.hangup",
+    CallReject = "m.call.reject",
+    CallSelectAnswer = "m.call.select_answer",
+    CallNegotiate = "m.call.negotiate",
+    CallReplaces = "m.call.replaces",
+    KeyVerificationRequest = "m.key.verification.request",
+    KeyVerificationStart = "m.key.verification.start",
+    KeyVerificationCancel = "m.key.verification.cancel",
+    KeyVerificationMac = "m.key.verification.mac",
+    // use of this is discouraged https://matrix.org/docs/spec/client_server/r0.6.1#m-room-message-feedback
+    RoomMessageFeedback = "m.room.message.feedback",
+
+    // Room ephemeral events
+    Typing = "m.typing",
+    Receipt = "m.receipt",
+    Presence = "m.presence",
+
+    // Room account_data events
+    FullyRead = "m.fully_read",
+    Tag = "m.tag",
+
+    // User account_data events
+    PushRules = "m.push_rules",
+    Direct = "m.direct",
+    IgnoredUserList = "m.ignored_user_list",
+
+    // to_device events
+    RoomKey = "m.room_key",
+    RoomKeyRequest = "m.room_key_request",
+    ForwardedRoomKey = "m.forwarded_room_key",
+    Dummy = "m.dummy",
+}
+
+export enum MsgType {
+    Text = "m.text",
+    Emote = "m.emote",
+    Notice = "m.notice",
+    Image = "m.image",
+    File = "m.file",
+    Audio = "m.audio",
+    Location = "m.location",
+    Video = "m.video",
+}

src/@types/global.d.ts

@@ -0,0 +1,48 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// this is needed to tell TS about global.Olm
+import * as Olm from "olm"; // eslint-disable-line @typescript-eslint/no-unused-vars
+
+export {};
+
+declare global {
+    namespace NodeJS {
+        interface Global {
+            localStorage: Storage;
+        }
+    }
+
+    interface MediaDevices {
+        // This is experimental and types don't know about it yet
+        // https://github.com/microsoft/TypeScript/issues/33232
+        getDisplayMedia(constraints: MediaStreamConstraints): Promise<MediaStream>;
+    }
+
+    interface HTMLAudioElement {
+        // sinkId & setSinkId are experimental and typescript doesn't know about them
+        sinkId: string;
+        setSinkId(outputId: string);
+    }
+
+    interface DummyInterfaceWeShouldntBeUsingThis {}
+
+    interface Navigator {
+        // We check for the webkit-prefixed getUserMedia to detect if we're
+        // on webkit: we should check if we still need to do this
+        webkitGetUserMedia: DummyInterfaceWeShouldntBeUsingThis;
+    }
+}

src/base-apis.js

@@ -66,7 +66,7 @@ function termsUrlForService(serviceType, baseUrl) {
  * callback that returns a Promise<String> of an identity access token to supply
  * with identity requests. If the object is unset, no access token will be
  * supplied.
- * See also https://github.com/vector-im/riot-web/issues/10615 which seeks to
+ * See also https://github.com/vector-im/element-web/issues/10615 which seeks to
  * replace the previous approach of manual access tokens params with this
  * callback throughout the SDK.
  *
@@ -206,9 +206,6 @@ MatrixBaseApis.prototype.register = function(
         inhibitLogin = undefined;
     }
 
-    if (auth === undefined || auth === null) {
-        auth = {};
-    }
     if (sessionId) {
         auth.session = sessionId;
     }
@@ -359,15 +356,22 @@ MatrixBaseApis.prototype.getCasLoginUrl = function(redirectUrl) {
  *     authenticates with the SSO.
  * @param {string} loginType The type of SSO login we are doing (sso or cas).
  *     Defaults to 'sso'.
+ * @param {string} idpId The ID of the Identity Provider being targeted, optional.
  * @return {string} The HS URL to hit to begin the SSO login process.
  */
-MatrixBaseApis.prototype.getSsoLoginUrl = function(redirectUrl, loginType) {
+MatrixBaseApis.prototype.getSsoLoginUrl = function(redirectUrl, loginType, idpId) {
     if (loginType === undefined) {
         loginType = "sso";
     }
-    return this._http.getUrl("/login/"+loginType+"/redirect", {
-        "redirectUrl": redirectUrl,
-    }, PREFIX_R0);
+
+    let prefix = PREFIX_R0;
+    let url = "/login/" + loginType + "/redirect";
+    if (idpId) {
+        url += "/" + idpId;
+        prefix = "/_matrix/client/unstable/org.matrix.msc2858";
+    }
+
+    return this._http.getUrl(url, { redirectUrl }, prefix);
 };
 
 /**
@@ -465,8 +469,26 @@ MatrixBaseApis.prototype.getFallbackAuthUrl = function(loginType, authSessionId)
  * room_alias: {string(opt)}}</code>
  * @return {module:http-api.MatrixError} Rejects: with an error response.
  */
-MatrixBaseApis.prototype.createRoom = function(options, callback) {
-    // valid options include: room_alias_name, visibility, invite
+MatrixBaseApis.prototype.createRoom = async function(options, callback) {
+    // some valid options include: room_alias_name, visibility, invite
+
+    // inject the id_access_token if inviting 3rd party addresses
+    const invitesNeedingToken = (options.invite_3pid || [])
+        .filter(i => !i.id_access_token);
+    if (
+        invitesNeedingToken.length > 0 &&
+        this.identityServer &&
+        this.identityServer.getAccessToken &&
+        await this.doesServerAcceptIdentityAccessToken()
+    ) {
+        const identityAccessToken = await this.identityServer.getAccessToken();
+        if (identityAccessToken) {
+            for (const invite of invitesNeedingToken) {
+                invite.id_access_token = identityAccessToken;
+            }
+        }
+    }
+
     return this._http.authedRequest(
         callback, "POST", "/createRoom", undefined, options,
     );

src/browser-index.js

@@ -34,7 +34,7 @@ matrixcs.request(function(opts, fn) {
 let indexedDB;
 try {
     indexedDB = global.indexedDB;
-} catch(e) {}
+} catch (e) {}
 
 // if our browser (appears to) support indexeddb, use an indexeddb crypto store.
 if (indexedDB) {

src/content-repo.js

@@ -75,35 +75,3 @@ export function getHttpUriForMxc(baseUrl, mxc, width, height,
         (utils.keys(params).length === 0 ? "" :
         ("?" + utils.encodeParams(params))) + fragment;
 }
-
-/**
- * Get an identicon URL from an arbitrary string.
- * @param {string} baseUrl The base homeserver url which has a content repo.
- * @param {string} identiconString The string to create an identicon for.
- * @param {Number} width The desired width of the image in pixels. Default: 96.
- * @param {Number} height The desired height of the image in pixels. Default: 96.
- * @return {string} The complete URL to the identicon.
- * @deprecated This is no longer in the specification.
- */
-export function getIdenticonUri(baseUrl, identiconString, width, height) {
-    if (!identiconString) {
-        return null;
-    }
-    if (!width) {
-        width = 96;
-    }
-    if (!height) {
-        height = 96;
-    }
-    const params = {
-        width: width,
-        height: height,
-    };
-
-    const path = utils.encodeUri("/_matrix/media/unstable/identicon/$ident", {
-        $ident: identiconString,
-    });
-    return baseUrl + path +
-        (utils.keys(params).length === 0 ? "" :
-            ("?" + utils.encodeParams(params)));
-}

src/crypto/CrossSigning.js

@@ -24,6 +24,9 @@ import {decodeBase64, encodeBase64, pkSign, pkVerify} from './olmlib';
 import {EventEmitter} from 'events';
 import {logger} from '../logger';
 import {IndexedDBCryptoStore} from '../crypto/store/indexeddb-crypto-store';
+import {decryptAES, encryptAES} from './aes';
+
+const KEY_REQUEST_TIMEOUT_MS = 1000 * 60;
 
 function publicKeyFromKeyInfo(keyInfo) {
     // `keys` is an object with { [`ed25519:${pubKey}`]: pubKey }
@@ -63,15 +66,34 @@ export class CrossSigningInfo extends EventEmitter {
         this.crossSigningVerifiedBefore = false;
     }
 
+    static fromStorage(obj, userId) {
+        const res = new CrossSigningInfo(userId);
+        for (const prop in obj) {
+            if (obj.hasOwnProperty(prop)) {
+                res[prop] = obj[prop];
+            }
+        }
+        return res;
+    }
+
+    toStorage() {
+        return {
+            keys: this.keys,
+            firstUse: this.firstUse,
+            crossSigningVerifiedBefore: this.crossSigningVerifiedBefore,
+        };
+    }
+
     /**
      * Calls the app callback to ask for a private key
+     *
      * @param {string} type The key type ("master", "self_signing", or "user_signing")
      * @param {string} expectedPubkey The matching public key or undefined to use
      *     the stored public key for the given key type.
      * @returns {Array} An array with [ public key, Olm.PkSigning ]
      */
     async getCrossSigningKey(type, expectedPubkey) {
-        const shouldCache = ["self_signing", "user_signing"].indexOf(type) >= 0;
+        const shouldCache = ["master", "self_signing", "user_signing"].indexOf(type) >= 0;
 
         if (!this._callbacks.getCrossSigningKey) {
             throw new Error("No getCrossSigningKey callback supplied");
@@ -124,24 +146,6 @@ export class CrossSigningInfo extends EventEmitter {
         );
     }
 
-    static fromStorage(obj, userId) {
-        const res = new CrossSigningInfo(userId);
-        for (const prop in obj) {
-            if (obj.hasOwnProperty(prop)) {
-                res[prop] = obj[prop];
-            }
-        }
-        return res;
-    }
-
-    toStorage() {
-        return {
-            keys: this.keys,
-            firstUse: this.firstUse,
-            crossSigningVerifiedBefore: this.crossSigningVerifiedBefore,
-        };
-    }
-
     /**
      * Check whether the private keys exist in secret storage.
      * XXX: This could be static, be we often seem to have an instance when we
@@ -177,12 +181,12 @@ export class CrossSigningInfo extends EventEmitter {
      * typically called in conjunction with the creation of new cross-signing
      * keys.
      *
-     * @param {object} keys The keys to store
+     * @param {Map} keys The keys to store
      * @param {SecretStorage} secretStorage The secret store using account data
      */
     static async storeInSecretStorage(keys, secretStorage) {
-        for (const type of Object.keys(keys)) {
-            const encodedKey = encodeBase64(keys[type]);
+        for (const [type, privateKey] of keys) {
+            const encodedKey = encodeBase64(privateKey);
             await secretStorage.store(`m.cross_signing.${type}`, encodedKey);
         }
     }
@@ -198,9 +202,50 @@ export class CrossSigningInfo extends EventEmitter {
      */
     static async getFromSecretStorage(type, secretStorage) {
         const encodedKey = await secretStorage.get(`m.cross_signing.${type}`);
+        if (!encodedKey) {
+            return null;
+        }
         return decodeBase64(encodedKey);
     }
 
+    /**
+     * Check whether the private keys exist in the local key cache.
+     *
+     * @param {string} [type] The type of key to get. One of "master",
+     * "self_signing", or "user_signing". Optional, will check all by default.
+     * @returns {boolean} True if all keys are stored in the local cache.
+     */
+    async isStoredInKeyCache(type) {
+        const cacheCallbacks = this._cacheCallbacks;
+        if (!cacheCallbacks) return false;
+        const types = type ? [type] : ["master", "self_signing", "user_signing"];
+        for (const t of types) {
+            if (!await cacheCallbacks.getCrossSigningKeyCache(t)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Get cross-signing private keys from the local cache.
+     *
+     * @returns {Map} A map from key type (string) to private key (Uint8Array)
+     */
+    async getCrossSigningKeysFromCache() {
+        const keys = new Map();
+        const cacheCallbacks = this._cacheCallbacks;
+        if (!cacheCallbacks) return keys;
+        for (const type of ["master", "self_signing", "user_signing"]) {
+            const privKey = await cacheCallbacks.getCrossSigningKeyCache(type);
+            if (!privKey) {
+                continue;
+            }
+            keys.set(type, privKey);
+        }
+        return keys;
+    }
+
     /**
      * Get the ID used to identify the user. This can also be used to test for
      * the existence of a given key type.
@@ -637,10 +682,10 @@ export class DeviceTrustLevel {
     }
 }
 
-export function createCryptoStoreCacheCallbacks(store) {
+export function createCryptoStoreCacheCallbacks(store, olmdevice) {
     return {
-        getCrossSigningKeyCache: function(type, _expectedPublicKey) {
-            return new Promise((resolve) => {
+        getCrossSigningKeyCache: async function(type, _expectedPublicKey) {
+            const key = await new Promise((resolve) => {
                 return store.doTxn(
                     'readonly',
                     [IndexedDBCryptoStore.STORE_ACCOUNT],
@@ -649,13 +694,23 @@ export function createCryptoStoreCacheCallbacks(store) {
                     },
                 );
             });
+
+            if (key && key.ciphertext) {
+                const pickleKey = Buffer.from(olmdevice._pickleKey);
+                const decrypted = await decryptAES(key, pickleKey, type);
+                return decodeBase64(decrypted);
+            } else {
+                return key;
+            }
         },
-        storeCrossSigningKeyCache: function(type, key) {
+        storeCrossSigningKeyCache: async function(type, key) {
             if (!(key instanceof Uint8Array)) {
                 throw new Error(
                     `storeCrossSigningKeyCache expects Uint8Array, got ${key}`,
                 );
             }
+            const pickleKey = Buffer.from(olmdevice._pickleKey);
+            key = await encryptAES(encodeBase64(key), pickleKey, type);
             return store.doTxn(
                 'readwrite',
                 [IndexedDBCryptoStore.STORE_ACCOUNT],
@@ -666,3 +721,96 @@ export function createCryptoStoreCacheCallbacks(store) {
         },
     };
 }
+
+/**
+ * Request cross-signing keys from another device during verification.
+ *
+ * @param {module:base-apis~MatrixBaseApis} baseApis base Matrix API interface
+ * @param {string} userId The user ID being verified
+ * @param {string} deviceId The device ID being verified
+ */
+export async function requestKeysDuringVerification(baseApis, userId, deviceId) {
+    // If this is a self-verification, ask the other party for keys
+    if (baseApis.getUserId() !== userId) {
+        return;
+    }
+    logger.log("Cross-signing: Self-verification done; requesting keys");
+    // This happens asynchronously, and we're not concerned about waiting for
+    // it.  We return here in order to test.
+    return new Promise((resolve, reject) => {
+        const client = baseApis;
+        const original = client._crypto._crossSigningInfo;
+
+        // We already have all of the infrastructure we need to validate and
+        // cache cross-signing keys, so instead of replicating that, here we set
+        // up callbacks that request them from the other device and call
+        // CrossSigningInfo.getCrossSigningKey() to validate/cache
+        const crossSigning = new CrossSigningInfo(
+            original.userId,
+            { getCrossSigningKey: async (type) => {
+                logger.debug("Cross-signing: requesting secret",
+                                type, deviceId);
+                const { promise } = client.requestSecret(
+                    `m.cross_signing.${type}`, [deviceId],
+                );
+                const result = await promise;
+                const decoded = decodeBase64(result);
+                return Uint8Array.from(decoded);
+            } },
+            original._cacheCallbacks,
+        );
+        crossSigning.keys = original.keys;
+
+        // XXX: get all keys out if we get one key out
+        // https://github.com/vector-im/element-web/issues/12604
+        // then change here to reject on the timeout
+        // Requests can be ignored, so don't wait around forever
+        const timeout = new Promise((resolve, reject) => {
+            setTimeout(
+                resolve,
+                KEY_REQUEST_TIMEOUT_MS,
+                new Error("Timeout"),
+            );
+        });
+
+        // also request and cache the key backup key
+        const backupKeyPromise = new Promise(async resolve => {
+            const cachedKey = await client._crypto.getSessionBackupPrivateKey();
+            if (!cachedKey) {
+                logger.info("No cached backup key found. Requesting...");
+                const secretReq = client.requestSecret(
+                    'm.megolm_backup.v1', [deviceId],
+                );
+                const base64Key = await secretReq.promise;
+                logger.info("Got key backup key, decoding...");
+                const decodedKey = decodeBase64(base64Key);
+                logger.info("Decoded backup key, storing...");
+                client._crypto.storeSessionBackupPrivateKey(
+                    Uint8Array.from(decodedKey),
+                );
+                logger.info("Backup key stored. Starting backup restore...");
+                const backupInfo = await client.getKeyBackupVersion();
+                // no need to await for this - just let it go in the bg
+                client.restoreKeyBackupWithCache(
+                    undefined, undefined, backupInfo,
+                ).then(() => {
+                    logger.info("Backup restored.");
+                });
+            }
+            resolve();
+        });
+
+        // We call getCrossSigningKey() for its side-effects
+        return Promise.race([
+            Promise.all([
+                crossSigning.getCrossSigningKey("master"),
+                crossSigning.getCrossSigningKey("self_signing"),
+                crossSigning.getCrossSigningKey("user_signing"),
+                backupKeyPromise,
+            ]),
+            timeout,
+        ]).then(resolve, reject);
+    }).catch((e) => {
+        logger.warn("Cross-signing: failure while requesting keys:", e);
+    });
+}

src/crypto/DeviceList.js

@@ -790,7 +790,7 @@ class DeviceListUpdateSerialiser {
 
             // yield to other things that want to execute in between users, to
             // avoid wedging the CPU
-            // (https://github.com/vector-im/riot-web/issues/3158)
+            // (https://github.com/vector-im/element-web/issues/3158)
             //
             // of course we ought to do this in a web worker or similar, but
             // this serves as an easy solution for now.
@@ -848,6 +848,7 @@ class DeviceListUpdateSerialiser {
 
             await _updateStoredDeviceKeysForUser(
                 this._olmDevice, userId, userStore, dkResponse || {},
+                this._baseApis.getUserId(), this._baseApis.deviceId,
             );
 
             // put the updates into the object that will be returned as our results
@@ -885,8 +886,9 @@ class DeviceListUpdateSerialiser {
 }
 
 
-async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
-        userResult) {
+async function _updateStoredDeviceKeysForUser(
+    _olmDevice, userId, userStore, userResult, localUserId, localDeviceId,
+) {
     let updated = false;
 
     // remove any devices in the store which aren't in the response
@@ -896,6 +898,13 @@ async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
         }
 
         if (!(deviceId in userResult)) {
+            if (userId === localUserId && deviceId === localDeviceId) {
+                logger.warn(
+                    `Local device ${deviceId} missing from sync, skipping removal`,
+                );
+                continue;
+            }
+
             logger.log("Device " + userId + ":" + deviceId +
                 " has been removed");
             delete userStore[deviceId];

src/crypto/EncryptionSetup.js

@@ -0,0 +1,357 @@
+import { logger } from "../logger";
+import {MatrixEvent} from "../models/event";
+import {EventEmitter} from "events";
+import {createCryptoStoreCacheCallbacks} from "./CrossSigning";
+import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
+import {
+    PREFIX_UNSTABLE,
+} from "../http-api";
+
+/**
+ * Builds an EncryptionSetupOperation by calling any of the add.. methods.
+ * Once done, `buildOperation()` can be called which allows to apply to operation.
+ *
+ * This is used as a helper by Crypto to keep track of all the network requests
+ * and other side-effects of bootstrapping, so it can be applied in one go (and retried in the future)
+ * Also keeps track of all the private keys created during bootstrapping, so we don't need to prompt for them
+ * more than once.
+ */
+export class EncryptionSetupBuilder {
+    /**
+     * @param {Object.<String, MatrixEvent>} accountData pre-existing account data, will only be read, not written.
+     * @param {CryptoCallbacks} delegateCryptoCallbacks crypto callbacks to delegate to if the key isn't in cache yet
+     */
+    constructor(accountData, delegateCryptoCallbacks) {
+        this.accountDataClientAdapter = new AccountDataClientAdapter(accountData);
+        this.crossSigningCallbacks = new CrossSigningCallbacks();
+        this.ssssCryptoCallbacks = new SSSSCryptoCallbacks(delegateCryptoCallbacks);
+
+        this._crossSigningKeys = null;
+        this._keySignatures = null;
+        this._keyBackupInfo = null;
+    }
+
+    /**
+     * Adds new cross-signing public keys
+     *
+     * @param {function} authUpload Function called to await an interactive auth
+     * flow when uploading device signing keys.
+     * Args:
+     *     {function} A function that makes the request requiring auth. Receives
+     *     the auth data as an object. Can be called multiple times, first with
+     *     an empty authDict, to obtain the flows.
+     * @param {Object} keys the new keys
+     */
+    addCrossSigningKeys(authUpload, keys) {
+        this._crossSigningKeys = {authUpload, keys};
+    }
+
+    /**
+     * Adds the key backup info to be updated on the server
+     *
+     * Used either to create a new key backup, or add signatures
+     * from the new MSK.
+     *
+     * @param {Object} keyBackupInfo as received from/sent to the server
+     */
+    addSessionBackup(keyBackupInfo) {
+        this._keyBackupInfo = keyBackupInfo;
+    }
+
+    /**
+     * Adds the session backup private key to be updated in the local cache
+     *
+     * Used after fixing the format of the key
+     *
+     * @param {Uint8Array} privateKey
+     */
+    addSessionBackupPrivateKeyToCache(privateKey) {
+        this._sessionBackupPrivateKey = privateKey;
+    }
+
+    /**
+     * Add signatures from a given user and device/x-sign key
+     * Used to sign the new cross-signing key with the device key
+     *
+     * @param {String} userId
+     * @param {String} deviceId
+     * @param {String} signature
+     */
+    addKeySignature(userId, deviceId, signature) {
+        if (!this._keySignatures) {
+            this._keySignatures = {};
+        }
+        const userSignatures = this._keySignatures[userId] || {};
+        this._keySignatures[userId] = userSignatures;
+        userSignatures[deviceId] = signature;
+    }
+
+
+    /**
+     * @param {String} type
+     * @param {Object} content
+     * @return {Promise}
+     */
+    setAccountData(type, content) {
+        return this.accountDataClientAdapter.setAccountData(type, content);
+    }
+
+    /**
+     * builds the operation containing all the parts that have been added to the builder
+     * @return {EncryptionSetupOperation}
+     */
+    buildOperation() {
+        const accountData = this.accountDataClientAdapter._values;
+        return new EncryptionSetupOperation(
+            accountData,
+            this._crossSigningKeys,
+            this._keyBackupInfo,
+            this._keySignatures,
+        );
+    }
+
+    /**
+     * Stores the created keys locally.
+     *
+     * This does not yet store the operation in a way that it can be restored,
+     * but that is the idea in the future.
+     *
+     * @param  {Crypto} crypto
+     * @return {Promise}
+     */
+    async persist(crypto) {
+        // store private keys in cache
+        if (this._crossSigningKeys) {
+            const cacheCallbacks = createCryptoStoreCacheCallbacks(
+                crypto._cryptoStore, crypto._olmDevice);
+            for (const type of ["master", "self_signing", "user_signing"]) {
+                logger.log(`Cache ${type} cross-signing private key locally`);
+                const privateKey = this.crossSigningCallbacks.privateKeys.get(type);
+                await cacheCallbacks.storeCrossSigningKeyCache(type, privateKey);
+            }
+            // store own cross-sign pubkeys as trusted
+            await crypto._cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    crypto._cryptoStore.storeCrossSigningKeys(
+                        txn, this._crossSigningKeys.keys);
+                },
+            );
+        }
+        // store session backup key in cache
+        if (this._sessionBackupPrivateKey) {
+            await crypto.storeSessionBackupPrivateKey(this._sessionBackupPrivateKey);
+        }
+    }
+}
+
+/**
+ * Can be created from EncryptionSetupBuilder, or
+ * (in a follow-up PR, not implemented yet) restored from storage, to retry.
+ *
+ * It does not have knowledge of any private keys, unlike the builder.
+ */
+export class EncryptionSetupOperation {
+    /**
+     * @param  {Map<String, Object>} accountData
+     * @param  {Object} crossSigningKeys
+     * @param  {Object} keyBackupInfo
+     * @param  {Object} keySignatures
+     */
+    constructor(accountData, crossSigningKeys, keyBackupInfo, keySignatures) {
+        this._accountData = accountData;
+        this._crossSigningKeys = crossSigningKeys;
+        this._keyBackupInfo = keyBackupInfo;
+        this._keySignatures = keySignatures;
+    }
+
+    /**
+     * Runs the (remaining part of, in the future) operation by sending requests to the server.
+     * @param  {Crypto} crypto
+     */
+    async apply(crypto) {
+        const baseApis = crypto._baseApis;
+        // upload cross-signing keys
+        if (this._crossSigningKeys) {
+            const keys = {};
+            for (const [name, key] of Object.entries(this._crossSigningKeys.keys)) {
+                keys[name + "_key"] = key;
+            }
+
+            // We must only call `uploadDeviceSigningKeys` from inside this auth
+            // helper to ensure we properly handle auth errors.
+            await this._crossSigningKeys.authUpload(authDict => {
+                return baseApis.uploadDeviceSigningKeys(authDict, keys);
+            });
+
+            // pass the new keys to the main instance of our own CrossSigningInfo.
+            crypto._crossSigningInfo.setKeys(this._crossSigningKeys.keys);
+        }
+        // set account data
+        if (this._accountData) {
+            for (const [type, content] of this._accountData) {
+                await baseApis.setAccountData(type, content);
+            }
+        }
+        // upload first cross-signing signatures with the new key
+        // (e.g. signing our own device)
+        if (this._keySignatures) {
+            await baseApis.uploadKeySignatures(this._keySignatures);
+        }
+        // need to create/update key backup info
+        if (this._keyBackupInfo) {
+            if (this._keyBackupInfo.version) {
+                // session backup signature
+                // The backup is trusted because the user provided the private key.
+                // Sign the backup with the cross signing key so the key backup can
+                // be trusted via cross-signing.
+                await baseApis._http.authedRequest(
+                    undefined, "PUT", "/room_keys/version/" + this._keyBackupInfo.version,
+                    undefined, {
+                        algorithm: this._keyBackupInfo.algorithm,
+                        auth_data: this._keyBackupInfo.auth_data,
+                    },
+                    {prefix: PREFIX_UNSTABLE},
+                );
+            } else {
+                // add new key backup
+                await baseApis._http.authedRequest(
+                    undefined, "POST", "/room_keys/version",
+                    undefined, this._keyBackupInfo,
+                    {prefix: PREFIX_UNSTABLE},
+                );
+            }
+        }
+    }
+}
+
+
+/**
+ * Catches account data set by SecretStorage during bootstrapping by
+ * implementing the methods related to account data in MatrixClient
+ */
+class AccountDataClientAdapter extends EventEmitter {
+    /**
+     * @param  {Object.<String, MatrixEvent>} accountData existing account data
+     */
+    constructor(accountData) {
+        super();
+        this._existingValues = accountData;
+        this._values = new Map();
+    }
+
+    /**
+     * @param  {String} type
+     * @return {Promise<Object>} the content of the account data
+     */
+    getAccountDataFromServer(type) {
+        return Promise.resolve(this.getAccountData(type));
+    }
+
+    /**
+     * @param  {String} type
+     * @return {Object} the content of the account data
+     */
+    getAccountData(type) {
+        const modifiedValue = this._values.get(type);
+        if (modifiedValue) {
+            return modifiedValue;
+        }
+        const existingValue = this._existingValues[type];
+        if (existingValue) {
+            return existingValue.getContent();
+        }
+        return null;
+    }
+
+    /**
+     * @param {String} type
+     * @param {Object} content
+     * @return {Promise}
+     */
+    setAccountData(type, content) {
+        const lastEvent = this._values.get(type);
+        this._values.set(type, content);
+        // ensure accountData is emitted on the next tick,
+        // as SecretStorage listens for it while calling this method
+        // and it seems to rely on this.
+        return Promise.resolve().then(() => {
+            const event = new MatrixEvent({type, content});
+            this.emit("accountData", event, lastEvent);
+        });
+    }
+}
+
+/**
+ * Catches the private cross-signing keys set during bootstrapping
+ * by both cache callbacks (see createCryptoStoreCacheCallbacks) as non-cache callbacks.
+ * See CrossSigningInfo constructor
+ */
+class CrossSigningCallbacks {
+    constructor() {
+        this.privateKeys = new Map();
+    }
+
+    // cache callbacks
+    getCrossSigningKeyCache(type, expectedPublicKey) {
+        return this.getCrossSigningKey(type, expectedPublicKey);
+    }
+
+    storeCrossSigningKeyCache(type, key) {
+        this.privateKeys.set(type, key);
+        return Promise.resolve();
+    }
+
+    // non-cache callbacks
+    getCrossSigningKey(type, _expectedPubkey) {
+        return Promise.resolve(this.privateKeys.get(type));
+    }
+
+    saveCrossSigningKeys(privateKeys) {
+        for (const [type, privateKey] of Object.entries(privateKeys)) {
+            this.privateKeys.set(type, privateKey);
+        }
+    }
+}
+
+/**
+ * Catches the 4S private key set during bootstrapping by implementing
+ * the SecretStorage crypto callbacks
+ */
+class SSSSCryptoCallbacks {
+    constructor(delegateCryptoCallbacks) {
+        this._privateKeys = new Map();
+        this._delegateCryptoCallbacks = delegateCryptoCallbacks;
+    }
+
+    async getSecretStorageKey({ keys }, name) {
+        for (const keyId of Object.keys(keys)) {
+            const privateKey = this._privateKeys.get(keyId);
+            if (privateKey) {
+                return [keyId, privateKey];
+            }
+        }
+        // if we don't have the key cached yet, ask
+        // for it to the general crypto callbacks and cache it
+        if (this._delegateCryptoCallbacks) {
+            const result = await this._delegateCryptoCallbacks.
+                getSecretStorageKey({keys}, name);
+            if (result) {
+                const [keyId, privateKey] = result;
+                this._privateKeys.set(keyId, privateKey);
+            }
+            return result;
+        }
+    }
+
+    addPrivateKey(keyId, keyInfo, privKey) {
+        this._privateKeys.set(keyId, privKey);
+        // Also pass along to application to cache if it wishes
+        if (
+            this._delegateCryptoCallbacks &&
+            this._delegateCryptoCallbacks.cacheSecretStorageKey
+        ) {
+            this._delegateCryptoCallbacks.cacheSecretStorageKey(keyId, keyInfo, privKey);
+        }
+    }
+}

src/crypto/OlmDevice.js

@@ -36,9 +36,15 @@ function checkPayloadLength(payloadString) {
         // Note that even if we manage to do the encryption, the message send may fail,
         // because by the time we've wrapped the ciphertext in the event object, it may
         // exceed 65K. But at least we won't just fail with "abort()" in that case.
-        throw new Error("Message too long (" + payloadString.length + " bytes). " +
+        const err = new Error("Message too long (" + payloadString.length + " bytes). " +
                         "The maximum for an encrypted message is " +
                         MAX_PLAINTEXT_LENGTH + " bytes.");
+        // TODO: [TypeScript] We should have our own error types
+        err.data = {
+            errcode: "M_TOO_LARGE",
+            error: "Payload too large for encrypted message",
+        };
+        throw err;
     }
 }
 
@@ -138,7 +144,7 @@ OlmDevice.prototype.init = async function(opts = {}) {
     try {
         if (fromExportedDevice) {
             if (pickleKey) {
-                console.warn(
+                logger.warn(
                     'ignoring opts.pickleKey'
                     + ' because opts.fromExportedDevice is present.',
                 );
@@ -471,6 +477,36 @@ OlmDevice.prototype.generateOneTimeKeys = function(numKeys) {
     );
 };
 
+/**
+ * Generate a new fallback keys
+ *
+ * @return {Promise} Resolved once the account is saved back having generated the key
+ */
+OlmDevice.prototype.generateFallbackKey = async function() {
+    await this._cryptoStore.doTxn(
+        'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                account.generate_fallback_key();
+                this._storeAccount(txn, account);
+            });
+        },
+    );
+};
+
+OlmDevice.prototype.getFallbackKey = async function() {
+    let result;
+    await this._cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                result = JSON.parse(account.fallback_key());
+            });
+        },
+    );
+    return result;
+};
+
 /**
  * Generate a new outbound session
  *
@@ -986,11 +1022,12 @@ OlmDevice.prototype._getInboundGroupSession = function(
  * @param {Object<string, string>} keysClaimed Other keys the sender claims.
  * @param {boolean} exportFormat true if the megolm keys are in export format
  *    (ie, they lack an ed25519 signature)
+ * @param {Object} [extraSessionData={}] any other data to be include with the session
  */
 OlmDevice.prototype.addInboundGroupSession = async function(
     roomId, senderKey, forwardingCurve25519KeyChain,
     sessionId, sessionKey, keysClaimed,
-    exportFormat,
+    exportFormat, extraSessionData = {},
 ) {
     await this._cryptoStore.doTxn(
         'readwrite', [
@@ -1022,9 +1059,14 @@ OlmDevice.prototype.addInboundGroupSession = async function(
                                     + senderKey + "/" + sessionId,
                             );
                             if (existingSession.first_known_index()
-                                <= session.first_known_index()) {
+                                <= session.first_known_index()
+                                && !(existingSession.first_known_index() == session.first_known_index()
+                                    && !extraSessionData.untrusted
+                                    && existingSessionData.untrusted)) {
                                 // existing session has lower index (i.e. can
-                                // decrypt more), so keep it
+                                // decrypt more), or they have the same index and
+                                // the new sessions trust does not win over the old
+                                // sessions trust, so keep it
                                 logger.log(
                                     `Keeping existing megolm session ${sessionId}`,
                                 );
@@ -1032,12 +1074,17 @@ OlmDevice.prototype.addInboundGroupSession = async function(
                             }
                         }
 
-                        const sessionData = {
+                        logger.info(
+                            "Storing megolm session " + senderKey + "/" + sessionId +
+                            " with first index " + session.first_known_index(),
+                        );
+
+                        const sessionData = Object.assign({}, extraSessionData, {
                             room_id: roomId,
                             session: session.pickle(this._pickleKey),
                             keysClaimed: keysClaimed,
                             forwardingCurve25519KeyChain: forwardingCurve25519KeyChain,
-                        };
+                        });
 
                         this._cryptoStore.storeEndToEndInboundGroupSession(
                             senderKey, sessionId, sessionData, txn,
@@ -1213,6 +1260,7 @@ OlmDevice.prototype.decryptGroupMessage = async function(
                         forwardingCurve25519KeyChain: (
                             sessionData.forwardingCurve25519KeyChain || []
                         ),
+                        untrusted: sessionData.untrusted,
                     };
                 },
             );

src/crypto/OutgoingRoomKeyRequestManager.js

@@ -97,10 +97,6 @@ export class OutgoingRoomKeyRequestManager {
      */
     start() {
         this._clientRunning = true;
-
-        // set the timer going, to handle any requests which didn't get sent
-        // on the previous run of the client.
-        this._startTimer();
     }
 
     /**
@@ -113,7 +109,14 @@ export class OutgoingRoomKeyRequestManager {
     }
 
     /**
-     * Send off a room key request, if we haven't already done so.
+     * Send any requests that have been queued
+     */
+    sendQueuedRequests() {
+        this._startTimer();
+    }
+
+    /**
+     * Queue up a room key request, if we haven't already queued or sent one.
      *
      * The `requestBody` is compared (with a deep-equality check) against
      * previous queued or sent requests and if it matches, no change is made.
@@ -129,7 +132,7 @@ export class OutgoingRoomKeyRequestManager {
      *    pending list (or we have established that a similar request already
      *    exists)
      */
-    async sendRoomKeyRequest(requestBody, recipients, resend=false) {
+    async queueRoomKeyRequest(requestBody, recipients, resend=false) {
         const req = await this._cryptoStore.getOutgoingRoomKeyRequest(
             requestBody,
         );
@@ -184,7 +187,7 @@ export class OutgoingRoomKeyRequestManager {
                         // in state ROOM_KEY_REQUEST_STATES.SENT, so we must have
                         // raced with another tab to mark the request cancelled.
                         // Try again, to make sure the request is resent.
-                        return await this.sendRoomKeyRequest(
+                        return await this.queueRoomKeyRequest(
                             requestBody, recipients, resend,
                         );
                     }
@@ -220,9 +223,6 @@ export class OutgoingRoomKeyRequestManager {
                 throw new Error('unhandled state: ' + req.state);
             }
         }
-        // some of the branches require the timer to be started.  Just start it
-        // all the time, because it doesn't hurt to start it.
-        this._startTimer();
     }
 
     /**
@@ -332,14 +332,14 @@ export class OutgoingRoomKeyRequestManager {
      * This is intended for situations where something substantial has changed, and we
      * don't really expect the other end to even care about the cancellation.
      * For example, after initialization or self-verification.
-     * @return {Promise} An array of `sendRoomKeyRequest` outputs.
+     * @return {Promise} An array of `queueRoomKeyRequest` outputs.
      */
     async cancelAndResendAllOutgoingRequests() {
         const outgoings = await this._cryptoStore.getAllOutgoingRoomKeyRequestsByState(
             ROOM_KEY_REQUEST_STATES.SENT,
         );
         return Promise.all(outgoings.map(({ requestBody, recipients }) =>
-            this.sendRoomKeyRequest(requestBody, recipients, true)));
+            this.queueRoomKeyRequest(requestBody, recipients, true)));
     }
 
     // start the background timer to send queued requests, if the timer isn't
@@ -381,15 +381,12 @@ export class OutgoingRoomKeyRequestManager {
             return Promise.resolve();
         }
 
-        logger.log("Looking for queued outgoing room key requests");
-
         return this._cryptoStore.getOutgoingRoomKeyRequestByState([
             ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
             ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
             ROOM_KEY_REQUEST_STATES.UNSENT,
         ]).then((req) => {
             if (!req) {
-                logger.log("No more outgoing room key requests");
                 this._sendOutgoingRoomKeyRequestsTimer = null;
                 return;
             }
@@ -413,7 +410,6 @@ export class OutgoingRoomKeyRequestManager {
             }).catch((e) => {
                 logger.error("Error sending room key request; will retry later.", e);
                 this._sendOutgoingRoomKeyRequestsTimer = null;
-                this._startTimer();
             });
         });
     }

src/crypto/SecretStorage.js

@@ -17,16 +17,12 @@ limitations under the License.
 import {EventEmitter} from 'events';
 import {logger} from '../logger';
 import * as olmlib from './olmlib';
-import {pkVerify} from './olmlib';
 import {randomString} from '../randomstring';
 import {encryptAES, decryptAES} from './aes';
 import {encodeBase64} from "./olmlib";
 
 export const SECRET_STORAGE_ALGORITHM_V1_AES
     = "m.secret_storage.v1.aes-hmac-sha2";
-// don't use curve25519 for writing data.
-export const SECRET_STORAGE_ALGORITHM_V1_CURVE25519
-    = "m.secret_storage.v1.curve25519-aes-sha2";
 
 const ZERO_STR = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
 
@@ -35,11 +31,10 @@ const ZERO_STR = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0
  * @module crypto/SecretStorage
  */
 export class SecretStorage extends EventEmitter {
-    constructor(baseApis, cryptoCallbacks, crossSigningInfo) {
+    constructor(baseApis, cryptoCallbacks) {
         super();
         this._baseApis = baseApis;
         this._cryptoCallbacks = cryptoCallbacks;
-        this._crossSigningInfo = crossSigningInfo;
         this._requests = {};
         this._incomingRequests = {};
     }
@@ -86,31 +81,29 @@ export class SecretStorage extends EventEmitter {
      * @param {string} [keyId] the ID of the key.  If not given, a random
      *     ID will be generated.
      *
-     * @return {string} the ID of the key
+     * @return {object} An object with:
+     *     keyId: {string} the ID of the key
+     *     keyInfo: {object} details about the key (iv, mac, passphrase)
      */
     async addKey(algorithm, opts, keyId) {
-        const keyData = {algorithm};
+        const keyInfo = {algorithm};
 
         if (!opts) opts = {};
 
         if (opts.name) {
-            keyData.name = opts.name;
+            keyInfo.name = opts.name;
         }
 
-        switch (algorithm) {
-        case SECRET_STORAGE_ALGORITHM_V1_AES:
-        {
+        if (algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
             if (opts.passphrase) {
-                keyData.passphrase = opts.passphrase;
+                keyInfo.passphrase = opts.passphrase;
             }
             if (opts.key) {
                 const {iv, mac} = await SecretStorage._calculateKeyCheck(opts.key);
-                keyData.iv = iv;
-                keyData.mac = mac;
+                keyInfo.iv = iv;
+                keyInfo.mac = mac;
             }
-            break;
-        }
-        default:
+        } else {
             throw new Error(`Unknown key algorithm ${opts.algorithm}`);
         }
 
@@ -124,40 +117,14 @@ export class SecretStorage extends EventEmitter {
             );
         }
 
-        await this._crossSigningInfo.signObject(keyData, 'master');
-
-        await this._baseApis.setAccountData(
-            `m.secret_storage.key.${keyId}`, keyData,
-        );
-
-        return keyId;
-    }
-
-    /**
-     * Signs a given secret storage key with the cross-signing master key.
-     *
-     * @param {string} [keyId = default key's ID] The ID of the key to sign.
-     *     Defaults to the default key ID if not provided.
-     */
-    async signKey(keyId) {
-        if (!keyId) {
-            keyId = await this.getDefaultKeyId();
-        }
-        if (!keyId) {
-            throw new Error("signKey requires a key ID");
-        }
-
-        const keyInfo = await this._baseApis.getAccountDataFromServer(
-            `m.secret_storage.key.${keyId}`,
-        );
-        if (!keyInfo) {
-            throw new Error(`Key ${keyId} does not exist in account data`);
-        }
-
-        await this._crossSigningInfo.signObject(keyInfo, 'master');
         await this._baseApis.setAccountData(
             `m.secret_storage.key.${keyId}`, keyInfo,
         );
+
+        return {
+            keyId,
+            keyInfo,
+        };
     }
 
     /**
@@ -193,15 +160,6 @@ export class SecretStorage extends EventEmitter {
         return !!(await this.getKey(keyId));
     }
 
-    async keyNeedsUpgrade(keyId) {
-        const keyInfo = await this.getKey(keyId);
-        if (keyInfo && keyInfo[1].algorithm === SECRET_STORAGE_ALGORITHM_V1_CURVE25519) {
-            return true;
-        } else {
-            return false;
-        }
-    }
-
     /**
      * Check whether a key matches what we expect based on the key info
      *
@@ -211,32 +169,15 @@ export class SecretStorage extends EventEmitter {
      * @return {boolean} whether or not the key matches
      */
     async checkKey(key, info) {
-        switch (info.algorithm) {
-        case SECRET_STORAGE_ALGORITHM_V1_AES:
-        {
+        if (info.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
             if (info.mac) {
                 const {mac} = await SecretStorage._calculateKeyCheck(key, info.iv);
-                return info.mac === mac;
+                return info.mac.replace(/=+$/g, '') === mac.replace(/=+$/g, '');
             } else {
                 // if we have no information, we have to assume the key is right
                 return true;
             }
-        }
-        case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
-        {
-            let decryption = null;
-            try {
-                decryption = new global.Olm.PkDecryption();
-                const gotPubkey = decryption.init_with_private_key(key);
-                // make sure it agrees with the given pubkey
-                return gotPubkey === info.pubkey;
-            } catch (e) {
-                return false;
-            } finally {
-                if (decryption) decryption.free();
-            }
-        }
-        default:
+        } else {
             throw new Error("Unknown algorithm");
         }
     }
@@ -278,15 +219,11 @@ export class SecretStorage extends EventEmitter {
             }
 
             // encrypt secret, based on the algorithm
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1_AES:
-            {
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
                 const keys = {[keyId]: keyInfo};
                 const [, encryption] = await this._getSecretStorageKey(keys, name);
                 encrypted[keyId] = await encryption.encrypt(secret);
-                break;
-            }
-            default:
+            } else {
                 logger.warn("unknown algorithm for secret storage key " + keyId
                             + ": " + keyInfo.algorithm);
                 // do nothing if we don't understand the encryption algorithm
@@ -315,7 +252,7 @@ export class SecretStorage extends EventEmitter {
         ) {
             const hasKey = await this.hasKey(keys[0]);
             if (hasKey) {
-                console.log("Fixing up passthrough secret: " + name);
+                logger.log("Fixing up passthrough secret: " + name);
                 await this.storePassthrough(name, keys[0]);
                 const newData = await this._baseApis.getAccountDataFromServer(name);
                 return newData;
@@ -352,27 +289,19 @@ export class SecretStorage extends EventEmitter {
                 "m.secret_storage.key." + keyId,
             );
             const encInfo = secretInfo.encrypted[keyId];
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1_AES:
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
                 if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
                     keys[keyId] = keyInfo;
                 }
-                break;
-            case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
-                if (
-                    keyInfo.pubkey && (
-                        (encInfo.ciphertext && encInfo.mac && encInfo.ephemeral) ||
-                        encInfo.passthrough
-                    )
-                ) {
-                    keys[keyId] = keyInfo;
-                }
-                break;
-            default:
-                // do nothing if we don't understand the encryption algorithm
             }
         }
 
+        if (Object.keys(keys).length === 0) {
+            throw new Error(`Could not decrypt ${name} because none of ` +
+                `the keys it is encrypted with are for a supported algorithm`);
+        }
+
         let keyId;
         let decryption;
         try {
@@ -418,8 +347,7 @@ export class SecretStorage extends EventEmitter {
 
         const ret = {};
 
-        // check if secret is encrypted by a known/trusted secret and
-        // encryption looks sane
+        // filter secret encryption keys with supported algorithm
         for (const keyId of Object.keys(secretInfo.encrypted)) {
             // get key information from key storage
             const keyInfo = await this._baseApis.getAccountDataFromServer(
@@ -428,49 +356,11 @@ export class SecretStorage extends EventEmitter {
             if (!keyInfo) continue;
             const encInfo = secretInfo.encrypted[keyId];
 
-            // We don't actually need the decryption object if it's a passthrough
-            // since we just want to return the key itself.
-            if (encInfo.passthrough) {
-                try {
-                    pkVerify(
-                        keyInfo,
-                        this._crossSigningInfo.getId('master'),
-                        this._crossSigningInfo.userId,
-                    );
-                } catch (e) {
-                    // not trusted, so move on to the next key
-                    continue;
-                }
-                ret[keyId] = keyInfo;
-                continue;
-            }
-
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1_AES:
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
                 if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
                     ret[keyId] = keyInfo;
                 }
-                break;
-            case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
-                if (keyInfo.pubkey && encInfo.ciphertext && encInfo.mac
-                    && encInfo.ephemeral) {
-                    if (checkKey) {
-                        try {
-                            pkVerify(
-                                keyInfo,
-                                this._crossSigningInfo.getId('master'),
-                                this._crossSigningInfo.userId,
-                            );
-                        } catch (e) {
-                            // not trusted, so move on to the next key
-                            continue;
-                        }
-                    }
-                    ret[keyId] = keyInfo;
-                }
-                break;
-            default:
-                // do nothing if we don't understand the encryption algorithm
             }
         }
         return Object.keys(ret).length ? ret : null;
@@ -488,6 +378,7 @@ export class SecretStorage extends EventEmitter {
         const requestId = this._baseApis.makeTxnId();
 
         const requestControl = this._requests[requestId] = {
+            name,
             devices,
         };
         const promise = new Promise((resolve, reject) => {
@@ -571,13 +462,13 @@ export class SecretStorage extends EventEmitter {
             if (!this._cryptoCallbacks.onSecretRequested) {
                 return;
             }
-            const secret = await this._cryptoCallbacks.onSecretRequested({
-                user_id: sender,
-                device_id: deviceId,
-                request_id: content.request_id,
-                name: content.name,
-                device_trust: this._baseApis.checkDeviceTrust(sender, deviceId),
-            });
+            const secret = await this._cryptoCallbacks.onSecretRequested(
+                sender,
+                deviceId,
+                content.request_id,
+                content.name,
+                this._baseApis.checkDeviceTrust(sender, deviceId),
+            );
             if (secret) {
                 logger.info(`Preparing ${content.name} secret for ${deviceId}`);
                 const payload = {
@@ -651,6 +542,10 @@ export class SecretStorage extends EventEmitter {
                 return;
             }
 
+            logger.log(
+                `Successfully received secret ${requestControl.name} ` +
+                `from ${deviceInfo.deviceId}`,
+            );
             requestControl.resolve(content.secret);
         }
     }
@@ -674,9 +569,7 @@ export class SecretStorage extends EventEmitter {
             throw new Error("App returned unknown key from getSecretStorageKey!");
         }
 
-        switch (keys[keyId].algorithm) {
-        case SECRET_STORAGE_ALGORITHM_V1_AES:
-        {
+        if (keys[keyId].algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
             const decryption = {
                 encrypt: async function(secret) {
                     return await encryptAES(secret, privateKey, name);
@@ -686,36 +579,7 @@ export class SecretStorage extends EventEmitter {
                 },
             };
             return [keyId, decryption];
-        }
-        case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
-        {
-            const pkDecryption = new global.Olm.PkDecryption();
-            let pubkey;
-            try {
-                pubkey = pkDecryption.init_with_private_key(privateKey);
-            } catch (e) {
-                pkDecryption.free();
-                throw new Error("getSecretStorageKey callback returned invalid key");
-            }
-            if (pubkey !== keys[keyId].pubkey) {
-                pkDecryption.free();
-                throw new Error(
-                    "getSecretStorageKey callback returned incorrect key",
-                );
-            }
-            const decryption = {
-                free: pkDecryption.free.bind(pkDecryption),
-                decrypt: async function(encInfo) {
-                    return pkDecryption.decrypt(
-                        encInfo.ephemeral, encInfo.mac, encInfo.ciphertext,
-                    );
-                },
-                // needed for passthrough
-                get_private_key: pkDecryption.get_private_key.bind(pkDecryption),
-            };
-            return [keyId, decryption];
-        }
-        default:
+        } else {
             throw new Error("Unknown key type: " + keys[keyId].algorithm);
         }
     }

src/crypto/aes.js

@@ -84,9 +84,9 @@ async function decryptNode(data, key, name) {
     const [aesKey, hmacKey] = deriveKeysNode(key, name);
 
     const hmac = crypto.createHmac("sha256", hmacKey)
-        .update(data.ciphertext, "base64").digest("base64");
+        .update(data.ciphertext, "base64").digest("base64").replace(/=+$/g, '');
 
-    if (hmac !== data.mac) {
+    if (hmac !== data.mac.replace(/=+$/g, '')) {
         throw new Error(`Error decrypting secret ${name}: bad MAC`);
     }
 

src/crypto/algorithms/megolm.js

@@ -264,11 +264,14 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
         await Promise.all([
             (async () => {
                 // share keys with devices that we already have a session for
+                logger.debug(`Sharing keys with existing Olm sessions in ${this._roomId}`);
                 await this._shareKeyWithOlmSessions(
                     session, key, payload, olmSessions,
                 );
+                logger.debug(`Shared keys with existing Olm sessions in ${this._roomId}`);
             })(),
             (async () => {
+                logger.debug(`Sharing keys with new Olm sessions in ${this._roomId}`);
                 const errorDevices = [];
 
                 // meanwhile, establish olm sessions for devices that we don't
@@ -311,7 +314,7 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
                         }
 
                         await this._shareKeyWithDevices(
-                            session, key, payload, retryDevices, failedDevices,
+                            session, key, payload, retryDevices, failedDevices, 30000,
                         );
 
                         await this._notifyFailedOlmDevices(session, key, failedDevices);
@@ -319,8 +322,10 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
                 } else {
                     await this._notifyFailedOlmDevices(session, key, errorDevices);
                 }
+                logger.debug(`Shared keys with new Olm sessions in ${this._roomId}`);
             })(),
             (async () => {
+                logger.debug(`Notifying blocked devices in ${this._roomId}`);
                 // also, notify blocked devices that they're blocked
                 const blockedMap = {};
                 for (const [userId, userBlockedDevices] of Object.entries(blocked)) {
@@ -336,6 +341,7 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
                 }
 
                 await this._notifyBlockedDevices(session, blockedMap);
+                logger.debug(`Notified blocked devices in ${this._roomId}`);
             })(),
         ]);
     };
@@ -348,6 +354,11 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
     // first wait for the previous share to complete
     const prom = this._setupPromise.then(prepareSession);
 
+    // Ensure any failures are logged for debugging
+    prom.catch(e => {
+        logger.error(`Failed to ensure outbound session in ${this._roomId}`, e);
+    });
+
     // _setupPromise resolves to `session` whether or not the share succeeds
     this._setupPromise = prom.then(returnSession, returnSession);
 
@@ -369,17 +380,11 @@ MegolmEncryption.prototype._prepareNewSession = async function() {
         key.key, {ed25519: this._olmDevice.deviceEd25519Key},
     );
 
-    if (this._crypto.backupInfo) {
-        // don't wait for it to complete
-        this._crypto.backupGroupSession(
-            this._roomId, this._olmDevice.deviceCurve25519Key, [],
-            sessionId, key.key,
-        ).catch((e) => {
-            // This throws if the upload failed, but this is fine
-            // since it will have written it to the db and will retry.
-            logger.log("Failed to back up megolm session", e);
-        });
-    }
+    // don't wait for it to complete
+    this._crypto.backupGroupSession(
+        this._roomId, this._olmDevice.deviceCurve25519Key, [],
+        sessionId, key.key,
+    );
 
     return new OutboundSessionInfo(sessionId);
 };
@@ -521,6 +526,33 @@ MegolmEncryption.prototype._encryptAndSendKeysToDevices = function(
     }
 
     return Promise.all(promises).then(() => {
+        // prune out any devices that encryptMessageForDevice could not encrypt for,
+        // in which case it will have just not added anything to the ciphertext object.
+        // There's no point sending messages to devices if we couldn't encrypt to them,
+        // since that's effectively a blank message.
+        for (const userId of Object.keys(contentMap)) {
+            for (const deviceId of Object.keys(contentMap[userId])) {
+                if (Object.keys(contentMap[userId][deviceId].ciphertext).length === 0) {
+                    logger.log(
+                        "No ciphertext for device " +
+                        userId + ":" + deviceId + ": pruning",
+                    );
+                    delete contentMap[userId][deviceId];
+                }
+            }
+            // No devices left for that user? Strip that too.
+            if (Object.keys(contentMap[userId]).length === 0) {
+                logger.log("Pruned all devices for user " + userId);
+                delete contentMap[userId];
+            }
+        }
+
+        // Is there anything left?
+        if (Object.keys(contentMap).length === 0) {
+            logger.log("No users left to send to: aborting");
+            return;
+        }
+
         return this._baseApis.sendToDevice("m.room.encrypted", contentMap).then(() => {
             // store that we successfully uploaded the keys of the current slice
             for (const userId of Object.keys(contentMap)) {
@@ -819,24 +851,41 @@ MegolmEncryption.prototype.prepareToEncrypt = function(room) {
         // We're already preparing something, so don't do anything else.
         // FIXME: check if we need to restart
         // (https://github.com/matrix-org/matrix-js-sdk/issues/1255)
+        const elapsedTime = Date.now() - this.encryptionPreparationMetadata.startTime;
+        logger.debug(
+            `Already started preparing to encrypt for ${this._roomId} ` +
+            `${elapsedTime} ms ago, skipping`,
+        );
         return;
     }
 
     logger.debug(`Preparing to encrypt events for ${this._roomId}`);
 
+    this.encryptionPreparationMetadata = {
+        startTime: Date.now(),
+    };
     this.encryptionPreparation = (async () => {
-        const [devicesInRoom, blocked] = await this._getDevicesInRoom(room);
+        try {
+            logger.debug(`Getting devices in ${this._roomId}`);
+            const [devicesInRoom, blocked] = await this._getDevicesInRoom(room);
 
-        if (this._crypto.getGlobalErrorOnUnknownDevices()) {
-            // Drop unknown devices for now.  When the message gets sent, we'll
-            // throw an error, but we'll still be prepared to send to the known
-            // devices.
-            this._removeUnknownDevices(devicesInRoom);
+            if (this._crypto.getGlobalErrorOnUnknownDevices()) {
+                // Drop unknown devices for now.  When the message gets sent, we'll
+                // throw an error, but we'll still be prepared to send to the known
+                // devices.
+                this._removeUnknownDevices(devicesInRoom);
+            }
+
+            logger.debug(`Ensuring outbound session in ${this._roomId}`);
+            await this._ensureOutboundSession(devicesInRoom, blocked, true);
+
+            logger.debug(`Ready to encrypt events for ${this._roomId}`);
+        } catch (e) {
+            logger.error(`Failed to prepare to encrypt events for ${this._roomId}`, e);
+        } finally {
+            delete this.encryptionPreparationMetadata;
+            delete this.encryptionPreparation;
         }
-
-        await this._ensureOutboundSession(devicesInRoom, blocked, true);
-
-        delete this.encryptionPreparation;
     })();
 };
 
@@ -988,7 +1037,7 @@ MegolmEncryption.prototype._getDevicesInRoom = async function(room) {
     // with them, which means that they will have announced any new devices via
     // device_lists in their /sync response.  This cache should then be maintained
     // using all the device_lists changes and left fields.
-    // See https://github.com/vector-im/riot-web/issues/2305 for details.
+    // See https://github.com/vector-im/element-web/issues/2305 for details.
     const devices = await this._crypto.downloadKeys(roomMembers, false);
     const blocked = {};
     // remove any blocked devices
@@ -1082,7 +1131,7 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {
     //
     // then, if the key turns up while decryption is in progress (and
     // decryption fails), we will schedule a retry.
-    // (fixes https://github.com/vector-im/riot-web/issues/5001)
+    // (fixes https://github.com/vector-im/element-web/issues/5001)
     this._addEventToPendingList(event);
 
     let res;
@@ -1174,6 +1223,7 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {
         senderCurve25519Key: res.senderKey,
         claimedEd25519Key: res.keysClaimed.ed25519,
         forwardingCurve25519KeyChain: res.forwardingCurve25519KeyChain,
+        untrusted: res.untrusted,
     };
 };
 
@@ -1296,7 +1346,6 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
         keysClaimed = event.getKeysClaimed();
     }
 
-    logger.log(`Received and adding key for megolm session ${senderKey}|${sessionId}`);
     return this._olmDevice.addInboundGroupSession(
         content.room_id, senderKey, forwardingKeyChain, sessionId,
         content.session_key, keysClaimed,
@@ -1320,18 +1369,12 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
                 }
             });
     }).then(() => {
-        if (this._crypto.backupInfo) {
-            // don't wait for the keys to be backed up for the server
-            this._crypto.backupGroupSession(
-                content.room_id, senderKey, forwardingKeyChain,
-                content.session_id, content.session_key, keysClaimed,
-                exportFormat,
-            ).catch((e) => {
-                // This throws if the upload failed, but this is fine
-                // since it will have written it to the db and will retry.
-                logger.log("Failed to back up megolm session", e);
-            });
-        }
+        // don't wait for the keys to be backed up for the server
+        this._crypto.backupGroupSession(
+            content.room_id, senderKey, forwardingKeyChain,
+            content.session_id, content.session_key, keysClaimed,
+            exportFormat,
+        );
     }).catch((e) => {
         logger.error(`Error handling m.room_key_event: ${e}`);
     });
@@ -1522,8 +1565,11 @@ MegolmDecryption.prototype._buildKeyForwardingMessage = async function(
  * @inheritdoc
  *
  * @param {module:crypto/OlmDevice.MegolmSessionData} session
+ * @param {object} [opts={}] options for the import
+ * @param {boolean} [opts.untrusted] whether the key should be considered as untrusted
+ * @param {string} [opts.source] where the key came from
  */
-MegolmDecryption.prototype.importRoomKey = function(session) {
+MegolmDecryption.prototype.importRoomKey = function(session, opts = {}) {
     return this._olmDevice.addInboundGroupSession(
         session.room_id,
         session.sender_key,
@@ -1532,8 +1578,9 @@ MegolmDecryption.prototype.importRoomKey = function(session) {
         session.session_key,
         session.sender_claimed_keys,
         true,
+        opts.untrusted ? { untrusted: opts.untrusted } : {},
     ).then(() => {
-        if (this._crypto.backupInfo) {
+        if (opts.source !== "backup") {
             // don't wait for it to complete
             this._crypto.backupGroupSession(
                 session.room_id,
@@ -1555,7 +1602,8 @@ MegolmDecryption.prototype.importRoomKey = function(session) {
 };
 
 /**
- * Have another go at decrypting events after we receive a key
+ * Have another go at decrypting events after we receive a key. Resolves once
+ * decryption has been re-attempted on all events.
  *
  * @private
  * @param {String} senderKey
@@ -1574,21 +1622,17 @@ MegolmDecryption.prototype._retryDecryption = async function(senderKey, sessionI
         return true;
     }
 
-    pending.delete(sessionId);
-    if (pending.size === 0) {
-        this._pendingEvents[senderKey];
-    }
+    logger.debug("Retrying decryption on events", [...pending]);
 
     await Promise.all([...pending].map(async (ev) => {
         try {
-            await ev.attemptDecryption(this._crypto);
+            await ev.attemptDecryption(this._crypto, true);
         } catch (e) {
             // don't die if something goes wrong
         }
     }));
 
-    // ev.attemptDecryption will re-add to this._pendingEvents if an event
-    // couldn't be decrypted
+    // If decrypted successfully, they'll have been removed from _pendingEvents
     return !((this._pendingEvents[senderKey] || {})[sessionId]);
 };
 

src/crypto/dehydration.ts

@@ -0,0 +1,288 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {decodeBase64, encodeBase64} from './olmlib';
+import {IndexedDBCryptoStore} from '../crypto/store/indexeddb-crypto-store';
+import {decryptAES, encryptAES} from './aes';
+import anotherjson from "another-json";
+import {logger} from '../logger';
+
+// FIXME: these types should eventually go in a different file
+type Signatures = Record<string, Record<string, string>>;
+
+interface DeviceKeys {
+    algorithms: Array<string>;
+    device_id: string; // eslint-disable-line camelcase
+    user_id: string; // eslint-disable-line camelcase
+    keys: Record<string, string>;
+    signatures?: Signatures;
+}
+
+interface OneTimeKey {
+    key: string;
+    fallback?: boolean;
+    signatures?: Signatures;
+}
+
+export const DEHYDRATION_ALGORITHM = "org.matrix.msc2697.v1.olm.libolm_pickle";
+
+const oneweek = 7 * 24 * 60 * 60 * 1000;
+
+export class DehydrationManager {
+    private inProgress = false;
+    private timeoutId: any;
+    private key: Uint8Array;
+    private keyInfo: {[props: string]: any};
+    private deviceDisplayName: string;
+    constructor(private crypto) {
+        this.getDehydrationKeyFromCache();
+    }
+    async getDehydrationKeyFromCache(): Promise<void> {
+        return await this.crypto._cryptoStore.doTxn(
+            'readonly',
+            [IndexedDBCryptoStore.STORE_ACCOUNT],
+            (txn) => {
+                this.crypto._cryptoStore.getSecretStorePrivateKey(
+                    txn,
+                    async (result) => {
+                        if (result) {
+                            const {key, keyInfo, deviceDisplayName, time} = result;
+                            const pickleKey = Buffer.from(this.crypto._olmDevice._pickleKey);
+                            const decrypted = await decryptAES(key, pickleKey, DEHYDRATION_ALGORITHM);
+                            this.key = decodeBase64(decrypted);
+                            this.keyInfo = keyInfo;
+                            this.deviceDisplayName = deviceDisplayName;
+                            const now = Date.now();
+                            const delay = Math.max(1, time + oneweek - now);
+                            this.timeoutId = global.setTimeout(
+                                this.dehydrateDevice.bind(this), delay,
+                            );
+                        }
+                    },
+                    "dehydration",
+                );
+            },
+        );
+    }
+
+    /** set the key, and queue periodic dehydration to the server in the background */
+    async setKeyAndQueueDehydration(
+        key: Uint8Array, keyInfo: {[props: string]: any} = {},
+        deviceDisplayName: string = undefined,
+    ): Promise<void> {
+        const matches = await this.setKey(key, keyInfo, deviceDisplayName);
+        if (!matches) {
+            // start dehydration in the background
+            this.dehydrateDevice();
+        }
+    }
+
+    async setKey(
+        key: Uint8Array, keyInfo: {[props: string]: any} = {},
+        deviceDisplayName: string = undefined,
+    ): Promise<boolean> {
+        if (!key) {
+            // unsetting the key -- cancel any pending dehydration task
+            if (this.timeoutId) {
+                global.clearTimeout(this.timeoutId);
+                this.timeoutId = undefined;
+            }
+            // clear storage
+            await this.crypto._cryptoStore.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    this.crypto._cryptoStore.storeSecretStorePrivateKey(
+                        txn, "dehydration", null,
+                    );
+                },
+            );
+            this.key = undefined;
+            this.keyInfo = undefined;
+            return;
+        }
+
+        // Check to see if it's the same key as before.  If it's different,
+        // dehydrate a new device.  If it's the same, we can keep the same
+        // device.  (Assume that keyInfo and deviceDisplayName will be the
+        // same if the key is the same.)
+        let matches: boolean = this.key && key.length == this.key.length;
+        for (let i = 0; matches && i < key.length; i++) {
+            if (key[i] != this.key[i]) {
+                matches = false;
+            }
+        }
+        if (!matches) {
+            this.key = key;
+            this.keyInfo = keyInfo;
+            this.deviceDisplayName = deviceDisplayName;
+        }
+        return matches;
+    }
+
+    /** returns the device id of the newly created dehydrated device */
+    async dehydrateDevice(): Promise<string> {
+        if (this.inProgress) {
+            logger.log("Dehydration already in progress -- not starting new dehydration");
+            return;
+        }
+        this.inProgress = true;
+        if (this.timeoutId) {
+            global.clearTimeout(this.timeoutId);
+            this.timeoutId = undefined;
+        }
+        try {
+            const pickleKey = Buffer.from(this.crypto._olmDevice._pickleKey);
+
+            // update the crypto store with the timestamp
+            const key = await encryptAES(encodeBase64(this.key), pickleKey, DEHYDRATION_ALGORITHM);
+            await this.crypto._cryptoStore.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    this.crypto._cryptoStore.storeSecretStorePrivateKey(
+                        txn, "dehydration",
+                        {
+                            keyInfo: this.keyInfo,
+                            key,
+                            deviceDisplayName: this.deviceDisplayName,
+                            time: Date.now(),
+                        },
+                    );
+                },
+            );
+            logger.log("Attempting to dehydrate device");
+
+            logger.log("Creating account");
+            // create the account and all the necessary keys
+            const account = new global.Olm.Account();
+            account.create();
+            const e2eKeys = JSON.parse(account.identity_keys());
+
+            const maxKeys = account.max_number_of_one_time_keys();
+            // FIXME: generate in small batches?
+            account.generate_one_time_keys(maxKeys / 2);
+            account.generate_fallback_key();
+            const otks: Record<string, string> = JSON.parse(account.one_time_keys());
+            const fallbacks: Record<string, string> = JSON.parse(account.fallback_key());
+            account.mark_keys_as_published();
+
+            // dehydrate the account and store it on the server
+            const pickledAccount = account.pickle(new Uint8Array(this.key));
+
+            const deviceData: {[props: string]: any} = {
+                algorithm: DEHYDRATION_ALGORITHM,
+                account: pickledAccount,
+            };
+            if (this.keyInfo.passphrase) {
+                deviceData.passphrase = this.keyInfo.passphrase;
+            }
+
+            logger.log("Uploading account to server");
+            const dehydrateResult = await this.crypto._baseApis._http.authedRequest(
+                undefined,
+                "PUT",
+                "/dehydrated_device",
+                undefined,
+                {
+                    device_data: deviceData,
+                    initial_device_display_name: this.deviceDisplayName,
+                },
+                {
+                    prefix: "/_matrix/client/unstable/org.matrix.msc2697.v2",
+                },
+            );
+
+            // send the keys to the server
+            const deviceId = dehydrateResult.device_id;
+            logger.log("Preparing device keys", deviceId);
+            const deviceKeys: DeviceKeys = {
+                algorithms: this.crypto._supportedAlgorithms,
+                device_id: deviceId,
+                user_id: this.crypto._userId,
+                keys: {
+                    [`ed25519:${deviceId}`]: e2eKeys.ed25519,
+                    [`curve25519:${deviceId}`]: e2eKeys.curve25519,
+                },
+            };
+            const deviceSignature = account.sign(anotherjson.stringify(deviceKeys));
+            deviceKeys.signatures = {
+                [this.crypto._userId]: {
+                    [`ed25519:${deviceId}`]: deviceSignature,
+                },
+            };
+            if (this.crypto._crossSigningInfo.getId("self_signing")) {
+                await this.crypto._crossSigningInfo.signObject(deviceKeys, "self_signing");
+            }
+
+            logger.log("Preparing one-time keys");
+            const oneTimeKeys = {};
+            for (const [keyId, key] of Object.entries(otks.curve25519)) {
+                const k: OneTimeKey = {key};
+                const signature = account.sign(anotherjson.stringify(k));
+                k.signatures = {
+                    [this.crypto._userId]: {
+                        [`ed25519:${deviceId}`]: signature,
+                    },
+                };
+                oneTimeKeys[`signed_curve25519:${keyId}`] = k;
+            }
+
+            logger.log("Preparing fallback keys");
+            const fallbackKeys = {};
+            for (const [keyId, key] of Object.entries(fallbacks.curve25519)) {
+                const k: OneTimeKey = {key, fallback: true};
+                const signature = account.sign(anotherjson.stringify(k));
+                k.signatures = {
+                    [this.crypto._userId]: {
+                        [`ed25519:${deviceId}`]: signature,
+                    },
+                };
+                fallbackKeys[`signed_curve25519:${keyId}`] = k;
+            }
+
+            logger.log("Uploading keys to server");
+            await this.crypto._baseApis._http.authedRequest(
+                undefined,
+                "POST",
+                "/keys/upload/" + encodeURI(deviceId),
+                undefined,
+                {
+                    "device_keys": deviceKeys,
+                    "one_time_keys": oneTimeKeys,
+                    "org.matrix.msc2732.fallback_keys": fallbackKeys,
+                },
+            );
+            logger.log("Done dehydrating");
+
+            // dehydrate again in a week
+            this.timeoutId = global.setTimeout(
+                this.dehydrateDevice.bind(this), oneweek,
+            );
+
+            return deviceId;
+        } finally {
+            this.inProgress = false;
+        }
+    }
+
+    private stop() {
+        if (this.timeoutId) {
+            global.clearTimeout(this.timeoutId);
+            this.timeoutId = undefined;
+        }
+    }
+}

src/crypto/olmlib.js

@@ -207,6 +207,25 @@ export async function ensureOlmSessionsForDevices(
         for (const deviceInfo of devices) {
             const deviceId = deviceInfo.deviceId;
             const key = deviceInfo.getIdentityKey();
+
+            if (key === olmDevice.deviceCurve25519Key) {
+                // We should never be trying to start a session with ourself.
+                // Apart from talking to yourself being the first sign of madness,
+                // olm sessions can't do this because they get confused when
+                // they get a message and see that the 'other side' has started a
+                // new chain when this side has an active sender chain.
+                // If you see this message being logged in the wild, we should find
+                // the thing that is trying to send Olm messages to itself and fix it.
+                logger.info("Attempted to start session with ourself! Ignoring");
+                // We must fill in the section in the return value though, as callers
+                // expect it to be there.
+                result[userId][deviceId] = {
+                    device: deviceInfo,
+                    sessionId: null,
+                };
+                continue;
+            }
+
             if (!olmDevice._sessionsInProgress[key]) {
                 // pre-emptively mark the session as in-progress to avoid race
                 // conditions.  If we find that we already have a session, then
@@ -238,6 +257,11 @@ export async function ensureOlmSessionsForDevices(
                 delete resolveSession[key];
             }
             if (sessionId === null || force) {
+                if (force) {
+                    logger.info("Forcing new Olm session for " + userId + ":" + deviceId);
+                } else {
+                    logger.info("Making new Olm session for " + userId + ":" + deviceId);
+                }
                 devicesWithoutSession.push([userId, deviceId]);
             }
             result[userId][deviceId] = {
@@ -277,6 +301,14 @@ export async function ensureOlmSessionsForDevices(
             const deviceInfo = devices[j];
             const deviceId = deviceInfo.deviceId;
             const key = deviceInfo.getIdentityKey();
+
+            if (key === olmDevice.deviceCurve25519Key) {
+                // We've already logged about this above. Skip here too
+                // otherwise we'll log saying there are no one-time keys
+                // which will be confusing.
+                continue;
+            }
+
             if (result[userId][deviceId].sessionId && !force) {
                 // we already have a result for this device
                 continue;

src/crypto/store/indexeddb-crypto-store-backend.js

@@ -131,7 +131,7 @@ export class Backend {
 
         cursorReq.onsuccess = (ev) => {
             const cursor = ev.target.result;
-            if(!cursor) {
+            if (!cursor) {
                 // no match found
                 callback(null);
                 return;

src/crypto/store/localStorage-crypto-store.js

@@ -369,7 +369,7 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore {
 
     getSecretStorePrivateKey(txn, func, type) {
         const key = getJsonItem(this.store, E2E_PREFIX + `ssss_cache.${type}`);
-        func(key ? Uint8Array.from(key) : key);
+        func(key);
     }
 
     storeCrossSigningKeys(txn, keys) {
@@ -380,7 +380,7 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore {
 
     storeSecretStorePrivateKey(txn, type, key) {
         setJsonItem(
-            this.store, E2E_PREFIX + `ssss_cache.${type}`, Array.from(key),
+            this.store, E2E_PREFIX + `ssss_cache.${type}`, key,
         );
     }
 

src/crypto/verification/Base.js

@@ -25,8 +25,7 @@ import {EventEmitter} from 'events';
 import {logger} from '../../logger';
 import {DeviceInfo} from '../deviceinfo';
 import {newTimeoutError} from "./Error";
-import {CrossSigningInfo} from "../CrossSigning";
-import {decodeBase64} from "../olmlib";
+import {requestKeysDuringVerification} from "../CrossSigning";
 
 const timeoutException = new Error("Verification timed out");
 
@@ -79,8 +78,6 @@ export class VerificationBase extends EventEmitter {
         this._transactionTimeoutTimer = null;
     }
 
-    static keyRequestTimeoutMs = 1000 * 60;
-
     get initiatedByMe() {
         // if there is no start event yet,
         // we probably want to send it,
@@ -198,93 +195,7 @@ export class VerificationBase extends EventEmitter {
         if (!this._done) {
             this.request.onVerifierFinished();
             this._resolve();
-
-            //#region Cross-signing keys request
-            // If this is a self-verification, ask the other party for keys
-            if (this._baseApis.getUserId() !== this.userId) {
-                return;
-            }
-            // FIXME: This is a lot of logic that isn't anything to do with verification
-            // and probably ought to be somewhere else.
-            console.log("VerificationBase.done: Self-verification done; requesting keys");
-            /* This happens asynchronously, and we're not concerned about
-             * waiting for it.  We return here in order to test. */
-            return new Promise((resolve, reject) => {
-                const client = this._baseApis;
-                const original = client._crypto._crossSigningInfo;
-
-                /* We already have all of the infrastructure we need to validate and
-                 * cache cross-signing keys, so instead of replicating that, here we
-                 * set up callbacks that request them from the other device and call
-                 * CrossSigningInfo.getCrossSigningKey() to validate/cache */
-                const crossSigning = new CrossSigningInfo(
-                    original.userId,
-                    { getCrossSigningKey: async (type) => {
-                        console.debug("VerificationBase.done: requesting secret",
-                                      type, this.deviceId);
-                        const { promise } = client.requestSecret(
-                            `m.cross_signing.${type}`, [this.deviceId],
-                        );
-                        const result = await promise;
-                        const decoded = decodeBase64(result);
-                        return Uint8Array.from(decoded);
-                    } },
-                    original._cacheCallbacks,
-                );
-                crossSigning.keys = original.keys;
-
-                // XXX: get all keys out if we get one key out
-                // https://github.com/vector-im/riot-web/issues/12604
-                // then change here to reject on the timeout
-                /* Requests can be ignored, so don't wait around forever */
-                const timeout = new Promise((resolve, reject) => {
-                    setTimeout(
-                        resolve,
-                        VerificationBase.keyRequestTimeoutMs,
-                        new Error("Timeout"),
-                    );
-                });
-
-                // also request and cache the key backup key
-                const backupKeyPromise = new Promise(async resolve => {
-                    const cachedKey = await client._crypto.getSessionBackupPrivateKey();
-                    if (!cachedKey) {
-                        logger.info("No cached backup key found. Requesting...");
-                        const secretReq = client.requestSecret(
-                            'm.megolm_backup.v1', [this.deviceId],
-                        );
-                        const base64Key = await secretReq.promise;
-                        logger.info("Got key backup key, decoding...");
-                        const decodedKey = decodeBase64(base64Key);
-                        logger.info("Decoded backup key, storing...");
-                        client._crypto.storeSessionBackupPrivateKey(
-                            Uint8Array.from(decodedKey),
-                        );
-                        logger.info("Backup key stored. Starting backup restore...");
-                        const backupInfo = await client.getKeyBackupVersion();
-                        // no need to await for this - just let it go in the bg
-                        client.restoreKeyBackupWithCache(
-                            undefined, undefined, backupInfo,
-                        ).then(() => {
-                            logger.info("Backup restored.");
-                        });
-                    }
-                    resolve();
-                });
-
-                /* We call getCrossSigningKey() for its side-effects */
-                return Promise.race([
-                    Promise.all([
-                        crossSigning.getCrossSigningKey("self_signing"),
-                        crossSigning.getCrossSigningKey("user_signing"),
-                        backupKeyPromise,
-                    ]),
-                    timeout,
-                ]).then(resolve, reject);
-            }).catch((e) => {
-                console.warn("VerificationBase: failure while requesting keys:", e);
-            });
-            //#endregion
+            return requestKeysDuringVerification(this._baseApis, this.userId, this.deviceId);
         }
     }
 

src/crypto/verification/QRCode.js

@@ -26,6 +26,7 @@ import {
     newUserCancelledError,
 } from './Error';
 import {encodeUnpaddedBase64, decodeBase64} from "../olmlib";
+import {logger} from '../../logger';
 
 export const SHOW_QR_CODE_METHOD = "m.qr_code.show.v1";
 export const SCAN_QR_CODE_METHOD = "m.qr_code.scan.v1";
@@ -94,7 +95,7 @@ export class ReciprocateQRCode extends Base {
             if (!targetKey) throw newKeyMismatchError();
 
             if (keyInfo !== targetKey) {
-                console.error("key ID from key info does not match");
+                logger.error("key ID from key info does not match");
                 throw newKeyMismatchError();
             }
             for (const deviceKeyId in device.keys) {
@@ -102,7 +103,7 @@ export class ReciprocateQRCode extends Base {
                 const deviceTargetKey = keys[deviceKeyId];
                 if (!deviceTargetKey) throw newKeyMismatchError();
                 if (device.keys[deviceKeyId] !== deviceTargetKey) {
-                    console.error("master key does not match");
+                    logger.error("master key does not match");
                     throw newKeyMismatchError();
                 }
             }

src/crypto/verification/SAS.js

@@ -175,11 +175,33 @@ function calculateMAC(olmSAS, method) {
     };
 }
 
+const calculateKeyAgreement = {
+    "curve25519-hkdf-sha256": function(sas, olmSAS, bytes) {
+        const ourInfo = `${sas._baseApis.getUserId()}|${sas._baseApis.deviceId}|`
+              + `${sas.ourSASPubKey}|`;
+        const theirInfo = `${sas.userId}|${sas.deviceId}|${sas.theirSASPubKey}|`;
+        const sasInfo =
+            "MATRIX_KEY_VERIFICATION_SAS|"
+              + (sas.initiatedByMe ? ourInfo + theirInfo : theirInfo + ourInfo)
+              + sas._channel.transactionId;
+        return olmSAS.generate_bytes(sasInfo, bytes);
+    },
+    "curve25519": function(sas, olmSAS, bytes) {
+        const ourInfo = `${sas._baseApis.getUserId()}${sas._baseApis.deviceId}`;
+        const theirInfo = `${sas.userId}${sas.deviceId}`;
+        const sasInfo =
+            "MATRIX_KEY_VERIFICATION_SAS"
+              + (sas.initiatedByMe ? ourInfo + theirInfo : theirInfo + ourInfo)
+              + sas._channel.transactionId;
+        return olmSAS.generate_bytes(sasInfo, bytes);
+    },
+};
+
 /* lists of algorithms/methods that are supported.  The key agreement, hashes,
  * and MAC lists should be sorted in order of preference (most preferred
  * first).
  */
-const KEY_AGREEMENT_LIST = ["curve25519"];
+const KEY_AGREEMENT_LIST = ["curve25519-hkdf-sha256", "curve25519"];
 const HASHES_LIST = ["sha256"];
 const MAC_LIST = ["hkdf-hmac-sha256", "hmac-sha256"];
 const SAS_LIST = Object.keys(sasGenerators);
@@ -291,12 +313,14 @@ export class SAS extends Base {
         if (typeof content.commitment !== "string") {
             throw newInvalidMessageError();
         }
+        const keyAgreement = content.key_agreement_protocol;
         const macMethod = content.message_authentication_code;
         const hashCommitment = content.commitment;
         const olmSAS = new global.Olm.SAS();
         try {
+            this.ourSASPubKey = olmSAS.get_pubkey();
             await this._send("m.key.verification.key", {
-                key: olmSAS.get_pubkey(),
+                key: this.ourSASPubKey,
             });
 
 
@@ -308,13 +332,10 @@ export class SAS extends Base {
             if (olmutil.sha256(commitmentStr) !== hashCommitment) {
                 throw newMismatchedCommitmentError();
             }
+            this.theirSASPubKey = content.key;
             olmSAS.set_their_key(content.key);
 
-            const sasInfo = "MATRIX_KEY_VERIFICATION_SAS"
-                  + this._baseApis.getUserId() + this._baseApis.deviceId
-                  + this.userId + this.deviceId
-                  + this._channel.transactionId;
-            const sasBytes = olmSAS.generate_bytes(sasInfo, 6);
+            const sasBytes = calculateKeyAgreement[keyAgreement](this, olmSAS, 6);
             const verifySAS = new Promise((resolve, reject) => {
                 this.sasEvent = {
                     sas: generateSas(sasBytes, sasMethods),
@@ -394,16 +415,14 @@ export class SAS extends Base {
             let e = await this._waitForEvent("m.key.verification.key");
             // FIXME: make sure event is properly formed
             content = e.getContent();
+            this.theirSASPubKey = content.key;
             olmSAS.set_their_key(content.key);
+            this.ourSASPubKey = olmSAS.get_pubkey();
             await this._send("m.key.verification.key", {
-                key: olmSAS.get_pubkey(),
+                key: this.ourSASPubKey,
             });
 
-            const sasInfo = "MATRIX_KEY_VERIFICATION_SAS"
-                  + this.userId + this.deviceId
-                  + this._baseApis.getUserId() + this._baseApis.deviceId
-                  + this._channel.transactionId;
-            const sasBytes = olmSAS.generate_bytes(sasInfo, 6);
+            const sasBytes = calculateKeyAgreement[keyAgreement](this, olmSAS, 6);
             const verifySAS = new Promise((resolve, reject) => {
                 this.sasEvent = {
                     sas: generateSas(sasBytes, sasMethods),
@@ -411,7 +430,7 @@ export class SAS extends Base {
                         try {
                             await this._sendMAC(olmSAS, macMethod);
                             resolve();
-                        } catch(err) {
+                        } catch (err) {
                             reject(err);
                         }
                     },

src/crypto/verification/request/ToDeviceChannel.js

@@ -356,4 +356,12 @@ export class ToDeviceRequests {
             }
         }
     }
+
+    getRequestsInProgress(userId) {
+        const requestsByTxnId = this._requestsByUserId.get(userId);
+        if (requestsByTxnId) {
+            return Array.from(requestsByTxnId.values()).filter(r => r.pending);
+        }
+        return [];
+    }
 }

src/crypto/verification/request/VerificationRequest.js

@@ -25,15 +25,17 @@ import {
 } from "../Error";
 import {QRCodeData, SCAN_QR_CODE_METHOD} from "../QRCode";
 
-// the recommended amount of time before a verification request
-// should be (automatically) cancelled without user interaction
-// and ignored.
-const VERIFICATION_REQUEST_TIMEOUT = 10 * 60 * 1000; //10m
+// How long after the event's timestamp that the request times out
+const TIMEOUT_FROM_EVENT_TS = 10 * 60 * 1000; // 10 minutes
+
+// How long after we receive the event that the request times out
+const TIMEOUT_FROM_EVENT_RECEIPT = 2 * 60 * 1000; // 2 minutes
+
 // to avoid almost expired verification notifications
 // from showing a notification and almost immediately
 // disappearing, also ignore verification requests that
 // are this amount of time away from expiring.
-const VERIFICATION_REQUEST_MARGIN = 3 * 1000; //3s
+const VERIFICATION_REQUEST_MARGIN = 3 * 1000; // 3 seconds
 
 
 export const EVENT_PREFIX = "m.key.verification.";
@@ -80,6 +82,9 @@ export class VerificationRequest extends EventEmitter {
         // cross-signing identity reset between the .ready and .start event
         // and signing the wrong key after .start
         this._qrCodeData = null;
+
+        // The timestamp when we received the request event from the other side
+        this._requestReceivedAt = null;
     }
 
     /**
@@ -165,12 +170,26 @@ export class VerificationRequest extends EventEmitter {
         return this._chosenMethod;
     }
 
+    calculateEventTimeout(event) {
+        let effectiveExpiresAt = this.channel.getTimestamp(event)
+            + TIMEOUT_FROM_EVENT_TS;
+
+        if (this._requestReceivedAt && !this.initiatedByMe &&
+            this.phase <= PHASE_REQUESTED
+        ) {
+            const expiresAtByReceipt = this._requestReceivedAt
+                + TIMEOUT_FROM_EVENT_RECEIPT;
+            effectiveExpiresAt = Math.min(effectiveExpiresAt, expiresAtByReceipt);
+        }
+
+        return Math.max(0, effectiveExpiresAt - Date.now());
+    }
+
     /** The current remaining amount of ms before the request should be automatically cancelled */
     get timeout() {
         const requestEvent = this._getEventByEither(REQUEST_TYPE);
         if (requestEvent) {
-            const elapsed = Date.now() - this.channel.getTimestamp(requestEvent);
-            return Math.max(0, VERIFICATION_REQUEST_TIMEOUT - elapsed);
+            return this.calculateEventTimeout(requestEvent);
         }
         return 0;
     }
@@ -735,7 +754,7 @@ export class VerificationRequest extends EventEmitter {
 
     _setupTimeout(phase) {
         const shouldTimeout = !this._timeoutTimer && !this.observeOnly &&
-            phase === PHASE_REQUESTED && this.initiatedByMe;
+            phase === PHASE_REQUESTED;
 
         if (shouldTimeout) {
             this._timeoutTimer = setTimeout(this._cancelOnTimeout, this.timeout);
@@ -754,7 +773,17 @@ export class VerificationRequest extends EventEmitter {
 
     _cancelOnTimeout = () => {
         try {
-            this.cancel({reason: "Other party didn't accept in time", code: "m.timeout"});
+            if (this.initiatedByMe) {
+                this.cancel({
+                    reason: "Other party didn't accept in time",
+                    code: "m.timeout",
+                });
+            } else {
+                this.cancel({
+                    reason: "User didn't accept in time",
+                    code: "m.timeout",
+                });
+            }
         } catch (err) {
             logger.error("Error while cancelling verification request", err);
         }
@@ -791,16 +820,8 @@ export class VerificationRequest extends EventEmitter {
         if (!isLiveEvent) {
             this._observeOnly = true;
         }
-        // a timestamp is not provided on all to_device events
-        const timestamp = this.channel.getTimestamp(event);
-        if (Number.isFinite(timestamp)) {
-            const elapsed = Date.now() - timestamp;
-            // don't allow interaction on old requests
-            if (elapsed > (VERIFICATION_REQUEST_TIMEOUT - VERIFICATION_REQUEST_MARGIN) ||
-                elapsed < -(VERIFICATION_REQUEST_TIMEOUT / 2)
-            ) {
-                this._observeOnly = true;
-            }
+        if (this.calculateEventTimeout(event) < VERIFICATION_REQUEST_MARGIN) {
+            this._observeOnly = true;
         }
     }
 
@@ -819,6 +840,8 @@ export class VerificationRequest extends EventEmitter {
                     this._eventsByThem.delete(type);
                 }
             }
+            // also remember when we received the request event
+            this._requestReceivedAt = Date.now();
         }
     }
 

src/filter-component.js

@@ -108,8 +108,9 @@ FilterComponent.prototype._checkFields =
         }
 
         const allowed_values = self[name];
-        if (allowed_values) {
-            if (!allowed_values.map(match_func)) {
+        if (allowed_values && allowed_values.length > 0) {
+            const anyMatch = allowed_values.some(match_func);
+            if (!anyMatch) {
                 return false;
             }
         }

src/http-api.js

@@ -651,12 +651,10 @@ MatrixHttpApi.prototype = {
 
         const self = this;
         if (this.opts.extraParams) {
-            for (const key in this.opts.extraParams) {
-                if (!this.opts.extraParams.hasOwnProperty(key)) {
-                    continue;
-                }
-                queryParams[key] = this.opts.extraParams[key];
-            }
+            queryParams = {
+              ...queryParams,
+              ...this.opts.extraParams,
+            };
         }
 
         const headers = utils.extend({}, opts.headers || {});
@@ -890,7 +888,7 @@ function getResponseContentType(response) {
 
     try {
         return parseContentType(contentType);
-    } catch(e) {
+    } catch (e) {
         throw new Error(`Error parsing Content-Type '${contentType}': ${e}`);
     }
 }
@@ -963,7 +961,7 @@ export async function retryNetworkOperation(maxAttempts, callback) {
         try {
             if (attempts > 0) {
                 const timeout = 1000 * Math.pow(2, attempts);
-                console.log(`network operation failed ${attempts} times,` +
+                logger.log(`network operation failed ${attempts} times,` +
                     ` retrying in ${timeout}ms...`);
                 await new Promise(r => setTimeout(r, timeout));
             }

src/index.ts

@@ -22,6 +22,7 @@ matrixcs.request(request);
 utils.runPolyfills();
 
 try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
     const crypto = require('crypto');
     utils.setCrypto(crypto);
 } catch (err) {

src/interactive-auth.js

@@ -143,11 +143,19 @@ InteractiveAuth.prototype = {
             this._resolveFunc = resolve;
             this._rejectFunc = reject;
 
-            // if we have no flows, try a request (we'll have
-            // just a session ID in _data if resuming)
-            if (!this._data.flows) {
+            const hasFlows = this._data && this._data.flows;
+
+            // if we have no flows, try a request to acquire the flows
+            if (!hasFlows) {
                 if (this._busyChangedCallback) this._busyChangedCallback(true);
-                this._doRequest(this._data).finally(() => {
+                // use the existing sessionid, if one is present.
+                let auth = null;
+                if (this._data.session) {
+                    auth = {
+                        session: this._data.session,
+                    };
+                }
+                this._doRequest(auth).finally(() => {
                     if (this._busyChangedCallback) this._busyChangedCallback(false);
                 });
             } else {
@@ -186,7 +194,11 @@ InteractiveAuth.prototype = {
                 }
                 authDict = {
                     type: EMAIL_STAGE_TYPE,
+                    // TODO: Remove `threepid_creds` once servers support proper UIA
+                    // See https://github.com/matrix-org/synapse/issues/5665
+                    // See https://github.com/matrix-org/matrix-doc/issues/2220
                     threepid_creds: creds,
+                    threepidCreds: creds,
                 };
             }
         }
@@ -264,11 +276,16 @@ InteractiveAuth.prototype = {
             }
         }
 
-        // use the sessionid from the last request.
-        const auth = {
-            session: this._data.session,
-        };
-        utils.extend(auth, authData);
+        // use the sessionid from the last request, if one is present.
+        let auth;
+        if (this._data.session) {
+            auth = {
+                session: this._data.session,
+            };
+            utils.extend(auth, authData);
+        } else {
+            auth = authData;
+        }
 
         try {
             // NB. the 'background' flag is deprecated by the busyChanged
@@ -325,7 +342,7 @@ InteractiveAuth.prototype = {
         } catch (error) {
             // sometimes UI auth errors don't come with flows
             const errorFlows = error.data ? error.data.flows : null;
-            const haveFlows = Boolean(this._data.flows) || Boolean(errorFlows);
+            const haveFlows = this._data.flows || Boolean(errorFlows);
             if (error.httpStatus !== 401 || !error.data || !haveFlows) {
                 // doesn't look like an interactive-auth failure.
                 if (!background) {
@@ -420,7 +437,7 @@ InteractiveAuth.prototype = {
             return;
         }
 
-        if (this._data.errcode || this._data.error) {
+        if (this._data && this._data.errcode || this._data.error) {
             this._stateUpdatedCallback(nextStage, {
                 errcode: this._data.errcode || "",
                 error: this._data.error || "",

src/logger.ts

@@ -40,11 +40,13 @@ log.methodFactory = function(methodName, logLevel, loggerName) {
             methodName === "warn" ||
             methodName === "trace" ||
             methodName === "info";
+        /* eslint-disable no-console */
         if (supportedByConsole) {
             return console[methodName](...args);
         } else {
             return console.log(...args);
         }
+        /* eslint-enable no-console */
     };
 };
 

src/matrix.ts

@@ -1,7 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,8 +16,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import type Request from "request";
+
 import {MemoryCryptoStore} from "./crypto/store/memory-crypto-store";
+import {LocalStorageCryptoStore} from "./crypto/store/localStorage-crypto-store";
+import {IndexedDBCryptoStore} from "./crypto/store/indexeddb-crypto-store";
 import {MemoryStore} from "./store/memory";
+import {StubStore} from "./store/stub";
+import {LocalIndexedDBStoreBackend} from "./store/indexeddb-local-backend";
+import {RemoteIndexedDBStoreBackend} from "./store/indexeddb-remote-backend";
 import {MatrixScheduler} from "./scheduler";
 import {MatrixClient} from "./client";
 
@@ -89,6 +96,10 @@ export function wrapRequest(wrapper) {
     };
 }
 
+type Store =
+    StubStore | MemoryStore | LocalIndexedDBStoreBackend | RemoteIndexedDBStoreBackend;
+
+type CryptoStore = MemoryCryptoStore | LocalStorageCryptoStore | IndexedDBCryptoStore;
 
 let cryptoStoreFactory = () => new MemoryCryptoStore;
 
@@ -102,6 +113,81 @@ export function setCryptoStoreFactory(fac) {
     cryptoStoreFactory = fac;
 }
 
+export interface ICreateClientOpts {
+    baseUrl: string;
+    idBaseUrl?: string;
+    store?: Store;
+    cryptoStore?: CryptoStore;
+    scheduler?: MatrixScheduler;
+    request?: Request;
+    userId?: string;
+    deviceId?: string;
+    accessToken?: string;
+    identityServer?: any;
+    localTimeoutMs?: number;
+    useAuthorizationHeader?: boolean;
+    timelineSupport?: boolean;
+    queryParams?: Record<string, unknown>;
+    deviceToImport?: {
+        olmDevice: {
+            pickledAccount: string;
+            sessions: Array<Record<string, any>>;
+            pickleKey: string;
+        };
+        userId: string;
+        deviceId: string;
+    };
+    pickleKey?: string;
+    sessionStore?: any;
+    unstableClientRelationAggregation?: boolean;
+    verificationMethods?: Array<any>;
+    forceTURN?: boolean;
+    supportsCallTransfer?: boolean,
+    fallbackICEServerAllowed?: boolean;
+    cryptoCallbacks?: ICryptoCallbacks;
+}
+
+export interface ICryptoCallbacks {
+    getCrossSigningKey?: (keyType: string, pubKey: Uint8Array) => Promise<Uint8Array>;
+    saveCrossSigningKeys?: (keys: Record<string, Uint8Array>) => void;
+    shouldUpgradeDeviceVerifications?: (
+        users: Record<string, any>
+    ) => Promise<string[]>;
+    getSecretStorageKey?: (
+        keys: {keys: Record<string, ISecretStorageKeyInfo>}, name: string
+    ) => Promise<[string, Uint8Array] | null>;
+    cacheSecretStorageKey?: (
+        keyId: string, keyInfo: ISecretStorageKeyInfo, key: Uint8Array
+    ) => void;
+    onSecretRequested?: (
+        userId: string, deviceId: string,
+        requestId: string, secretName: string, deviceTrust: IDeviceTrustLevel
+    ) => Promise<string>;
+    getDehydrationKey?: (
+        keyInfo: ISecretStorageKeyInfo,
+        checkFunc: (Uint8Array) => void,
+    ) => Promise<Uint8Array>;
+}
+
+// TODO: Move this to `SecretStorage` once converted
+export interface ISecretStorageKeyInfo {
+    passphrase?: {
+        algorithm: "m.pbkdf2";
+        iterations: number;
+        salt: string;
+    };
+    iv?: string;
+    mac?: string;
+}
+
+// TODO: Move this to `CrossSigning` once converted
+export interface IDeviceTrustLevel {
+    isVerified(): boolean;
+    isCrossSigningVerified(): boolean;
+    isLocallyVerified(): boolean;
+    isTofu(): boolean;
+}
+
 /**
  * Construct a Matrix Client. Similar to {@link module:client.MatrixClient}
  * except that the 'request', 'store' and 'scheduler' dependencies are satisfied.
@@ -125,10 +211,10 @@ export function setCryptoStoreFactory(fac) {
  * @see {@link module:client.MatrixClient} for the full list of options for
  * <code>opts</code>.
  */
-export function createClient(opts) {
+export function createClient(opts: ICreateClientOpts | string) {
     if (typeof opts === "string") {
         opts = {
-            "baseUrl": opts,
+            "baseUrl": opts as string,
         };
     }
     opts.request = opts.request || requestInstance;
@@ -167,7 +253,7 @@ export function createClient(opts) {
  * @param {requestCallback} callback The request callback.
  */
 
- /**
+/**
   * The request callback interface for performing HTTP requests. This matches the
   * API for the {@link https://github.com/request/request#requestoptions-callback|
   * request NPM module}. The SDK will implement a callback which meets this

src/models/event-timeline-set.js

@@ -647,7 +647,8 @@ EventTimelineSet.prototype.compareEventOrdering = function(eventId1, eventId2) {
     if (timeline1 === timeline2) {
         // both events are in the same timeline - figure out their
         // relative indices
-        let idx1, idx2;
+        let idx1;
+        let idx2;
         const events = timeline1.getEvents();
         for (let idx = 0; idx < events.length &&
              (idx1 === undefined || idx2 === undefined); idx++) {
@@ -805,7 +806,7 @@ EventTimelineSet.prototype.aggregateRelations = function(event) {
             this.room,
         );
         isNewRelations = true;
-        relatesToEvent = this.findEventById(relatesToEventId);
+        relatesToEvent = this.findEventById(relatesToEventId) || this.room.getPendingEvent(relatesToEventId);
         if (relatesToEvent) {
             relationsWithEventType.setTargetEvent(relatesToEvent);
         }

src/models/event.js

@@ -87,7 +87,7 @@ export const MatrixEvent = function(
     // amount of needless string duplication. This can save moderate amounts of
     // memory (~10% on a 350MB heap).
     // 'membership' at the event level (rather than the content level) is a legacy
-    // field that Riot never otherwise looks at, but it will still take up a lot
+    // field that Element never otherwise looks at, but it will still take up a lot
     // of space if we don't intern it.
     ["state_key", "type", "sender", "room_id", "membership"].forEach((prop) => {
         if (!event[prop]) {
@@ -144,6 +144,10 @@ export const MatrixEvent = function(
      */
     this._forwardingCurve25519KeyChain = [];
 
+    /* where the decryption key is untrusted
+     */
+    this._untrusted = null;
+
     /* if we have a process decrypting this event, a Promise which resolves
      * when it is finished. Normally null.
      */
@@ -160,12 +164,24 @@ export const MatrixEvent = function(
      * so it can be easily accessed from the timeline.
      */
     this.verificationRequest = null;
+
+    /* The txnId with which this event was sent if it was during this session,
+       allows for a unique ID which does not change when the event comes back down sync.
+     */
+    this._txnId = null;
+
+    /* Set an approximate timestamp for the event relative the local clock.
+     * This will inherently be approximate because it doesn't take into account
+     * the time between the server putting the 'age' field on the event as it sent
+     * it to us and the time we're now constructing this event, but that's better
+     * than assuming the local clock is in sync with the origin HS's clock.
+     */
+    this._localTimestamp = Date.now() - this.getAge();
 };
 utils.inherits(MatrixEvent, EventEmitter);
 
 
 utils.extend(MatrixEvent.prototype, {
-
     /**
      * Get the event_id for this event.
      * @return {string} The event ID, e.g. <code>$143350589368169JsLZx:localhost
@@ -303,11 +319,12 @@ utils.extend(MatrixEvent.prototype, {
 
     /**
      * Get the age of the event when this function was called.
-     * Relies on the local clock being in sync with the clock of the original homeserver.
+     * This is the 'age' field adjusted according to how long this client has
+     * had the event.
      * @return {Number} The age of this event in milliseconds.
      */
     getLocalAge: function() {
-        return Date.now() - this.getTs();
+        return Date.now() - this._localTimestamp;
     },
 
     /**
@@ -390,11 +407,12 @@ utils.extend(MatrixEvent.prototype, {
      * @internal
      *
      * @param {module:crypto} crypto crypto module
+     * @param {bool} isRetry True if this is a retry (enables more logging)
      *
      * @returns {Promise} promise which resolves (to undefined) when the decryption
      * attempt is completed.
      */
-    attemptDecryption: async function(crypto) {
+    attemptDecryption: async function(crypto, isRetry) {
         // start with a couple of sanity checks.
         if (!this.isEncrypted()) {
             throw new Error("Attempt to decrypt event which isn't encrypted");
@@ -406,7 +424,7 @@ utils.extend(MatrixEvent.prototype, {
         ) {
             // we may want to just ignore this? let's start with rejecting it.
             throw new Error(
-                "Attempt to decrypt event which has already been encrypted",
+                "Attempt to decrypt event which has already been decrypted",
             );
         }
 
@@ -424,7 +442,7 @@ utils.extend(MatrixEvent.prototype, {
             return this._decryptionPromise;
         }
 
-        this._decryptionPromise = this._decryptionLoop(crypto);
+        this._decryptionPromise = this._decryptionLoop(crypto, isRetry);
         return this._decryptionPromise;
     },
 
@@ -469,7 +487,7 @@ utils.extend(MatrixEvent.prototype, {
         return recipients;
     },
 
-    _decryptionLoop: async function(crypto) {
+    _decryptionLoop: async function(crypto, isRetry) {
         // make sure that this method never runs completely synchronously.
         // (doing so would mean that we would clear _decryptionPromise *before*
         // it is set in attemptDecryption - and hence end up with a stuck
@@ -486,13 +504,18 @@ utils.extend(MatrixEvent.prototype, {
                     res = this._badEncryptedMessage("Encryption not enabled");
                 } else {
                     res = await crypto.decryptEvent(this);
+                    if (isRetry) {
+                        logger.info(`Decrypted event on retry (id=${this.getId()})`);
+                    }
                 }
             } catch (e) {
                 if (e.name !== "DecryptionError") {
                     // not a decryption error: log the whole exception as an error
                     // (and don't bother with a retry)
+                    const re = isRetry ? 're' : '';
                     logger.error(
-                        `Error decrypting event (id=${this.getId()}): ${e.stack || e}`,
+                        `Error ${re}decrypting event ` +
+                        `(id=${this.getId()}): ${e.stack || e}`,
                     );
                     this._decryptionPromise = null;
                     this._retryDecryption = false;
@@ -593,6 +616,7 @@ utils.extend(MatrixEvent.prototype, {
             decryptionResult.claimedEd25519Key || null;
         this._forwardingCurve25519KeyChain =
             decryptionResult.forwardingCurve25519KeyChain || [];
+        this._untrusted = decryptionResult.untrusted || false;
     },
 
     /**
@@ -611,7 +635,7 @@ utils.extend(MatrixEvent.prototype, {
      * @return {boolean} True if this event is encrypted.
      */
     isEncrypted: function() {
-        return this.event.type === "m.room.encrypted";
+        return !this.isState() && this.event.type === "m.room.encrypted";
     },
 
     /**
@@ -683,6 +707,16 @@ utils.extend(MatrixEvent.prototype, {
         return this._forwardingCurve25519KeyChain;
     },
 
+    /**
+     * Whether the decryption key was obtained from an untrusted source. If so,
+     * we cannot verify the authenticity of the message.
+     *
+     * @return {boolean}
+     */
+    isKeySourceUntrusted: function() {
+        return this._untrusted;
+    },
+
     getUnsigned: function() {
         return this.event.unsigned || {};
     },
@@ -879,6 +913,8 @@ utils.extend(MatrixEvent.prototype, {
     /**
      * Set an event that replaces the content of this event, through an m.replace relation.
      *
+     * @fires module:models/event.MatrixEvent#"Event.replaced"
+     *
      * @param {MatrixEvent?} newEvent the event with the replacing content, if any.
      */
     makeReplaced(newEvent) {
@@ -1064,6 +1100,14 @@ utils.extend(MatrixEvent.prototype, {
     setVerificationRequest: function(request) {
         this.verificationRequest = request;
     },
+
+    setTxnId(txnId) {
+        this._txnId = txnId;
+    },
+
+    getTxnId() {
+        return this._txnId;
+    },
 });
 
 

src/models/relations.js

@@ -16,6 +16,7 @@ limitations under the License.
 
 import {EventEmitter} from 'events';
 import {EventStatus} from '../models/event';
+import {logger} from '../logger';
 
 /**
  * A container for relation events that supports easy access to common ways of
@@ -60,7 +61,7 @@ export class Relations extends EventEmitter {
 
         const relation = event.getRelation();
         if (!relation) {
-            console.error("Event must have relation info");
+            logger.error("Event must have relation info");
             return;
         }
 
@@ -68,7 +69,7 @@ export class Relations extends EventEmitter {
         const eventType = event.getType();
 
         if (this.relationType !== relationType || this.eventType !== eventType) {
-            console.error("Event relation info doesn't match this container");
+            logger.error("Event relation info doesn't match this container");
             return;
         }
 
@@ -104,7 +105,7 @@ export class Relations extends EventEmitter {
 
         const relation = event.getRelation();
         if (!relation) {
-            console.error("Event must have relation info");
+            logger.error("Event must have relation info");
             return;
         }
 
@@ -112,7 +113,7 @@ export class Relations extends EventEmitter {
         const eventType = event.getType();
 
         if (this.relationType !== relationType || this.eventType !== eventType) {
-            console.error("Event relation info doesn't match this container");
+            logger.error("Event relation info doesn't match this container");
             return;
         }
 

src/models/room-member.js

@@ -20,7 +20,7 @@ limitations under the License.
  */
 
 import {EventEmitter} from "events";
-import {getHttpUriForMxc, getIdenticonUri} from "../content-repo";
+import {getHttpUriForMxc} from "../content-repo";
 import * as utils from "../utils";
 
 /**
@@ -274,10 +274,6 @@ RoomMember.prototype.getAvatarUrl =
     );
     if (httpUrl) {
         return httpUrl;
-    } else if (allowDefault) {
-        return getIdenticonUri(
-            baseUrl, this.userId, width, height,
-        );
     }
     return null;
 };
@@ -286,9 +282,9 @@ RoomMember.prototype.getAvatarUrl =
  * @return {string} the mxc avatar url
  */
 RoomMember.prototype.getMxcAvatarUrl = function() {
-    if(this.events.member) {
+    if (this.events.member) {
         return this.events.member.getDirectionalContent().avatar_url;
-    } else if(this.user) {
+    } else if (this.user) {
         return this.user.avatarUrl;
     }
     return null;

src/models/room-state.js

@@ -68,9 +68,7 @@ export function RoomState(roomId, oobMemberFlags = undefined) {
     this.members = {
         // userId: RoomMember
     };
-    this.events = {
-        // eventType: { stateKey: MatrixEvent }
-    };
+    this.events = new Map(); // Map<eventType, Map<stateKey, MatrixEvent>>
     this.paginationToken = null;
 
     this._sentinels = {
@@ -211,14 +209,14 @@ RoomState.prototype.getSentinelMember = function(userId) {
  * <code>undefined</code>, else a single event (or null if no match found).
  */
 RoomState.prototype.getStateEvents = function(eventType, stateKey) {
-    if (!this.events[eventType]) {
+    if (!this.events.has(eventType)) {
         // no match
         return stateKey === undefined ? [] : null;
     }
     if (stateKey === undefined) { // return all values
-        return utils.values(this.events[eventType]);
+        return Array.from(this.events.get(eventType).values());
     }
-    const event = this.events[eventType][stateKey];
+    const event = this.events.get(eventType).get(stateKey);
     return event ? event : null;
 };
 
@@ -238,9 +236,8 @@ RoomState.prototype.clone = function() {
     const status = this._oobMemberFlags.status;
     this._oobMemberFlags.status = OOB_STATUS_NOTSTARTED;
 
-    Object.values(this.events).forEach((eventsByStateKey) => {
-        const eventsForType = Object.values(eventsByStateKey);
-        copy.setStateEvents(eventsForType);
+    Array.from(this.events.values()).forEach((eventsByStateKey) => {
+        copy.setStateEvents(Array.from(eventsByStateKey.values()));
     });
 
     // Ugly hack: see above
@@ -276,8 +273,8 @@ RoomState.prototype.clone = function() {
  */
 RoomState.prototype.setUnknownStateEvents = function(events) {
     const unknownStateEvents = events.filter((event) => {
-        return this.events[event.getType()] === undefined ||
-            this.events[event.getType()][event.getStateKey()] === undefined;
+        return !this.events.has(event.getType()) ||
+            !this.events.get(event.getType()).has(event.getStateKey());
     });
 
     this.setStateEvents(unknownStateEvents);
@@ -306,6 +303,7 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
             return;
         }
 
+        const lastStateEvent = self._getStateEventMatching(event);
         self._setStateEvent(event);
         if (event.getType() === "m.room.member") {
             _updateDisplayNameCache(
@@ -313,7 +311,7 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
             );
             _updateThirdPartyTokenCache(self, event);
         }
-        self.emit("RoomState.events", event, self);
+        self.emit("RoomState.events", event, self, lastStateEvent);
     });
 
     // update higher level data structures. This needs to be done AFTER the
@@ -385,10 +383,15 @@ RoomState.prototype._getOrCreateMember = function(userId, event) {
 };
 
 RoomState.prototype._setStateEvent = function(event) {
-    if (this.events[event.getType()] === undefined) {
-        this.events[event.getType()] = {};
+    if (!this.events.has(event.getType())) {
+        this.events.set(event.getType(), new Map());
     }
-    this.events[event.getType()][event.getStateKey()] = event;
+    this.events.get(event.getType()).set(event.getStateKey(), event);
+};
+
+RoomState.prototype._getStateEventMatching = function(event) {
+    if (!this.events.has(event.getType())) return null;
+    return this.events.get(event.getType()).get(event.getStateKey());
 };
 
 RoomState.prototype._updateMember = function(member) {
@@ -670,7 +673,7 @@ RoomState.prototype._maySendEventOfType = function(eventType, userId, state) {
         const userPowerLevel = power_levels.users && power_levels.users[userId];
         if (Number.isFinite(userPowerLevel)) {
             powerLevel = userPowerLevel;
-        } else if(Number.isFinite(power_levels.users_default)) {
+        } else if (Number.isFinite(power_levels.users_default)) {
             powerLevel = power_levels.users_default;
         }
 
@@ -769,8 +772,12 @@ function _updateDisplayNameCache(roomState, userId, displayName) {
  * @param {MatrixEvent} event The matrix event which caused this event to fire.
  * @param {RoomState} state The room state whose RoomState.events dictionary
  * was updated.
+ * @param {MatrixEvent} prevEvent The event being replaced by the new state, if
+ * known. Note that this can differ from `getPrevContent()` on the new state event
+ * as this is the store's view of the last state, not the previous state provided
+ * by the server.
  * @example
- * matrixClient.on("RoomState.events", function(event, state){
+ * matrixClient.on("RoomState.events", function(event, state, prevEvent){
  *   var newStateEvent = event;
  * });
  */

src/models/room.js

@@ -23,7 +23,7 @@ limitations under the License.
 import {EventEmitter} from "events";
 import {EventTimelineSet} from "./event-timeline-set";
 import {EventTimeline} from "./event-timeline";
-import {getHttpUriForMxc, getIdenticonUri} from "../content-repo";
+import {getHttpUriForMxc} from "../content-repo";
 import * as utils from "../utils";
 import {EventStatus, MatrixEvent} from "./event";
 import {RoomMember} from "./room-member";
@@ -37,8 +37,8 @@ import {ReEmitter} from '../ReEmitter';
 // room versions which are considered okay for people to run without being asked
 // to upgrade (ie: "stable"). Eventually, we should remove these when all homeservers
 // return an m.room_versions capability.
-const KNOWN_SAFE_ROOM_VERSION = '5';
-const SAFE_ROOM_VERSIONS = ['1', '2', '3', '4', '5'];
+const KNOWN_SAFE_ROOM_VERSION = '6';
+const SAFE_ROOM_VERSIONS = ['1', '2', '3', '4', '5', '6'];
 
 function synthesizeReceipt(userId, event, receiptType) {
     // console.log("synthesizing receipt for "+event.getId());
@@ -122,6 +122,10 @@ export function Room(roomId, client, myUserId, opts) {
     opts = opts || {};
     opts.pendingEventOrdering = opts.pendingEventOrdering || "chronological";
 
+    // In some cases, we add listeners for every displayed Matrix event, so it's
+    // common to have quite a few more than the default limit.
+    this.setMaxListeners(100);
+
     this.reEmitter = new ReEmitter(this);
 
     if (["chronological", "detached"].indexOf(opts.pendingEventOrdering) === -1) {
@@ -209,7 +213,7 @@ utils.inherits(Room, EventEmitter);
 Room.prototype.getVersion = function() {
     const createEvent = this.currentState.getStateEvents("m.room.create", "");
     if (!createEvent) {
-        logger.warn("Room " + this.room_id + " does not have an m.room.create event");
+        logger.warn("Room " + this.roomId + " does not have an m.room.create event");
         return '1';
     }
     const ver = createEvent.getContent()['room_version'];
@@ -354,21 +358,33 @@ Room.prototype.getPendingEvents = function() {
 
 /**
  * Check whether the pending event list contains a given event by ID.
+ * If pending event ordering is not "detached" then this returns false.
  *
  * @param {string} eventId The event ID to check for.
  * @return {boolean}
- * @throws If <code>opts.pendingEventOrdering</code> was not 'detached'
  */
 Room.prototype.hasPendingEvent = function(eventId) {
     if (this._opts.pendingEventOrdering !== "detached") {
-        throw new Error(
-            "Cannot call hasPendingEvent with pendingEventOrdering == " +
-                this._opts.pendingEventOrdering);
+        return false;
     }
 
     return this._pendingEventList.some(event => event.getId() === eventId);
 };
 
+/**
+ * Get a specific event from the pending event list, if configured, null otherwise.
+ *
+ * @param {string} eventId The event ID to check for.
+ * @return {MatrixEvent}
+ */
+Room.prototype.getPendingEvent = function(eventId) {
+    if (this._opts.pendingEventOrdering !== "detached") {
+        return null;
+    }
+
+    return this._pendingEventList.find(event => event.getId() === eventId);
+};
+
 /**
  * Get the live unfiltered timeline for this room.
  *
@@ -814,10 +830,6 @@ Room.prototype.getAvatarUrl = function(baseUrl, width, height, resizeMethod,
         return getHttpUriForMxc(
             baseUrl, mainUrl, width, height, resizeMethod,
         );
-    } else if (allowDefault) {
-        return getIdenticonUri(
-            baseUrl, this.roomId, width, height,
-        );
     }
 
     return null;
@@ -1793,8 +1805,9 @@ Room.prototype.addAccountData = function(events) {
         if (event.getType() === "m.tag") {
             this.addTags(event);
         }
+        const lastEvent = this.accountData[event.getType()];
         this.accountData[event.getType()] = event;
-        this.emit("Room.accountData", event, this);
+        this.emit("Room.accountData", event, this, lastEvent);
     }
 };
 
@@ -1899,14 +1912,14 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
     // let's try to figure out who was here before
     let leftNames = otherNames;
     // if we didn't have heroes, try finding them in the room state
-    if(!leftNames.length) {
+    if (!leftNames.length) {
         leftNames = room.currentState.getMembers().filter((m) => {
             return m.userId !== userId &&
                 m.membership !== "invite" &&
                 m.membership !== "join";
         }).map((m) => m.name);
     }
-    if(leftNames.length) {
+    if (leftNames.length) {
         return `Empty room (was ${memberNamesToRoomName(leftNames)})`;
     } else {
         return "Empty room";
@@ -1991,8 +2004,10 @@ function memberNamesToRoomName(names, count = (names.length + 1)) {
  * @event module:client~MatrixClient#"Room.accountData"
  * @param {event} event The account_data event
  * @param {Room} room The room whose account_data was updated.
+ * @param {MatrixEvent} prevEvent The event being replaced by
+ * the new account data, if known.
  * @example
- * matrixClient.on("Room.accountData", function(event, room){
+ * matrixClient.on("Room.accountData", function(event, room, oldEvent){
  *   if (event.getType() === "m.room.colorscheme") {
  *       applyColorScheme(event.getContents());
  *   }

src/models/user.js

@@ -129,7 +129,11 @@ User.prototype.setPresenceEvent = function(event) {
  */
 User.prototype.setDisplayName = function(name) {
     const oldName = this.displayName;
-    this.displayName = name;
+    if (typeof name === "string") {
+        this.displayName = name;
+    } else {
+        this.displayName = undefined;
+    }
     if (name !== oldName) {
         this._updateModifiedTime();
     }
@@ -142,7 +146,11 @@ User.prototype.setDisplayName = function(name) {
  * @param {string} name The new display name.
  */
 User.prototype.setRawDisplayName = function(name) {
-    this.rawDisplayName = name;
+    if (typeof name === "string") {
+        this.rawDisplayName = name;
+    } else {
+        this.rawDisplayName = undefined;
+    }
 };
 
 

src/pushprocessor.js

@@ -16,6 +16,7 @@ limitations under the License.
 */
 
 import {escapeRegExp, globToRegexp, isNullOrUndefined} from "./utils";
+import {logger} from './logger';
 
 /**
  * @module pushprocessor
@@ -23,9 +24,8 @@ import {escapeRegExp, globToRegexp, isNullOrUndefined} from "./utils";
 
 const RULEKINDS_IN_ORDER = ['override', 'content', 'room', 'sender', 'underride'];
 
-// The default override rules to apply when calculating actions for an event. These
-// defaults apply under no other circumstances to avoid confusing the client with server
-// state. We do this for two reasons:
+// The default override rules to apply to the push rules that arrive from the server.
+// We do this for two reasons:
 //   1. Synapse is unlikely to send us the push rule in an incremental sync - see
 //      https://github.com/matrix-org/synapse/pull/4867#issuecomment-481446072 for
 //      more details.
@@ -85,12 +85,15 @@ export function PushProcessor(client) {
         // $glob: RegExp,
     };
 
-    const matchingRuleFromKindSet = (ev, kindset, device) => {
+    const matchingRuleFromKindSet = (ev, kindset) => {
         for (let ruleKindIndex = 0;
                 ruleKindIndex < RULEKINDS_IN_ORDER.length;
                 ++ruleKindIndex) {
             const kind = RULEKINDS_IN_ORDER[ruleKindIndex];
             const ruleset = kindset[kind];
+            if (!ruleset) {
+                continue;
+            }
 
             for (let ruleIndex = 0; ruleIndex < ruleset.length; ++ruleIndex) {
                 const rule = ruleset[ruleIndex];
@@ -98,7 +101,7 @@ export function PushProcessor(client) {
                     continue;
                 }
 
-                const rawrule = templateRuleToRaw(kind, rule, device);
+                const rawrule = templateRuleToRaw(kind, rule);
                 if (!rawrule) {
                     continue;
                 }
@@ -112,7 +115,7 @@ export function PushProcessor(client) {
         return null;
     };
 
-    const templateRuleToRaw = function(kind, tprule, device) {
+    const templateRuleToRaw = function(kind, tprule) {
         const rawrule = {
             'rule_id': tprule.rule_id,
             'actions': tprule.actions,
@@ -154,19 +157,12 @@ export function PushProcessor(client) {
                 });
                 break;
         }
-        if (device) {
-            rawrule.conditions.push({
-                'kind': 'device',
-                'profile_tag': device,
-            });
-        }
         return rawrule;
     };
 
     const eventFulfillsCondition = function(cond, ev) {
         const condition_functions = {
             "event_match": eventFulfillsEventMatchCondition,
-            "device": eventFulfillsDeviceCondition,
             "contains_display_name": eventFulfillsDisplayNameCondition,
             "room_member_count": eventFulfillsRoomMemberCountCondition,
             "sender_notification_permission": eventFulfillsSenderNotifPermCondition,
@@ -258,10 +254,6 @@ export function PushProcessor(client) {
         return content.body.search(pat) > -1;
     };
 
-    const eventFulfillsDeviceCondition = function(cond, ev) {
-        return false; // XXX: Allow a profile tag to be set for the web client instance
-    };
-
     const eventFulfillsEventMatchCondition = function(cond, ev) {
         if (!cond.key) {
             return false;
@@ -326,23 +318,13 @@ export function PushProcessor(client) {
     };
 
     const matchingRuleForEventWithRulesets = function(ev, rulesets) {
-        if (!rulesets || !rulesets.device) {
+        if (!rulesets) {
             return null;
         }
-        if (ev.getSender() == client.credentials.userId) {
+        if (ev.getSender() === client.credentials.userId) {
             return null;
         }
 
-        const allDevNames = Object.keys(rulesets.device);
-        for (let i = 0; i < allDevNames.length; ++i) {
-            const devname = allDevNames[i];
-            const devrules = rulesets.device[devname];
-
-            const matchingRule = matchingRuleFromKindSet(devrules, devname);
-            if (matchingRule) {
-                return matchingRule;
-            }
-        }
         return matchingRuleFromKindSet(ev, rulesets.global);
     };
 
@@ -364,33 +346,6 @@ export function PushProcessor(client) {
         return actionObj;
     };
 
-    const applyRuleDefaults = function(clientRuleset) {
-        // Deep clone the object before we mutate it
-        const ruleset = JSON.parse(JSON.stringify(clientRuleset));
-
-        if (!clientRuleset['global']) {
-            clientRuleset['global'] = {};
-        }
-        if (!clientRuleset['global']['override']) {
-            clientRuleset['global']['override'] = [];
-        }
-
-        // Apply default overrides
-        const globalOverrides = clientRuleset['global']['override'];
-        for (const override of DEFAULT_OVERRIDE_RULES) {
-            const existingRule = globalOverrides
-                .find((r) => r.rule_id === override.rule_id);
-
-            if (!existingRule) {
-                const ruleId = override.rule_id;
-                console.warn(`Adding default global override for ${ruleId}`);
-                globalOverrides.push(override);
-            }
-        }
-
-        return ruleset;
-    };
-
     this.ruleMatchesEvent = function(rule, ev) {
         let ret = true;
         for (let i = 0; i < rule.conditions.length; ++i) {
@@ -410,8 +365,7 @@ export function PushProcessor(client) {
      * @return {PushAction}
      */
     this.actionsForEvent = function(ev) {
-        const rules = applyRuleDefaults(client.pushRules);
-        return pushActionsForEventAndRulesets(ev, rules);
+        return pushActionsForEventAndRulesets(ev, client.pushRules);
     };
 
     /**
@@ -421,7 +375,7 @@ export function PushProcessor(client) {
      * @return {object} The push rule, or null if no such rule was found
      */
     this.getPushRuleById = function(ruleId) {
-        for (const scope of ['device', 'global']) {
+        for (const scope of ['global']) {
             if (client.pushRules[scope] === undefined) continue;
 
             for (const kind of RULEKINDS_IN_ORDER) {
@@ -476,18 +430,25 @@ PushProcessor.rewriteDefaultRules = function(incomingRules) {
     if (!newRules.global) newRules.global = {};
     if (!newRules.global.override) newRules.global.override = [];
 
-    // Fix default override rules
-    newRules.global.override = newRules.global.override.map(r => {
-        const defaultRule = DEFAULT_OVERRIDE_RULES.find(d => d.rule_id === r.rule_id);
-        if (!defaultRule) return r;
+    // Merge the client-level defaults with the ones from the server
+    const globalOverrides = newRules.global.override;
+    for (const override of DEFAULT_OVERRIDE_RULES) {
+        const existingRule = globalOverrides
+            .find((r) => r.rule_id === override.rule_id);
 
-        // Copy over the actions, default, and conditions. Don't touch the user's
-        // preference.
-        r.default = defaultRule.default;
-        r.conditions = defaultRule.conditions;
-        r.actions = defaultRule.actions;
-        return r;
-    });
+        if (existingRule) {
+            // Copy over the actions, default, and conditions. Don't touch the user's
+            // preference.
+            existingRule.default = override.default;
+            existingRule.conditions = override.conditions;
+            existingRule.actions = override.actions;
+        } else {
+            // Add the rule
+            const ruleId = override.rule_id;
+            logger.warn(`Adding default global override for ${ruleId}`);
+            globalOverrides.push(override);
+        }
+    }
 
     return newRules;
 };

src/realtime-callbacks.js

@@ -185,8 +185,8 @@ function _runCallbacks() {
  */
 function binarySearch(array, func) {
     // min is inclusive, max exclusive.
-    let min = 0,
-        max = array.length;
+    let min = 0;
+    let max = array.length;
 
     while (min < max) {
         const mid = (min + max) >> 1;

src/store/indexeddb-local-backend.js

@@ -226,7 +226,7 @@ LocalIndexedDBStoreBackend.prototype = {
                 account_data: {
                     events: accountData,
                 },
-            });
+            }, true);
         });
     },
 
@@ -416,7 +416,7 @@ LocalIndexedDBStoreBackend.prototype = {
     },
 
     syncToDatabase: function(userTuples) {
-        const syncData = this._syncAccumulator.getJSON();
+        const syncData = this._syncAccumulator.getJSON(true);
 
         return Promise.all([
             this._persistUserPresenceEvents(userTuples),

src/sync-accumulator.js

@@ -87,8 +87,8 @@ export class SyncAccumulator {
         };
     }
 
-    accumulate(syncResponse) {
-        this._accumulateRooms(syncResponse);
+    accumulate(syncResponse, fromDatabase) {
+        this._accumulateRooms(syncResponse, fromDatabase);
         this._accumulateGroups(syncResponse);
         this._accumulateAccountData(syncResponse);
         this.nextBatch = syncResponse.next_batch;
@@ -107,35 +107,36 @@ export class SyncAccumulator {
     /**
      * Accumulate incremental /sync room data.
      * @param {Object} syncResponse the complete /sync JSON
+     * @param {boolean} fromDatabase True if the sync response is one saved to the database
      */
-    _accumulateRooms(syncResponse) {
+    _accumulateRooms(syncResponse, fromDatabase) {
         if (!syncResponse.rooms) {
             return;
         }
         if (syncResponse.rooms.invite) {
             Object.keys(syncResponse.rooms.invite).forEach((roomId) => {
                 this._accumulateRoom(
-                    roomId, "invite", syncResponse.rooms.invite[roomId],
+                    roomId, "invite", syncResponse.rooms.invite[roomId], fromDatabase,
                 );
             });
         }
         if (syncResponse.rooms.join) {
             Object.keys(syncResponse.rooms.join).forEach((roomId) => {
                 this._accumulateRoom(
-                    roomId, "join", syncResponse.rooms.join[roomId],
+                    roomId, "join", syncResponse.rooms.join[roomId], fromDatabase,
                 );
             });
         }
         if (syncResponse.rooms.leave) {
             Object.keys(syncResponse.rooms.leave).forEach((roomId) => {
                 this._accumulateRoom(
-                    roomId, "leave", syncResponse.rooms.leave[roomId],
+                    roomId, "leave", syncResponse.rooms.leave[roomId], fromDatabase,
                 );
             });
         }
     }
 
-    _accumulateRoom(roomId, category, data) {
+    _accumulateRoom(roomId, category, data, fromDatabase) {
         // Valid /sync state transitions
         //       +--------+ <======+            1: Accept an invite
         //   +== | INVITE |        | (5)        2: Leave a room
@@ -159,7 +160,7 @@ export class SyncAccumulator {
                     delete this.inviteRooms[roomId];
                 }
                 // (3)
-                this._accumulateJoinState(roomId, data);
+                this._accumulateJoinState(roomId, data, fromDatabase);
                 break;
             case "leave":
                 if (this.inviteRooms[roomId]) { // (4)
@@ -203,7 +204,7 @@ export class SyncAccumulator {
     }
 
     // Accumulate timeline and state events in a room.
-    _accumulateJoinState(roomId, data) {
+    _accumulateJoinState(roomId, data, fromDatabase) {
         // We expect this function to be called a lot (every /sync) so we want
         // this to be fast. /sync stores events in an array but we often want
         // to clobber based on type/state_key. Rather than convert arrays to
@@ -337,8 +338,20 @@ export class SyncAccumulator {
                 setState(currentData._currentState, e);
                 // append the event to the timeline. The back-pagination token
                 // corresponds to the first event in the timeline
+                let transformedEvent;
+                if (!fromDatabase) {
+                    transformedEvent = Object.assign({}, e);
+                    if (transformedEvent.unsigned !== undefined) {
+                        transformedEvent.unsigned = Object.assign({}, transformedEvent.unsigned);
+                    }
+                    const age = e.unsigned ? e.unsigned.age : e.age;
+                    if (age !== undefined) transformedEvent._localTs = Date.now() - age;
+                } else {
+                    transformedEvent = e;
+                }
+
                 currentData._timeline.push({
-                    event: e,
+                    event: transformedEvent,
                     token: index === 0 ? data.timeline.prev_batch : null,
                 });
             });
@@ -405,6 +418,7 @@ export class SyncAccumulator {
      * represents all room data that should be stored. This should be paired
      * with the sync token which represents the most recent /sync response
      * provided to accumulate().
+     * @param {boolean} forDatabase True to generate a sync to be saved to storage
      * @return {Object} An object with a "nextBatch", "roomsData" and "accountData"
      * keys.
      * The "nextBatch" key is a string which represents at what point in the
@@ -414,7 +428,7 @@ export class SyncAccumulator {
      * /sync response from the 'rooms' key onwards. The "accountData" key is
      * a list of raw events which represent global account data.
      */
-    getJSON() {
+    getJSON(forDatabase) {
         const data = {
             join: {},
             invite: {},
@@ -486,7 +500,28 @@ export class SyncAccumulator {
                     }
                     roomJson.timeline.prev_batch = msgData.token;
                 }
-                roomJson.timeline.events.push(msgData.event);
+
+                let transformedEvent;
+                if (!forDatabase && msgData.event._localTs) {
+                    // This means we have to copy each event so we can fix it up to
+                    // set a correct 'age' parameter whilst keeping the local timestamp
+                    // on our stored event. If this turns out to be a bottleneck, it could
+                    // be optimised either by doing this in the main process after the data
+                    // has been structured-cloned to go between the worker & main process,
+                    // or special-casing data from saved syncs to read the local timstamp
+                    // directly rather than turning it into age to then immediately be
+                    // transformed back again into a local timestamp.
+                    transformedEvent = Object.assign({}, msgData.event);
+                    if (transformedEvent.unsigned !== undefined) {
+                        transformedEvent.unsigned = Object.assign({}, transformedEvent.unsigned);
+                    }
+                    delete transformedEvent._localTs;
+                    transformedEvent.unsigned = transformedEvent.unsigned || {};
+                    transformedEvent.unsigned.age = Date.now() - msgData.event._localTs;
+                } else {
+                    transformedEvent = msgData.event;
+                }
+                roomJson.timeline.events.push(transformedEvent);
             });
 
             // Add state data: roll back current state to the start of timeline,

src/sync.js

@@ -475,9 +475,9 @@ SyncApi.prototype.sync = function() {
 
     this._running = true;
 
-    if (global.document) {
+    if (global.window) {
         this._onOnlineBound = this._onOnline.bind(this);
-        global.document.addEventListener("online", this._onOnlineBound, false);
+        global.window.addEventListener("online", this._onOnlineBound, false);
     }
 
     let savedSyncPromise = Promise.resolve();
@@ -643,8 +643,8 @@ SyncApi.prototype.sync = function() {
  */
 SyncApi.prototype.stop = function() {
     debuglog("SyncApi.stop");
-    if (global.document) {
-        global.document.removeEventListener("online", this._onOnlineBound, false);
+    if (global.window) {
+        global.window.removeEventListener("online", this._onOnlineBound, false);
         this._onOnlineBound = undefined;
     }
     this._running = false;
@@ -701,7 +701,7 @@ SyncApi.prototype._syncFromCache = async function(savedSync) {
 
     try {
         await this._processSyncResponse(syncEventData, data);
-    } catch(e) {
+    } catch (e) {
         logger.error("Error processing cached sync", e.stack || e);
     }
 
@@ -774,7 +774,7 @@ SyncApi.prototype._sync = async function(syncOptions) {
 
     try {
         await this._processSyncResponse(syncEventData, data);
-    } catch(e) {
+    } catch (e) {
         // log the exception with stack if we have it, else fall back
         // to the plain description
         logger.error("Caught /sync error", e.stack || e);
@@ -894,7 +894,7 @@ SyncApi.prototype._onSyncError = function(err, syncOptions) {
     logger.error("/sync error %s", err);
     logger.error(err);
 
-    if(this._shouldAbortSync(err)) {
+    if (this._shouldAbortSync(err)) {
         return;
     }
 
@@ -1023,18 +1023,23 @@ SyncApi.prototype._processSyncResponse = async function(
     // handle non-room account_data
     if (data.account_data && utils.isArray(data.account_data.events)) {
         const events = data.account_data.events.map(client.getEventMapper());
+        const prevEventsMap = events.reduce((m, c) => {
+            m[c.getId()] = client.store.getAccountData(c.getType());
+            return m;
+        }, {});
         client.store.storeAccountDataEvents(events);
         events.forEach(
             function(accountDataEvent) {
                 // Honour push rules that come down the sync stream but also
                 // honour push rules that were previously cached. Base rules
-                // will be updated when we recieve push rules via getPushRules
+                // will be updated when we receive push rules via getPushRules
                 // (see SyncApi.prototype.sync) before syncing over the network.
                 if (accountDataEvent.getType() === 'm.push_rules') {
                     const rules = accountDataEvent.getContent();
                     client.pushRules = PushProcessor.rewriteDefaultRules(rules);
                 }
-                client.emit("accountData", accountDataEvent);
+                const prevEvent = prevEventsMap[accountDataEvent.getId()];
+                client.emit("accountData", accountDataEvent, prevEvent);
                 return accountDataEvent;
             },
         );
@@ -1356,6 +1361,16 @@ SyncApi.prototype._processSyncResponse = async function(
         const currentCount = data.device_one_time_keys_count.signed_curve25519 || 0;
         this.opts.crypto.updateOneTimeKeyCount(currentCount);
     }
+    if (this.opts.crypto && data["org.matrix.msc2732.device_unused_fallback_key_types"]) {
+        // The presence of device_unused_fallback_key_types indicates that the
+        // server supports fallback keys. If there's no unused
+        // signed_curve25519 fallback key we need a new one.
+        const unusedFallbackKeys = data["org.matrix.msc2732.device_unused_fallback_key_types"];
+        this.opts.crypto.setNeedsNewFallback(
+            unusedFallbackKeys instanceof Array &&
+            !unusedFallbackKeys.includes("signed_curve25519"),
+        );
+    }
 };
 
 /**

src/timeline-window.js

@@ -396,7 +396,8 @@ TimelineWindow.prototype.getEvents = function() {
         // (Note that both this._start.index and this._end.index are relative
         // to their respective timelines' BaseIndex).
         //
-        let startIndex = 0, endIndex = events.length;
+        let startIndex = 0;
+        let endIndex = events.length;
         if (timeline === this._start.timeline) {
             startIndex = this._start.index + timeline.getBaseIndex();
         }

src/utils.ts

@@ -21,7 +21,6 @@ limitations under the License.
  */
 
 import unhomoglyph from 'unhomoglyph';
-import {ConnectionError} from "./http-api";
 
 /**
  * Encode a dictionary of query parameters.
@@ -50,7 +49,7 @@ export function encodeParams(params: Record<string, string>): string {
  * @return {string} The result of replacing all template variables e.g. '/foo/baz'.
  */
 export function encodeUri(pathTemplate: string,
-                          variables: Record<string, string>): string {
+    variables: Record<string, string>): string {
     for (const key in variables) {
         if (!variables.hasOwnProperty(key)) {
             continue;
@@ -86,7 +85,7 @@ export function map<T, S>(array: T[], fn: (t: T) => S): S[] {
  * @return {Array} A new array with the results of the function.
  */
 export function filter<T>(array: T[],
-                          fn: (t: T, i?: number, a?: T[]) => boolean): T[] {
+    fn: (t: T, i?: number, a?: T[]) => boolean): T[] {
     const results: T[] = [];
     for (let i = 0; i < array.length; i++) {
         if (fn(array[i], i, array)) {
@@ -152,10 +151,10 @@ export function forEach<T>(array: T[], fn: (t: T, i: number) => void) {
  * the given function.
  */
 export function findElement<T>(
-        array: T[],
-        fn: (t: T, i?: number, a?: T[]) => boolean,
-        reverse?: boolean
-    ) {
+    array: T[],
+    fn: (t: T, i?: number, a?: T[]) => boolean,
+    reverse?: boolean,
+) {
     let i;
     if (reverse) {
         for (i = array.length - 1; i >= 0; i--) {
@@ -183,10 +182,10 @@ export function findElement<T>(
  * @return {boolean} True if an element was removed.
  */
 export function removeElement<T>(
-        array: T[],
-        fn: (t: T, i?: number, a?: T[]) => boolean,
-        reverse?: boolean
-    ) {
+    array: T[],
+    fn: (t: T, i?: number, a?: T[]) => boolean,
+    reverse?: boolean,
+) {
     let i;
     let removed;
     if (reverse) {
@@ -266,7 +265,7 @@ export function checkObjectHasNoAdditionalKeys(obj: object, allowedKeys: string[
  * @param {Object} obj The object to deep copy.
  * @return {Object} A copy of the object without any references to the original.
  */
-export function deepCopy(obj: object): object {
+export function deepCopy<T>(obj: T): T {
     return JSON.parse(JSON.stringify(obj));
 }
 
@@ -294,7 +293,7 @@ export function deepCompare(x: any, y: any): boolean {
 
     // special-case NaN (since NaN !== NaN)
     if (typeof x === 'number' && isNaN(x) && isNaN(y)) {
-         return true;
+        return true;
     }
 
     // special-case null (since typeof null == 'object', but null.constructor
@@ -369,10 +368,11 @@ export function deepCompare(x: any, y: any): boolean {
  *
  * @return {Object} target
  */
-export function extend() {
-    const target = arguments[0] || {};
-    for (let i = 1; i < arguments.length; i++) {
-        const source = arguments[i];
+export function extend(...restParams) {
+    const target = restParams[0] || {};
+    for (let i = 1; i < restParams.length; i++) {
+        const source = restParams[i];
+        if (!source) continue;
         for (const propName in source) { // eslint-disable-line guard-for-in
             target[propName] = source[propName];
         }
@@ -389,36 +389,36 @@ export function runPolyfills() {
     // SOURCE:
     // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/filter
     if (!Array.prototype.filter) {
-      Array.prototype.filter = function(fun: Function/*, thisArg*/) {
-        if (this === void 0 || this === null) {
-          throw new TypeError();
-        }
-
-        const t = Object(this);
-        const len = t.length >>> 0;
-        if (typeof fun !== 'function') {
-          throw new TypeError();
-        }
-
-        const res = [];
-        const thisArg = arguments.length >= 2 ? arguments[1] : void 0;
-        for (let i = 0; i < len; i++) {
-          if (i in t) {
-            const val = t[i];
-
-            // NOTE: Technically this should Object.defineProperty at
-            //       the next index, as push can be affected by
-            //       properties on Object.prototype and Array.prototype.
-            //       But that method's new, and collisions should be
-            //       rare, so use the more-compatible alternative.
-            if (fun.call(thisArg, val, i, t)) {
-              res.push(val);
+        // eslint-disable-next-line no-extend-native
+        Array.prototype.filter = function(fun: Function/*, thisArg*/, ...restProps) {
+            if (this === void 0 || this === null) {
+                throw new TypeError();
             }
-          }
-        }
 
-        return res;
-      };
+            const t = Object(this);
+            const len = t.length >>> 0;
+            if (typeof fun !== 'function') {
+                throw new TypeError();
+            }
+
+            const res = [];
+            const thisArg = restProps ? restProps[0] : void 0;
+            for (let i = 0; i < len; i++) {
+                if (i in t) {
+                    const val = t[i];
+
+                    // NOTE: Technically this should Object.defineProperty at
+                    //       the next index, as push can be affected by
+                    //       properties on Object.prototype and Array.prototype.
+                    //       But that method's new, and collisions should be
+                    //       rare, so use the more-compatible alternative.
+                    if (fun.call(thisArg, val, i, t)) {
+                        res.push(val);
+                    }
+                }
+            }
+            return res;
+        };
     }
 
     //                Array.prototype.map
@@ -428,151 +428,156 @@ export function runPolyfills() {
     // Production steps of ECMA-262, Edition 5, 15.4.4.19
     // Reference: http://es5.github.io/#x15.4.4.19
     if (!Array.prototype.map) {
-      Array.prototype.map = function(callback, thisArg) {
-        let T, k;
+        // eslint-disable-next-line no-extend-native
+        Array.prototype.map = function(callback, thisArg) {
+            let T;
+            let k;
 
-        if (this === null || this === undefined) {
-          throw new TypeError(' this is null or not defined');
-        }
+            if (this === null || this === undefined) {
+                throw new TypeError(' this is null or not defined');
+            }
 
-        // 1. Let O be the result of calling ToObject passing the |this|
-        //    value as the argument.
-        const O = Object(this);
+            // 1. Let O be the result of calling ToObject passing the |this|
+            //    value as the argument.
+            const O = Object(this);
 
-        // 2. Let lenValue be the result of calling the Get internal
-        //    method of O with the argument "length".
-        // 3. Let len be ToUint32(lenValue).
-        const len = O.length >>> 0;
+            // 2. Let lenValue be the result of calling the Get internal
+            //    method of O with the argument "length".
+            // 3. Let len be ToUint32(lenValue).
+            const len = O.length >>> 0;
 
-        // 4. If IsCallable(callback) is false, throw a TypeError exception.
-        // See: http://es5.github.com/#x9.11
-        if (typeof callback !== 'function') {
-          throw new TypeError(callback + ' is not a function');
-        }
+            // 4. If IsCallable(callback) is false, throw a TypeError exception.
+            // See: http://es5.github.com/#x9.11
+            if (typeof callback !== 'function') {
+                throw new TypeError(callback + ' is not a function');
+            }
 
-        // 5. If thisArg was supplied, let T be thisArg; else let T be undefined.
-        if (arguments.length > 1) {
-          T = thisArg;
-        }
+            // 5. If thisArg was supplied, let T be thisArg; else let T be undefined.
+            if (arguments.length > 1) {
+                T = thisArg;
+            }
 
-        // 6. Let A be a new array created as if by the expression new Array(len)
-        //    where Array is the standard built-in constructor with that name and
-        //    len is the value of len.
-        const A = new Array(len);
+            // 6. Let A be a new array created as if by the expression new Array(len)
+            //    where Array is the standard built-in constructor with that name and
+            //    len is the value of len.
+            const A = new Array(len);
 
-        // 7. Let k be 0
-        k = 0;
+            // 7. Let k be 0
+            k = 0;
 
-        // 8. Repeat, while k < len
-        while (k < len) {
-          let kValue, mappedValue;
+            // 8. Repeat, while k < len
+            while (k < len) {
+                let kValue;
+                let mappedValue;
 
-          // a. Let Pk be ToString(k).
-          //   This is implicit for LHS operands of the in operator
-          // b. Let kPresent be the result of calling the HasProperty internal
-          //    method of O with argument Pk.
-          //   This step can be combined with c
-          // c. If kPresent is true, then
-          if (k in O) {
-            // i. Let kValue be the result of calling the Get internal
-            //    method of O with argument Pk.
-            kValue = O[k];
+                // a. Let Pk be ToString(k).
+                //   This is implicit for LHS operands of the in operator
+                // b. Let kPresent be the result of calling the HasProperty internal
+                //    method of O with argument Pk.
+                //   This step can be combined with c
+                // c. If kPresent is true, then
+                if (k in O) {
+                    // i. Let kValue be the result of calling the Get internal
+                    //    method of O with argument Pk.
+                    kValue = O[k];
 
-            // ii. Let mappedValue be the result of calling the Call internal
-            //     method of callback with T as the this value and argument
-            //     list containing kValue, k, and O.
-            mappedValue = callback.call(T, kValue, k, O);
+                    // ii. Let mappedValue be the result of calling the Call internal
+                    //     method of callback with T as the this value and argument
+                    //     list containing kValue, k, and O.
+                    mappedValue = callback.call(T, kValue, k, O);
 
-            // iii. Call the DefineOwnProperty internal method of A with arguments
-            // Pk, Property Descriptor
-            // { Value: mappedValue,
-            //   Writable: true,
-            //   Enumerable: true,
-            //   Configurable: true },
-            // and false.
+                    // iii. Call the DefineOwnProperty internal method of A with arguments
+                    // Pk, Property Descriptor
+                    // { Value: mappedValue,
+                    //   Writable: true,
+                    //   Enumerable: true,
+                    //   Configurable: true },
+                    // and false.
 
-            // In browsers that support Object.defineProperty, use the following:
-            // Object.defineProperty(A, k, {
-            //   value: mappedValue,
-            //   writable: true,
-            //   enumerable: true,
-            //   configurable: true
-            // });
+                    // In browsers that support Object.defineProperty, use the following:
+                    // Object.defineProperty(A, k, {
+                    //   value: mappedValue,
+                    //   writable: true,
+                    //   enumerable: true,
+                    //   configurable: true
+                    // });
 
-            // For best browser support, use the following:
-            A[k] = mappedValue;
-          }
-          // d. Increase k by 1.
-          k++;
-        }
+                    // For best browser support, use the following:
+                    A[k] = mappedValue;
+                }
+                // d. Increase k by 1.
+                k++;
+            }
 
-        // 9. return A
-        return A;
-      };
+            // 9. return A
+            return A;
+        };
     }
 
     //                Array.prototype.forEach
     // ========================================================
     // SOURCE:
-  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach
+    // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach
     // Production steps of ECMA-262, Edition 5, 15.4.4.18
     // Reference: http://es5.github.io/#x15.4.4.18
     if (!Array.prototype.forEach) {
-      Array.prototype.forEach = function(callback, thisArg) {
-        let T, k;
+        // eslint-disable-next-line no-extend-native
+        Array.prototype.forEach = function(callback, thisArg) {
+            let T;
+            let k;
 
-        if (this === null || this === undefined) {
-          throw new TypeError(' this is null or not defined');
-        }
+            if (this === null || this === undefined) {
+                throw new TypeError(' this is null or not defined');
+            }
 
-        // 1. Let O be the result of calling ToObject passing the |this| value as the
-        // argument.
-        const O = Object(this);
+            // 1. Let O be the result of calling ToObject passing the |this| value as the
+            // argument.
+            const O = Object(this);
 
-        // 2. Let lenValue be the result of calling the Get internal method of O with the
-        // argument "length".
-        // 3. Let len be ToUint32(lenValue).
-        const len = O.length >>> 0;
+            // 2. Let lenValue be the result of calling the Get internal method of O with the
+            // argument "length".
+            // 3. Let len be ToUint32(lenValue).
+            const len = O.length >>> 0;
 
-        // 4. If IsCallable(callback) is false, throw a TypeError exception.
-        // See: http://es5.github.com/#x9.11
-        if (typeof callback !== "function") {
-          throw new TypeError(callback + ' is not a function');
-        }
+            // 4. If IsCallable(callback) is false, throw a TypeError exception.
+            // See: http://es5.github.com/#x9.11
+            if (typeof callback !== "function") {
+                throw new TypeError(callback + ' is not a function');
+            }
 
-        // 5. If thisArg was supplied, let T be thisArg; else let T be undefined.
-        if (arguments.length > 1) {
-          T = thisArg;
-        }
+            // 5. If thisArg was supplied, let T be thisArg; else let T be undefined.
+            if (arguments.length > 1) {
+                T = thisArg;
+            }
 
-        // 6. Let k be 0
-        k = 0;
+            // 6. Let k be 0
+            k = 0;
 
-        // 7. Repeat, while k < len
-        while (k < len) {
-          let kValue;
+            // 7. Repeat, while k < len
+            while (k < len) {
+                let kValue;
 
-          // a. Let Pk be ToString(k).
-          //   This is implicit for LHS operands of the in operator
-          // b. Let kPresent be the result of calling the HasProperty internal
-          //    method of O with
-          //    argument Pk.
-          //   This step can be combined with c
-          // c. If kPresent is true, then
-          if (k in O) {
-            // i. Let kValue be the result of calling the Get internal method of O with
-            // argument Pk
-            kValue = O[k];
+                // a. Let Pk be ToString(k).
+                //   This is implicit for LHS operands of the in operator
+                // b. Let kPresent be the result of calling the HasProperty internal
+                //    method of O with
+                //    argument Pk.
+                //   This step can be combined with c
+                // c. If kPresent is true, then
+                if (k in O) {
+                    // i. Let kValue be the result of calling the Get internal method of O with
+                    // argument Pk
+                    kValue = O[k];
 
-            // ii. Call the Call internal method of callback with T as the this value and
-            // argument list containing kValue, k, and O.
-            callback.call(T, kValue, k, O);
-          }
-          // d. Increase k by 1.
-          k++;
-        }
+                    // ii. Call the Call internal method of callback with T as the this value and
+                    // argument list containing kValue, k, and O.
+                    callback.call(T, kValue, k, O);
+                }
+                // d. Increase k by 1.
+                k++;
+            }
         // 8. return undefined
-      };
+        };
     }
 }
 
@@ -656,7 +661,10 @@ export function isNumber(value: any): boolean {
  * @return {string} a string with the hidden characters removed
  */
 export function removeHiddenChars(str: string): string {
-    return unhomoglyph(str.normalize('NFD').replace(removeHiddenCharsRegex, ''));
+    if (typeof str === "string") {
+        return unhomoglyph(str.normalize('NFD').replace(removeHiddenCharsRegex, ''));
+    }
+    return "";
 }
 
 // Regex matching bunch of unicode control characters and otherwise misleading/invisible characters.
@@ -666,6 +674,7 @@ export function removeHiddenChars(str: string): string {
 // LTR/RTL and other directional formatting marks U+202A - U+202F
 // Combining characters U+0300 - U+036F
 // Zero width no-break space (BOM) U+FEFF
+// eslint-disable-next-line no-misleading-character-class
 const removeHiddenCharsRegex = /[\u2000-\u200F\u202A-\u202F\u0300-\u036f\uFEFF\s]/g;
 
 export function escapeRegExp(string: string): string {
@@ -724,9 +733,9 @@ export function defer() {
 }
 
 export async function promiseMapSeries<T>(
-        promises: Promise<T>[],
-        fn: (t: T) => void
-    ): Promise<void> {
+    promises: Promise<T>[],
+    fn: (t: T) => void,
+): Promise<void> {
     for (const o of await promises) {
         await fn(await o);
     }

src/webrtc/callEventHandler.ts

@@ -0,0 +1,274 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import MatrixEvent from '../models/event';
+import {logger} from '../logger';
+import { createNewMatrixCall, MatrixCall, CallErrorCode, CallState, CallDirection } from './call';
+import { EventType } from '../@types/event';
+import { MatrixClient } from '../client';
+
+// Don't ring unless we'd be ringing for at least 3 seconds: the user needs some
+// time to press the 'accept' button
+const RING_GRACE_PERIOD = 3000;
+
+export class CallEventHandler {
+    client: MatrixClient;
+    calls: Map<string, MatrixCall>;
+    callEventBuffer: MatrixEvent[];
+    candidateEventsByCall: Map<string, Array<MatrixEvent>>;
+
+    constructor(client: MatrixClient) {
+        this.client = client;
+        this.calls = new Map<string, MatrixCall>();
+        // The sync code always emits one event at a time, so it will patiently
+        // wait for us to finish processing a call invite before delivering the
+        // next event, even if that next event is a hangup. We therefore accumulate
+        // all our call events and then process them on the 'sync' event, ie.
+        // each time a sync has completed. This way, we can avoid emitting incoming
+        // call events if we get both the invite and answer/hangup in the same sync.
+        // This happens quite often, eg. replaying sync from storage, catchup sync
+        // after loading and after we've been offline for a bit.
+        this.callEventBuffer = [];
+        this.candidateEventsByCall = new Map<string, Array<MatrixEvent>>();
+        this.client.on("sync", this.evaluateEventBuffer);
+        this.client.on("event", this.onEvent);
+    }
+
+    public stop() {
+        this.client.removeListener("sync", this.evaluateEventBuffer);
+        this.client.removeListener("event", this.onEvent);
+    }
+
+    private evaluateEventBuffer = () => {
+        if (this.client.getSyncState() === "SYNCING") {
+            // don't process any events until they are all decrypted
+            if (this.callEventBuffer.some((e) => e.isBeingDecrypted())) return;
+
+            const ignoreCallIds = new Set<String>();
+            // inspect the buffer and mark all calls which have been answered
+            // or hung up before passing them to the call event handler.
+            for (const ev of this.callEventBuffer) {
+                if (ev.getType() === EventType.CallAnswer ||
+                        ev.getType() === EventType.CallHangup) {
+                    ignoreCallIds.add(ev.getContent().call_id);
+                }
+            }
+            // now loop through the buffer chronologically and inject them
+            for (const e of this.callEventBuffer) {
+                if (
+                    e.getType() === EventType.CallInvite &&
+                    ignoreCallIds.has(e.getContent().call_id)
+                ) {
+                    // This call has previously been answered or hung up: ignore it
+                    continue;
+                }
+                try {
+                    this.handleCallEvent(e);
+                } catch (e) {
+                    logger.error("Caught exception handling call event", e);
+                }
+            }
+            this.callEventBuffer = [];
+        }
+    }
+
+    private onEvent = (event: MatrixEvent) => {
+        // any call events or ones that might be once they're decrypted
+        if (event.getType().indexOf("m.call.") === 0 || event.isBeingDecrypted()) {
+            // queue up for processing once all events from this sync have been
+            // processed (see above).
+            this.callEventBuffer.push(event);
+        }
+
+        if (event.isBeingDecrypted() || event.isDecryptionFailure()) {
+            // add an event listener for once the event is decrypted.
+            event.once("Event.decrypted", () => {
+                if (event.getType().indexOf("m.call.") === -1) return;
+
+                if (this.callEventBuffer.includes(event)) {
+                    // we were waiting for that event to decrypt, so recheck the buffer
+                    this.evaluateEventBuffer();
+                } else {
+                    // This one wasn't buffered so just run the event handler for it
+                    // straight away
+                    try {
+                        this.handleCallEvent(event);
+                    } catch (e) {
+                        logger.error("Caught exception handling call event", e);
+                    }
+                }
+            });
+        }
+    }
+
+    private handleCallEvent(event: MatrixEvent) {
+        const content = event.getContent();
+        let call = content.call_id ? this.calls.get(content.call_id) : undefined;
+        //console.info("RECV %s content=%s", event.getType(), JSON.stringify(content));
+
+        if (event.getType() === EventType.CallInvite) {
+            if (event.getSender() === this.client.credentials.userId) {
+                return; // ignore invites you send
+            }
+
+            if (event.getLocalAge() > content.lifetime - RING_GRACE_PERIOD) {
+                return; // expired call
+            }
+
+            if (call && call.state === CallState.Ended) {
+                return; // stale/old invite event
+            }
+            if (call) {
+                logger.log(
+                    `WARN: Already have a MatrixCall with id ${content.call_id} but got an ` +
+                    `invite. Clobbering.`,
+                );
+            }
+
+            call = createNewMatrixCall(this.client, event.getRoomId(), {
+                forceTURN: this.client._forceTURN,
+            });
+            if (!call) {
+                logger.log(
+                    "Incoming call ID " + content.call_id + " but this client " +
+                    "doesn't support WebRTC",
+                );
+                // don't hang up the call: there could be other clients
+                // connected that do support WebRTC and declining the
+                // the call on their behalf would be really annoying.
+                return;
+            }
+
+            call.callId = content.call_id;
+            call.initWithInvite(event);
+            this.calls.set(call.callId, call);
+
+            // if we stashed candidate events for that call ID, play them back now
+            if (this.candidateEventsByCall.get(call.callId)) {
+                for (const ev of this.candidateEventsByCall.get(call.callId)) {
+                    call.onRemoteIceCandidatesReceived(ev);
+                }
+            }
+
+            // Were we trying to call that user (room)?
+            let existingCall;
+            for (const thisCall of this.calls.values()) {
+                const isCalling = [CallState.WaitLocalMedia, CallState.CreateOffer, CallState.InviteSent].includes(
+                    thisCall.state,
+                );
+
+                if (
+                    call.roomId === thisCall.roomId &&
+                    thisCall.direction === CallDirection.Outbound &&
+                    isCalling
+                ) {
+                    existingCall = thisCall;
+                    break;
+                }
+            }
+
+            if (existingCall) {
+                // If we've only got to wait_local_media or create_offer and
+                // we've got an invite, pick the incoming call because we know
+                // we haven't sent our invite yet otherwise, pick whichever
+                // call has the lowest call ID (by string comparison)
+                if (existingCall.state === CallState.WaitLocalMedia ||
+                        existingCall.state === CallState.CreateOffer ||
+                        existingCall.callId > call.callId) {
+                    logger.log(
+                        "Glare detected: answering incoming call " + call.callId +
+                        " and canceling outgoing call " + existingCall.callId,
+                    );
+                    existingCall.replacedBy(call);
+                    call.answer();
+                } else {
+                    logger.log(
+                        "Glare detected: rejecting incoming call " + call.callId +
+                        " and keeping outgoing call " + existingCall.callId,
+                    );
+                    call.hangup(CallErrorCode.Replaced, true);
+                }
+            } else {
+                this.client.emit("Call.incoming", call);
+            }
+        } else if (event.getType() === EventType.CallAnswer) {
+            if (!call) {
+                return;
+            }
+            if (event.getSender() === this.client.credentials.userId) {
+                if (call.state === CallState.Ringing) {
+                    call.onAnsweredElsewhere(content);
+                }
+            } else {
+                call.onAnswerReceived(event);
+            }
+        } else if (event.getType() === EventType.CallCandidates) {
+            if (event.getSender() === this.client.credentials.userId) {
+                return;
+            }
+            if (!call) {
+                // store the candidates; we may get a call eventually.
+                if (!this.candidateEventsByCall.has(content.call_id)) {
+                    this.candidateEventsByCall.set(content.call_id, []);
+                }
+                this.candidateEventsByCall.get(content.call_id).push(event);
+            } else {
+                call.onRemoteIceCandidatesReceived(event);
+            }
+        } else if ([EventType.CallHangup, EventType.CallReject].includes(event.getType())) {
+            // Note that we also observe our own hangups here so we can see
+            // if we've already rejected a call that would otherwise be valid
+            if (!call) {
+                // if not live, store the fact that the call has ended because
+                // we're probably getting events backwards so
+                // the hangup will come before the invite
+                call = createNewMatrixCall(this.client, event.getRoomId());
+                if (call) {
+                    call.callId = content.call_id;
+                    call.initWithHangup(event);
+                    this.calls.set(content.call_id, call);
+                }
+            } else {
+                if (call.state !== CallState.Ended) {
+                    if (event.getType() === EventType.CallHangup) {
+                        call.onHangupReceived(content);
+                    } else {
+                        call.onRejectReceived(content);
+                    }
+                    this.calls.delete(content.call_id);
+                }
+            }
+        } else if (event.getType() === EventType.CallSelectAnswer) {
+            if (!call) return;
+
+            if (event.getContent().party_id === call.ourPartyId) {
+                // Ignore remote echo
+                return;
+            }
+
+            call.onSelectAnswerReceived(event);
+        } else if (event.getType() === EventType.CallNegotiate) {
+            if (!call) return;
+
+            if (event.getContent().party_id === call.ourPartyId) {
+                // Ignore remote echo
+                return;
+            }
+
+            call.onNegotiateReceived(event);
+        }
+    }
+}

src/webrtc/callEventTypes.ts

@@ -0,0 +1,37 @@
+// allow camelcase as these are events type that go onto the wire
+/* eslint-disable camelcase */
+
+interface CallOfferAnswer {
+    type: string;
+    sdp: string;
+}
+
+export interface CallCapabilities {
+    'm.call.transferee': boolean;
+}
+
+export interface MCallAnswer {
+    answer: CallOfferAnswer;
+    capabilities: CallCapabilities;
+}
+
+export interface MCallOfferNegotiate {
+    offer: CallOfferAnswer;
+    description: CallOfferAnswer;
+    lifetime: number;
+    capabilities: CallCapabilities;
+}
+
+export interface MCallReplacesTarget {
+    id: string;
+    display_name: string;
+    avatar_url: string;
+}
+
+export interface MCallReplacesEvent {
+    replacement_id: string;
+    target_user: MCallReplacesTarget;
+    create_call: string;
+    target_room: string;
+}
+/* eslint-enable camelcase */

tsconfig.json

@@ -2,6 +2,7 @@
   "compilerOptions": {
     "experimentalDecorators": true,
     "emitDecoratorMetadata": true,
+    "esModuleInterop": true,
     "module": "commonjs",
     "moduleResolution": "node",
     "target": "es2016",

tslint.json

@@ -1,72 +0,0 @@
-{
-  "rules": {
-    "class-name": false,
-    "comment-format": [
-      true
-    ],
-    "curly": false,
-    "eofline": false,
-    "forin": false,
-    "indent": [
-      true,
-      "spaces"
-    ],
-    "label-position": true,
-    "max-line-length": false,
-    "member-access": false,
-    "member-ordering": [
-      true,
-      "static-after-instance",
-      "variables-before-functions"
-    ],
-    "no-arg": true,
-    "no-bitwise": false,
-    "no-console": false,
-    "no-construct": true,
-    "no-debugger": true,
-    "no-duplicate-variable": true,
-    "no-empty": false,
-    "no-eval": true,
-    "no-inferrable-types": true,
-    "no-shadowed-variable": true,
-    "no-string-literal": false,
-    "no-switch-case-fall-through": true,
-    "no-trailing-whitespace": true,
-    "no-unused-expression": true,
-    "no-use-before-declare": false,
-    "no-var-keyword": true,
-    "object-literal-sort-keys": false,
-    "one-line": [
-      true,
-      "check-open-brace",
-      "check-catch",
-      "check-else",
-      "check-whitespace"
-    ],
-    "quotemark": false,
-    "radix": true,
-    "semicolon": [
-      "always"
-    ],
-    "triple-equals": [],
-    "typedef-whitespace": [
-      true,
-      {
-        "call-signature": "nospace",
-        "index-signature": "nospace",
-        "parameter": "nospace",
-        "property-declaration": "nospace",
-        "variable-declaration": "nospace"
-      }
-    ],
-    "variable-name": false,
-    "whitespace": [
-      true,
-      "check-branch",
-      "check-decl",
-      "check-operator",
-      "check-separator",
-      "check-type"
-    ]
-  }
-}