v0.9.2-cryptowraning.1

Disable crypto if indexeddb version too new

.babelrc

@@ -1,3 +1,13 @@
 {
-  "presets": ["es2015"]
+    "presets": ["es2015"],
+    "plugins": [
+        // this transforms async functions into generator functions, which
+        // are then made to use the regenerator module by babel's
+        // transform-regnerator plugin (which is enabled by es2015).
+        "transform-async-to-bluebird",
+
+        // This makes sure that the regenerator runtime is available to
+        // the transpiled code.
+        "transform-runtime",
+    ],
 }

.eslintrc.js

@@ -9,6 +9,10 @@ module.exports = {
     env: {
         browser: true,
         node: true,
+
+        // babel's transform-runtime converts references to ES6 globals such as
+        // Promise and Map to core-js polyfills, so we can use ES6 globals.
+        es6: true,
     },
     extends: ["eslint:recommended", "google"],
     rules: {

.gitignore

@@ -1,4 +1,6 @@
-.jsdoc
+/.jsdocbuild
+/.jsdoc
+
 node_modules
 .lock-wscript
 build/Release

.travis.yml

@@ -2,5 +2,4 @@ language: node_js
 node_js:
     - node # Latest stable version of nodejs.
 script:
-    - npm run lint
-    - npm run test
+    - ./travis.sh

CHANGELOG.md

@@ -1,3 +1,335 @@
+Changes in [0.9.2-cryptowraning.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.2-cryptowraning.1) (2018-03-26)
+================================================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.2...v0.9.2-cryptowraning.1)
+
+ * Disable crypto if indexeddb version too new
+   [\#631](https://github.com/matrix-org/matrix-js-sdk/pull/631)
+
+Changes in [0.9.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.2) (2017-12-04)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.1...v0.9.2)
+
+
+Changes in [0.9.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.1) (2017-11-17)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.0...v0.9.1)
+
+ * Fix the force TURN option
+   [\#577](https://github.com/matrix-org/matrix-js-sdk/pull/577)
+
+Changes in [0.9.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.0) (2017-11-15)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.0-rc.1...v0.9.0)
+
+
+Changes in [0.9.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.0-rc.1) (2017-11-10)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.5...v0.9.0-rc.1)
+
+ * Modify addRoomToGroup to allow setting isPublic, create alias
+   updateGroupRoomAssociation
+   [\#567](https://github.com/matrix-org/matrix-js-sdk/pull/567)
+ * Expose more functionality of pushprocessor
+   [\#565](https://github.com/matrix-org/matrix-js-sdk/pull/565)
+ * Function for working out notif trigger permission
+   [\#566](https://github.com/matrix-org/matrix-js-sdk/pull/566)
+ * keep track of event ID and timestamp of decrypted messages
+   [\#555](https://github.com/matrix-org/matrix-js-sdk/pull/555)
+ * Fix notifEvent computation
+   [\#564](https://github.com/matrix-org/matrix-js-sdk/pull/564)
+ * Fix power level of sentinel members
+   [\#563](https://github.com/matrix-org/matrix-js-sdk/pull/563)
+ * don't try to decrypt a redacted message (fixes vector-im/riot-web#3744)
+   [\#554](https://github.com/matrix-org/matrix-js-sdk/pull/554)
+ * Support room notifs
+   [\#562](https://github.com/matrix-org/matrix-js-sdk/pull/562)
+ * Fix the glob-to-regex code
+   [\#558](https://github.com/matrix-org/matrix-js-sdk/pull/558)
+
+Changes in [0.8.5](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.5) (2017-10-16)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.5-rc.1...v0.8.5)
+
+ * Make unknown pushrule conditions not match
+   [\#559](https://github.com/matrix-org/matrix-js-sdk/pull/559)
+
+Changes in [0.8.5-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.5-rc.1) (2017-10-13)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.4...v0.8.5-rc.1)
+
+ * Implement wrapper API for removing a room from a group
+   [\#553](https://github.com/matrix-org/matrix-js-sdk/pull/553)
+ * Fix typo which resulted in stuck key download requests
+   [\#552](https://github.com/matrix-org/matrix-js-sdk/pull/552)
+ * Store group when it's created
+   [\#549](https://github.com/matrix-org/matrix-js-sdk/pull/549)
+ * Luke/groups remove rooms users from summary
+   [\#548](https://github.com/matrix-org/matrix-js-sdk/pull/548)
+ * Clean on prepublish
+   [\#546](https://github.com/matrix-org/matrix-js-sdk/pull/546)
+ * Implement wrapper APIs for adding rooms to group summary
+   [\#545](https://github.com/matrix-org/matrix-js-sdk/pull/545)
+
+Changes in [0.8.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.4) (2017-09-21)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.3...v0.8.4)
+
+ * Fix build issue
+
+Changes in [0.8.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.3) (2017-09-20)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.3-rc.1...v0.8.3)
+
+ * No changes
+
+Changes in [0.8.3-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.3-rc.1) (2017-09-19)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.2...v0.8.3-rc.1)
+
+ * consume trailing slash when creating Matrix Client in HS and IS urls
+   [\#526](https://github.com/matrix-org/matrix-js-sdk/pull/526)
+ * Add ignore users API
+   [\#539](https://github.com/matrix-org/matrix-js-sdk/pull/539)
+ * Upgrade to jsdoc 3.5.5
+   [\#540](https://github.com/matrix-org/matrix-js-sdk/pull/540)
+ * Make re-emitting events much more memory efficient
+   [\#538](https://github.com/matrix-org/matrix-js-sdk/pull/538)
+ * Only re-emit events from Event objects if needed
+   [\#536](https://github.com/matrix-org/matrix-js-sdk/pull/536)
+ * Handle 'left' users in the deviceList mananagement
+   [\#535](https://github.com/matrix-org/matrix-js-sdk/pull/535)
+ * Factor out devicelist integration tests to a separate file
+   [\#534](https://github.com/matrix-org/matrix-js-sdk/pull/534)
+ * Refactor sync._sync as an async function
+   [\#533](https://github.com/matrix-org/matrix-js-sdk/pull/533)
+ * Add es6 to eslint environments
+   [\#532](https://github.com/matrix-org/matrix-js-sdk/pull/532)
+
+Changes in [0.8.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.2) (2017-08-24)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.1...v0.8.2)
+
+ * Handle m.call.* events which are decrypted asynchronously
+   [\#530](https://github.com/matrix-org/matrix-js-sdk/pull/530)
+ * Re-emit events from, er, Event objects
+   [\#529](https://github.com/matrix-org/matrix-js-sdk/pull/529)
+
+Changes in [0.8.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.1) (2017-08-23)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.1-rc.1...v0.8.1)
+
+ * [No changes]
+
+Changes in [0.8.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.1-rc.1) (2017-08-22)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.0...v0.8.1-rc.1)
+
+ * Fix error handling in interactive-auth
+   [\#527](https://github.com/matrix-org/matrix-js-sdk/pull/527)
+ * Make lots of OlmDevice asynchronous
+   [\#524](https://github.com/matrix-org/matrix-js-sdk/pull/524)
+ * Make crypto.decryptMessage return decryption results
+   [\#523](https://github.com/matrix-org/matrix-js-sdk/pull/523)
+
+Changes in [0.8.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.0) (2017-08-15)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.13...v0.8.0)
+
+BREAKING CHANGE
+---------------
+
+In order to support a move to a more scalable storage backend, we need to make
+a number of the APIs related end-to-end encryption asynchronous.
+
+This release of the JS-SDK includes the following changes which will affect
+applications which support end-to-end encryption:
+
+1. `MatrixClient` now provides a new (asynchronous) method,
+   `initCrypto`. Applications which support end-to-end encryption must call
+   this method (and wait for it to complete) before calling `startClient`, to
+   give the crypto layer a chance to initialise.
+
+2. The following APIs have been changed to return promises:
+
+   * `MatrixClient.getStoredDevicesForUser`
+   * `MatrixClient.getStoredDevice`
+   * `MatrixClient.setDeviceVerified`
+   * `MatrixClient.setDeviceBlocked`
+   * `MatrixClient.setDeviceKnown`
+   * `MatrixClient.getEventSenderDeviceInfo`
+   * `MatrixClient.isEventSenderVerified`
+   * `MatrixClient.importRoomKeys`
+
+   Applications using the results of any of the above methods will need to be
+   updated to wait for the result of the promise.
+
+
+3. `MatrixClient.listDeviceKeys` has been removed altogether. It's been
+   deprecated for some time. Applications using it should instead be changed to
+   use `MatrixClient.getStoredDevices`, which is similar but returns its results
+   in a slightly different format.
+
+
+ * Make bits of `olmlib` asynchronous
+   [\#521](https://github.com/matrix-org/matrix-js-sdk/pull/521)
+ * Make some of DeviceList asynchronous
+   [\#520](https://github.com/matrix-org/matrix-js-sdk/pull/520)
+ * Make methods in crypto/algorithms async
+   [\#519](https://github.com/matrix-org/matrix-js-sdk/pull/519)
+ * Avoid sending unencrypted messages in e2e room
+   [\#518](https://github.com/matrix-org/matrix-js-sdk/pull/518)
+ * Make tests wait for syncs to happen
+   [\#517](https://github.com/matrix-org/matrix-js-sdk/pull/517)
+ * Make a load of methods in the 'Crypto' module asynchronous
+   [\#510](https://github.com/matrix-org/matrix-js-sdk/pull/510)
+ * Set `rawDisplayName` to `userId` if membership has `displayname=null`
+   [\#515](https://github.com/matrix-org/matrix-js-sdk/pull/515)
+ * Refactor handling of crypto events for async
+   [\#508](https://github.com/matrix-org/matrix-js-sdk/pull/508)
+ * Let event decryption be asynchronous
+   [\#509](https://github.com/matrix-org/matrix-js-sdk/pull/509)
+ * Transform `async` functions to bluebird promises
+   [\#511](https://github.com/matrix-org/matrix-js-sdk/pull/511)
+ * Add more group APIs
+   [\#512](https://github.com/matrix-org/matrix-js-sdk/pull/512)
+ * Retrying test: wait for localEchoUpdated event
+   [\#507](https://github.com/matrix-org/matrix-js-sdk/pull/507)
+ * Fix member events breaking on timeline reset, 2
+   [\#504](https://github.com/matrix-org/matrix-js-sdk/pull/504)
+ * Make bits of the js-sdk api asynchronous
+   [\#503](https://github.com/matrix-org/matrix-js-sdk/pull/503)
+ * Yet more js-sdk test deflakification
+   [\#499](https://github.com/matrix-org/matrix-js-sdk/pull/499)
+ * Fix racy 'matrixclient retrying' test
+   [\#497](https://github.com/matrix-org/matrix-js-sdk/pull/497)
+ * Fix spamming of key-share-requests
+   [\#495](https://github.com/matrix-org/matrix-js-sdk/pull/495)
+ * Add progress handler to `uploadContent`
+   [\#500](https://github.com/matrix-org/matrix-js-sdk/pull/500)
+ * Switch matrix-js-sdk to bluebird
+   [\#490](https://github.com/matrix-org/matrix-js-sdk/pull/490)
+ * Fix some more flakey tests
+   [\#492](https://github.com/matrix-org/matrix-js-sdk/pull/492)
+ * make the npm test script windows-friendly
+   [\#489](https://github.com/matrix-org/matrix-js-sdk/pull/489)
+ * Fix a bunch of races in the tests
+   [\#488](https://github.com/matrix-org/matrix-js-sdk/pull/488)
+ * Fix early return in MatrixClient.setGuestAccess
+   [\#487](https://github.com/matrix-org/matrix-js-sdk/pull/487)
+ * Remove testUtils.failTest
+   [\#486](https://github.com/matrix-org/matrix-js-sdk/pull/486)
+ * Add test:watch script
+   [\#485](https://github.com/matrix-org/matrix-js-sdk/pull/485)
+ * Make it possible to use async/await
+   [\#484](https://github.com/matrix-org/matrix-js-sdk/pull/484)
+ * Remove m.new_device support
+   [\#483](https://github.com/matrix-org/matrix-js-sdk/pull/483)
+ * Use access-token in header
+   [\#478](https://github.com/matrix-org/matrix-js-sdk/pull/478)
+ * Sanity-check response from /thirdparty/protocols
+   [\#482](https://github.com/matrix-org/matrix-js-sdk/pull/482)
+ * Avoid parsing plain-text errors as JSON
+   [\#479](https://github.com/matrix-org/matrix-js-sdk/pull/479)
+ * Use external mock-request
+   [\#481](https://github.com/matrix-org/matrix-js-sdk/pull/481)
+ * Fix some races in the tests
+   [\#480](https://github.com/matrix-org/matrix-js-sdk/pull/480)
+ * Fall back to MemoryCryptoStore if indexeddb fails
+   [\#475](https://github.com/matrix-org/matrix-js-sdk/pull/475)
+ * Fix load failure in firefox when indexedDB is disabled
+   [\#474](https://github.com/matrix-org/matrix-js-sdk/pull/474)
+ * Fix a race in a test
+   [\#471](https://github.com/matrix-org/matrix-js-sdk/pull/471)
+ * Avoid throwing an unhandled error when the indexeddb is deleted
+   [\#470](https://github.com/matrix-org/matrix-js-sdk/pull/470)
+ * fix jsdoc
+   [\#469](https://github.com/matrix-org/matrix-js-sdk/pull/469)
+ * Handle m.forwarded_room_key events
+   [\#468](https://github.com/matrix-org/matrix-js-sdk/pull/468)
+ * Improve error reporting from indexeddbstore.clearDatabase
+   [\#466](https://github.com/matrix-org/matrix-js-sdk/pull/466)
+ * Implement sharing of megolm keys
+   [\#454](https://github.com/matrix-org/matrix-js-sdk/pull/454)
+ * Process received room key requests
+   [\#449](https://github.com/matrix-org/matrix-js-sdk/pull/449)
+ * Send m.room_key_request events when we fail to decrypt an event
+   [\#448](https://github.com/matrix-org/matrix-js-sdk/pull/448)
+
+Changes in [0.7.13](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.13) (2017-06-22)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.12...v0.7.13)
+
+ * Fix failure on Tor browser
+   [\#473](https://github.com/matrix-org/matrix-js-sdk/pull/473)
+ * Fix issues with firefox private browsing
+   [\#472](https://github.com/matrix-org/matrix-js-sdk/pull/472)
+
+Changes in [0.7.12](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.12) (2017-06-19)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.12-rc.1...v0.7.12)
+
+ * No changes
+
+
+Changes in [0.7.12-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.12-rc.1) (2017-06-15)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.11...v0.7.12-rc.1)
+
+ * allow setting iceTransportPolicy to relay through forceTURN option
+   [\#462](https://github.com/matrix-org/matrix-js-sdk/pull/462)
+
+Changes in [0.7.11](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.11) (2017-06-12)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.11-rc.1...v0.7.11)
+
+ * Add a bunch of logging around sending messages
+   [\#460](https://github.com/matrix-org/matrix-js-sdk/pull/460)
+
+Changes in [0.7.11-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.11-rc.1) (2017-06-09)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.10...v0.7.11-rc.1)
+
+ * Make TimelineWindow.load resolve quicker if we have the events
+   [\#458](https://github.com/matrix-org/matrix-js-sdk/pull/458)
+ * Stop peeking when a matrix client is stopped
+   [\#451](https://github.com/matrix-org/matrix-js-sdk/pull/451)
+ * Update README: Clarify how to install libolm
+   [\#450](https://github.com/matrix-org/matrix-js-sdk/pull/450)
+
+Changes in [0.7.10](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.10) (2017-06-02)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.9...v0.7.10)
+
+ * BREAKING CHANGE: The SDK no longer ``require``s ``olm`` - instead it expects
+   libolm to be provided as an ``Olm`` global. This will only affect
+   applications which use end-to-end encryption. See the
+   [README](README.md#end-to-end-encryption-support) for details.
+
+ * indexeddb-crypto-store: fix db deletion
+   [\#447](https://github.com/matrix-org/matrix-js-sdk/pull/447)
+ * Load Olm from the global rather than requiring it.
+   [\#446](https://github.com/matrix-org/matrix-js-sdk/pull/446)
+
+Changes in [0.7.9](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.9) (2017-06-01)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.8...v0.7.9)
+
+ * Initial framework for indexeddb-backed crypto store
+   [\#445](https://github.com/matrix-org/matrix-js-sdk/pull/445)
+ * Factor out reEmit to a common module
+   [\#444](https://github.com/matrix-org/matrix-js-sdk/pull/444)
+ * crypto/algorithms/base.js: Convert to es6
+   [\#443](https://github.com/matrix-org/matrix-js-sdk/pull/443)
+ * maySendRedactionForEvent for userId
+   [\#435](https://github.com/matrix-org/matrix-js-sdk/pull/435)
+ * MatrixClient: add getUserId()
+   [\#441](https://github.com/matrix-org/matrix-js-sdk/pull/441)
+ * Run jsdoc on a custom babeling of the source
+   [\#442](https://github.com/matrix-org/matrix-js-sdk/pull/442)
+ * Add in a public api getStoredDevice allowing clients to get a specific
+   device
+   [\#439](https://github.com/matrix-org/matrix-js-sdk/pull/439)
+
 Changes in [0.7.8](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.8) (2017-05-22)
 ================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.8-rc.1...v0.7.8)

README.md

@@ -13,7 +13,8 @@ In a browser
 Download either the full or minified version from
 https://github.com/matrix-org/matrix-js-sdk/releases/latest and add that as a
 ``<script>`` to your page. There will be a global variable ``matrixcs``
-attached to ``window`` through which you can access the SDK.
+attached to ``window`` through which you can access the SDK. See below for how to
+include libolm to enable end-to-end-encryption.
 
 Please check [the working browser example](examples/browser) for more information.
 
@@ -29,8 +30,9 @@ In Node.js
     console.log("Public Rooms: %s", JSON.stringify(data));
   });
 ```
+See below for how to include libolm to enable end-to-end-encryption. Please check
+[the Node.js terminal app](examples/node) for a more complex example.
 
-Please check [the Node.js terminal app](examples/node) for a more complex example.
 
 What does this SDK do?
 ----------------------
@@ -236,6 +238,46 @@ host the API reference from the source files like this:
 
 Then visit ``http://localhost:8005`` to see the API docs.
 
+End-to-end encryption support
+=============================
+
+The SDK supports end-to-end encryption via the Olm and Megolm protocols, using
+[libolm](http://matrix.org/git/olm). It is left up to the application to make
+libolm available, via the ``Olm`` global.
+
+It is also necessry to call ``matrixClient.initCrypto()`` after creating a new
+``MatrixClient`` (but **before** calling ``matrixClient.startClient()``) to
+initialise the crypto layer.
+
+If the ``Olm`` global is not available, the SDK will show a warning, as shown
+below; ``initCrypto()`` will also fail.
+
+```
+Unable to load crypto module: crypto will be disabled: Error: global.Olm is not defined
+```
+
+If the crypto layer is not (successfully) initialised, the SDK will continue to
+work for unencrypted rooms, but it will not support the E2E parts of the Matrix
+specification.
+
+To provide the Olm library in a browser application:
+
+ * download the transpiled libolm (from https://matrix.org/packages/npm/olm/).
+ * load ``olm.js`` as a ``<script>`` *before* ``browser-matrix.js``.
+ 
+To provide the Olm library in a node.js application:
+
+ * ``npm install https://matrix.org/packages/npm/olm/olm-2.2.2.tgz``
+   (replace the URL with the latest version you want to use from
+    https://matrix.org/packages/npm/olm/)
+ * ``global.Olm = require('olm');`` *before* loading ``matrix-js-sdk``.
+
+If you want to package Olm as dependency for your node.js application, you
+can use ``npm install https://matrix.org/packages/npm/olm/olm-2.2.2.tgz 
+--save-optional`` (if your application also works without e2e crypto enabled)
+or ``--save`` (if it doesn't) to do so.
+
+
 Contributing
 ============
 *This section is for people who want to modify the SDK. If you just

browser-index.js

@@ -1,4 +1,23 @@
 var matrixcs = require("./lib/matrix");
 matrixcs.request(require("browser-request"));
+
+// just *accessing* indexedDB throws an exception in firefox with
+// indexeddb disabled.
+var indexedDB;
+try {
+    indexedDB = global.indexedDB;
+} catch(e) {}
+
+// if our browser (appears to) support indexeddb, use an indexeddb crypto store.
+if (indexedDB) {
+    matrixcs.setCryptoStoreFactory(
+        function() {
+            return new matrixcs.IndexedDBCryptoStore(
+                indexedDB, "matrix-js-sdk:crypto"
+            );
+        }
+    );
+}
+
 module.exports = matrixcs; // keep export for browserify package deps
 global.matrixcs = matrixcs;

package.json

@@ -1,20 +1,22 @@
 {
   "name": "matrix-js-sdk",
-  "version": "0.7.8",
+  "version": "0.9.2-cryptowraning.1",
   "description": "Matrix Client-Server SDK for Javascript",
   "main": "index.js",
   "scripts": {
     "test:build": "babel -s -d specbuild spec",
-    "test:run": "istanbul cover --report text --report cobertura --config .istanbul.yml -i \"lib/**/*.js\" _mocha -- --recursive specbuild --colors --reporter mocha-jenkins-reporter --reporter-options junit_report_path=reports/test-results.xml",
+    "test:run": "istanbul cover --report text --report cobertura --config .istanbul.yml -i \"lib/**/*.js\" node_modules/mocha/bin/_mocha -- --recursive specbuild --colors --reporter mocha-jenkins-reporter --reporter-options junit_report_path=reports/test-results.xml",
+    "test:watch": "mocha --watch --compilers js:babel-core/register --recursive spec --colors",
     "test": "npm run test:build && npm run test:run",
     "check": "npm run test:build && _mocha --recursive specbuild --colors",
-    "gendoc": "jsdoc -r lib -P package.json -R README.md -d .jsdoc",
+    "gendoc": "babel --no-babelrc -d .jsdocbuild src && jsdoc -r .jsdocbuild -P package.json -R README.md -d .jsdoc",
     "start": "babel -s -w -d lib src",
+    "clean": "rimraf lib dist",
     "build": "babel -s -d lib src && rimraf dist && mkdir dist && browserify -d browser-index.js | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js && uglifyjs -c -m -o dist/browser-matrix.min.js --source-map dist/browser-matrix.min.js.map --in-source-map dist/browser-matrix.js.map dist/browser-matrix.js",
     "dist": "npm run build",
     "watch": "watchify -d browser-index.js -o 'exorcist dist/browser-matrix.js.map > dist/browser-matrix.js' -v",
-    "lint": "eslint --max-warnings 115 src spec",
-    "prepublish": "npm run build && git rev-parse HEAD > git-revision.txt"
+    "lint": "eslint --max-warnings 109 src spec",
+    "prepublish": "npm run clean && npm run build && git rev-parse HEAD > git-revision.txt"
   },
   "repository": {
     "url": "https://github.com/matrix-org/matrix-js-sdk"
@@ -48,13 +50,17 @@
   ],
   "dependencies": {
     "another-json": "^0.2.0",
+    "babel-runtime": "^6.26.0",
+    "bluebird": "^3.5.0",
     "browser-request": "^0.3.3",
-    "q": "^1.4.1",
+    "content-type": "^1.0.2",
     "request": "^2.53.0"
   },
   "devDependencies": {
     "babel-cli": "^6.18.0",
     "babel-eslint": "^7.1.1",
+    "babel-plugin-transform-async-to-bluebird": "^1.1.1",
+    "babel-plugin-transform-runtime": "^6.23.0",
     "babel-preset-es2015": "^6.18.0",
     "browserify": "^14.0.0",
     "browserify-shim": "^3.8.13",
@@ -63,8 +69,9 @@
     "exorcist": "^0.4.0",
     "expect": "^1.20.2",
     "istanbul": "^0.4.5",
-    "jsdoc": "^3.4.0",
+    "jsdoc": "^3.5.5",
     "lolex": "^1.5.2",
+    "matrix-mock-request": "^1.2.0",
     "mocha": "^3.2.0",
     "mocha-jenkins-reporter": "^0.3.6",
     "rimraf": "^2.5.4",
@@ -73,16 +80,9 @@
     "uglify-js": "^2.8.26",
     "watchify": "^3.2.1"
   },
-  "optionalDependencies": {
-    "olm": "https://matrix.org/packages/npm/olm/olm-2.2.1.tgz"
-  },
   "browserify": {
     "transform": [
-      "sourceify",
-      "browserify-shim"
+      "sourceify"
     ]
-  },
-  "browserify-shim": {
-    "olm": "global:Olm"
   }
 }

release.sh

@@ -206,7 +206,9 @@ if [ -n "$signing_id" ]; then
 
     tarfile="$tag.tar.gz"
     gh_project_url=$(git remote get-url origin |
-                            sed -e 's#^git@github.com:#https://github.com/#' -e 's/\.git$//')
+                            sed -e 's#^git@github\.com:#https://github.com/#' \
+                                -e 's#^git\+ssh://git@github\.com/#https://github.com/#' \
+                                -e 's/\.git$//')
     project_name="${gh_project_url##*/}"
     curl -L "${gh_project_url}/archive/${tarfile}" -o "${tarfile}"
 

spec/TestClient.js

@@ -17,11 +17,14 @@ limitations under the License.
 
 "use strict";
 
+// load olm before the sdk if possible
+import './olm-loader';
+
 import sdk from '..';
 import testUtils from './test-utils';
-import MockHttpBackend from './mock-request';
+import MockHttpBackend from 'matrix-mock-request';
 import expect from 'expect';
-import q from 'q';
+import Promise from 'bluebird';
 
 /**
  * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
@@ -30,12 +33,20 @@ import q from 'q';
  * @param {string} userId
  * @param {string} deviceId
  * @param {string} accessToken
+ *
+ * @param {WebStorage=} sessionStoreBackend a web storage object to use for the
+ *     session store. If undefined, we will create a MockStorageApi.
  */
-export default function TestClient(userId, deviceId, accessToken) {
+export default function TestClient(
+    userId, deviceId, accessToken, sessionStoreBackend,
+) {
     this.userId = userId;
     this.deviceId = deviceId;
 
-    this.storage = new sdk.WebStorageSessionStore(new testUtils.MockStorageApi());
+    if (sessionStoreBackend === undefined) {
+        sessionStoreBackend = new testUtils.MockStorageApi();
+    }
+    this.storage = new sdk.WebStorageSessionStore(sessionStoreBackend);
     this.httpBackend = new MockHttpBackend();
     this.client = sdk.createClient({
         baseUrl: "http://" + userId + ".test.server",
@@ -74,7 +85,10 @@ TestClient.prototype.start = function() {
         pendingEventOrdering: 'detached',
     });
 
-    return this.httpBackend.flush().then(() => {
+    return Promise.all([
+        this.httpBackend.flushAllExpected(),
+        testUtils.syncPromise(this.client),
+    ]).then(() => {
         console.log(this + ': started');
     });
 };
@@ -115,7 +129,7 @@ TestClient.prototype.expectDeviceKeyUpload = function() {
 TestClient.prototype.awaitOneTimeKeyUpload = function() {
     if (Object.keys(this.oneTimeKeys).length != 0) {
         // already got one-time keys
-        return q(this.oneTimeKeys);
+        return Promise.resolve(this.oneTimeKeys);
     }
 
     this.httpBackend.when("POST", "/keys/upload")
@@ -140,7 +154,8 @@ TestClient.prototype.awaitOneTimeKeyUpload = function() {
               }};
           });
 
-    return this.httpBackend.flush('/keys/upload', 2).then((flushed) => {
+    // this can take ages
+    return this.httpBackend.flush('/keys/upload', 2, 1000).then((flushed) => {
         expect(flushed).toEqual(2);
         return this.oneTimeKeys;
     });
@@ -184,3 +199,18 @@ TestClient.prototype.getSigningKey = function() {
     const keyId = 'ed25519:' + this.deviceId;
     return this.deviceKeys.keys[keyId];
 };
+
+/**
+ * flush a single /sync request, and wait for the syncing event
+ *
+ * @returns {Promise} promise which completes once the sync has been flushed
+ */
+TestClient.prototype.flushSync = function() {
+    console.log(`${this}: flushSync`);
+    return Promise.all([
+        this.httpBackend.flush('/sync', 1),
+        testUtils.syncPromise(this.client),
+    ]).then(() => {
+        console.log(`${this}: flushSync completed`);
+    });
+};

spec/integ/devicelist-integ-spec.js

@@ -0,0 +1,369 @@
+import expect from 'expect';
+import Promise from 'bluebird';
+
+import TestClient from '../TestClient';
+import testUtils from '../test-utils';
+
+const ROOM_ID = "!room:id";
+
+/**
+ * get a /sync response which contains a single e2e room (ROOM_ID), with the
+ * members given
+ *
+ * @param {string[]} roomMembers
+ *
+ * @return {object} sync response
+ */
+function getSyncResponse(roomMembers) {
+    const stateEvents = [
+        testUtils.mkEvent({
+            type: 'm.room.encryption',
+            skey: '',
+            content: {
+                algorithm: 'm.megolm.v1.aes-sha2',
+            },
+        }),
+    ];
+
+    Array.prototype.push.apply(
+        stateEvents,
+        roomMembers.map(
+            (m) => testUtils.mkMembership({
+                mship: 'join',
+                sender: m,
+            }),
+        ),
+    );
+
+    const syncResponse = {
+        next_batch: 1,
+        rooms: {
+            join: {
+                [ROOM_ID]: {
+                    state: {
+                        events: stateEvents,
+                    },
+                },
+            },
+        },
+    };
+
+    return syncResponse;
+}
+
+
+describe("DeviceList management:", function() {
+    if (!global.Olm) {
+        console.warn('not running deviceList tests: Olm not present');
+        return;
+    }
+
+    let sessionStoreBackend;
+    let aliceTestClient;
+
+    async function createTestClient() {
+        const testClient = new TestClient(
+            "@alice:localhost", "xzcvb", "akjgkrgjs", sessionStoreBackend,
+        );
+        await testClient.client.initCrypto();
+        return testClient;
+    }
+
+    beforeEach(async function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+
+        // we create our own sessionStoreBackend so that we can use it for
+        // another TestClient.
+        sessionStoreBackend = new testUtils.MockStorageApi();
+
+        aliceTestClient = await createTestClient();
+    });
+
+    afterEach(function() {
+        aliceTestClient.stop();
+    });
+
+    it("Alice shouldn't do a second /query for non-e2e-capable devices", function() {
+        return aliceTestClient.start().then(function() {
+            const syncResponse = getSyncResponse(['@bob:xyz']);
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
+
+            return aliceTestClient.flushSync();
+        }).then(function() {
+            console.log("Forcing alice to download our device keys");
+
+            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(200, {
+                device_keys: {
+                    '@bob:xyz': {},
+                },
+            });
+
+            return Promise.all([
+                aliceTestClient.client.downloadKeys(['@bob:xyz']),
+                aliceTestClient.httpBackend.flush('/keys/query', 1),
+            ]);
+        }).then(function() {
+            console.log("Telling alice to send a megolm message");
+
+            aliceTestClient.httpBackend.when(
+                'PUT', '/send/',
+            ).respond(200, {
+                    event_id: '$event_id',
+            });
+
+            return Promise.all([
+                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+
+                // the crypto stuff can take a while, so give the requests a whole second.
+                aliceTestClient.httpBackend.flushAllExpected({
+                    timeout: 1000,
+                }),
+            ]);
+        });
+    });
+
+
+    it("We should not get confused by out-of-order device query responses",
+       () => {
+           // https://github.com/vector-im/riot-web/issues/3126
+           return aliceTestClient.start().then(() => {
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                   200, getSyncResponse(['@bob:xyz', '@chris:abc']));
+               return aliceTestClient.flushSync();
+           }).then(() => {
+               // to make sure the initial device queries are flushed out, we
+               // attempt to send a message.
+
+               aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
+                   200, {
+                       device_keys: {
+                           '@bob:xyz': {},
+                           '@chris:abc': {},
+                       },
+                   },
+               );
+
+               aliceTestClient.httpBackend.when('PUT', '/send/').respond(
+                   200, {event_id: '$event1'});
+
+               return Promise.all([
+                   aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+                   aliceTestClient.httpBackend.flush('/keys/query', 1).then(
+                       () => aliceTestClient.httpBackend.flush('/send/', 1),
+                   ),
+               ]);
+           }).then(() => {
+               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(1);
+
+               // invalidate bob's and chris's device lists in separate syncs
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                   next_batch: '2',
+                   device_lists: {
+                       changed: ['@bob:xyz'],
+                   },
+               });
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                   next_batch: '3',
+                   device_lists: {
+                       changed: ['@chris:abc'],
+                   },
+               });
+               // flush both syncs
+               return aliceTestClient.flushSync().then(() => {
+                   return aliceTestClient.flushSync();
+               });
+           }).then(() => {
+               // check that we don't yet have a request for chris's devices.
+               aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                   device_keys: {
+                       '@chris:abc': {},
+                   },
+                   token: '3',
+               }).respond(200, {
+                   device_keys: {'@chris:abc': {}},
+               });
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(0);
+               const bobStat = aliceTestClient.storage
+                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
+               if (bobStat != 1 && bobStat != 2) {
+                   throw new Error('Unexpected status for bob: wanted 1 or 2, got ' +
+                                   bobStat);
+               }
+
+               const chrisStat = aliceTestClient.storage
+                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
+               if (chrisStat != 1 && chrisStat != 2) {
+                   throw new Error('Unexpected status for chris: wanted 1 or 2, got ' +
+                                   chrisStat);
+               }
+
+               // now add an expectation for a query for bob's devices, and let
+               // it complete.
+               aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                   device_keys: {
+                       '@bob:xyz': {},
+                   },
+                   token: '2',
+               }).respond(200, {
+                   device_keys: {'@bob:xyz': {}},
+               });
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(1);
+
+               // wait for the client to stop processing the response
+               return aliceTestClient.client.downloadKeys(['@bob:xyz']);
+           }).then(() => {
+               const bobStat = aliceTestClient.storage
+                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
+               expect(bobStat).toEqual(3);
+               const chrisStat = aliceTestClient.storage
+                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
+               if (chrisStat != 1 && chrisStat != 2) {
+                   throw new Error('Unexpected status for chris: wanted 1 or 2, got ' +
+                                   bobStat);
+               }
+
+               // now let the query for chris's devices complete.
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(1);
+
+               // wait for the client to stop processing the response
+               return aliceTestClient.client.downloadKeys(['@chris:abc']);
+           }).then(() => {
+               const bobStat = aliceTestClient.storage
+                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
+               const chrisStat = aliceTestClient.storage
+                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
+
+               expect(bobStat).toEqual(3);
+               expect(chrisStat).toEqual(3);
+               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(3);
+           });
+       });
+
+    // https://github.com/vector-im/riot-web/issues/4983
+    describe("Alice should know she has stale device lists", () => {
+        beforeEach(async function() {
+            await aliceTestClient.start();
+
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                200, getSyncResponse(['@bob:xyz']));
+            await aliceTestClient.flushSync();
+
+            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
+                200, {
+                    device_keys: {
+                        '@bob:xyz': {},
+                    },
+                },
+            );
+            await aliceTestClient.httpBackend.flush('/keys/query', 1);
+
+            const bobStat = aliceTestClient.storage
+                      .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
+
+            expect(bobStat).toBeGreaterThan(
+                0, "Alice should be tracking bob's device list",
+            );
+        });
+
+        it("when Bob leaves", async function() {
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                200, {
+                    next_batch: 2,
+                    device_lists: {
+                        left: ['@bob:xyz'],
+                    },
+                    rooms: {
+                        join: {
+                            [ROOM_ID]: {
+                                timeline: {
+                                    events: [
+                                        testUtils.mkMembership({
+                                            mship: 'leave',
+                                            sender: '@bob:xyz',
+                                        }),
+                                    ],
+                                },
+                            },
+                        },
+                    },
+                },
+            );
+
+
+            await aliceTestClient.flushSync();
+
+            const bobStat = aliceTestClient.storage
+                      .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
+            expect(bobStat).toEqual(
+                0, "Alice should have marked bob's device list as untracked",
+            );
+        });
+
+        it("when Alice leaves", async function() {
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                200, {
+                    next_batch: 2,
+                    device_lists: {
+                        left: ['@bob:xyz'],
+                    },
+                    rooms: {
+                        leave: {
+                            [ROOM_ID]: {
+                                timeline: {
+                                    events: [
+                                        testUtils.mkMembership({
+                                            mship: 'leave',
+                                            sender: '@bob:xyz',
+                                        }),
+                                    ],
+                                },
+                            },
+                        },
+                    },
+                },
+            );
+
+            await aliceTestClient.flushSync();
+
+            const bobStat = aliceTestClient.storage
+                      .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
+            expect(bobStat).toEqual(
+                0, "Alice should have marked bob's device list as untracked",
+            );
+        });
+
+        it("when Bob leaves whilst Alice is offline", async function() {
+            aliceTestClient.stop();
+
+            const anotherTestClient = await createTestClient();
+
+            try {
+                anotherTestClient.httpBackend.when('GET', '/keys/changes').respond(
+                    200, {
+                        changed: [],
+                        left: ['@bob:xyz'],
+                    },
+                );
+                await anotherTestClient.start();
+                anotherTestClient.httpBackend.when('GET', '/sync').respond(
+                    200, getSyncResponse([]));
+                await anotherTestClient.flushSync();
+
+                const bobStat = anotherTestClient.storage
+                      .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
+
+                expect(bobStat).toEqual(
+                    0, "Alice should have marked bob's device list as untracked",
+                );
+            } finally {
+                anotherTestClient.stop();
+            }
+        });
+    });
+});

spec/integ/matrix-client-crypto.spec.js

@@ -25,9 +25,13 @@ limitations under the License.
 
 "use strict";
 import 'source-map-support/register';
+
+// load olm before the sdk if possible
+import '../olm-loader';
+
 import expect from 'expect';
 const sdk = require("../..");
-const q = require("q");
+import Promise from 'bluebird';
 const utils = require("../../lib/utils");
 const testUtils = require("../test-utils");
 const TestClient = require('../TestClient').default;
@@ -46,7 +50,7 @@ let bobMessages;
 
 function bobUploadsDeviceKeys() {
     bobTestClient.expectDeviceKeyUpload();
-    return q.all([
+    return Promise.all([
         bobTestClient.client.uploadKeys(),
         bobTestClient.httpBackend.flush(),
     ]).then(() => {
@@ -129,8 +133,8 @@ function expectAliClaimKeys() {
         // it can take a while to process the key query, so give it some extra
         // time, and make sure the claim actually happens rather than ploughing on
         // confusingly.
-        return aliTestClient.httpBackend.flush("/keys/claim", 1, 20).then((r) => {
-            expect(r).toEqual(1);
+        return aliTestClient.httpBackend.flush("/keys/claim", 1, 500).then((r) => {
+            expect(r).toEqual(1, "Ali did not claim Bob's keys");
         });
     });
 }
@@ -141,19 +145,16 @@ function aliDownloadsKeys() {
     expect(bobTestClient.getSigningKey()).toBeTruthy();
 
     const p1 = aliTestClient.client.downloadKeys([bobUserId]).then(function() {
-        expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([{
-            id: "bvcxz",
-            key: bobTestClient.getSigningKey(),
-            verified: false,
-            blocked: false,
-            display_name: null,
-        }]);
+        return aliTestClient.client.getStoredDevicesForUser(bobUserId);
+    }).then((devices) => {
+        expect(devices.length).toEqual(1);
+        expect(devices[0].deviceId).toEqual("bvcxz");
     });
     const p2 = expectAliQueryKeys();
 
     // check that the localStorage is updated as we expect (not sure this is
     // an integration test, but meh)
-    return q.all([p1, p2]).then(function() {
+    return Promise.all([p1, p2]).then(function() {
         const devices = aliTestClient.storage.getEndToEndDevicesForUser(bobUserId);
         expect(devices[bobDeviceId].keys).toEqual(bobTestClient.deviceKeys.keys);
         expect(devices[bobDeviceId].verified).
@@ -184,7 +185,7 @@ function bobEnablesEncryption() {
  * @return {promise} which resolves to the ciphertext for Bob's device.
  */
 function aliSendsFirstMessage() {
-    return q.all([
+    return Promise.all([
         sendMessage(aliTestClient.client),
         expectAliQueryKeys()
             .then(expectAliClaimKeys)
@@ -201,7 +202,7 @@ function aliSendsFirstMessage() {
  * @return {promise} which resolves to the ciphertext for Bob's device.
  */
 function aliSendsMessage() {
-    return q.all([
+    return Promise.all([
         sendMessage(aliTestClient.client),
         expectAliSendMessageRequest(),
     ]).spread(function(_, ciphertext) {
@@ -216,7 +217,7 @@ function aliSendsMessage() {
  * @return {promise} which resolves to the ciphertext for Ali's device.
  */
 function bobSendsReplyMessage() {
-    return q.all([
+    return Promise.all([
         sendMessage(bobTestClient.client),
         expectBobQueryKeys()
             .then(expectBobSendMessageRequest),
@@ -265,7 +266,7 @@ function sendMessage(client) {
 
 function expectSendMessageRequest(httpBackend) {
     const path = "/send/m.room.encrypted/";
-    const deferred = q.defer();
+    const deferred = Promise.defer();
     httpBackend.when("PUT", path).respond(200, function(path, content) {
         deferred.resolve(content);
         return {
@@ -273,8 +274,8 @@ function expectSendMessageRequest(httpBackend) {
         };
     });
 
-    // it can take a while to process the key query, so give it 20ms
-    return httpBackend.flush(path, 1, 20).then(() => deferred.promise);
+    // it can take a while to process the key query
+    return httpBackend.flush(path, 1).then(() => deferred.promise);
 }
 
 function aliRecvMessage() {
@@ -313,31 +314,37 @@ function recvMessage(httpBackend, client, sender, message) {
         },
     };
     httpBackend.when("GET", "/sync").respond(200, syncData);
-    const deferred = q.defer();
-    const onEvent = function(event) {
-        console.log(client.credentials.userId + " received event",
-                    event);
 
-        // ignore the m.room.member events
-        if (event.getType() == "m.room.member") {
-            return;
-        }
+    const eventPromise = new Promise((resolve, reject) => {
+        const onEvent = function(event) {
+            // ignore the m.room.member events
+            if (event.getType() == "m.room.member") {
+                return;
+            }
+            console.log(client.credentials.userId + " received event",
+                        event);
 
+            client.removeListener("event", onEvent);
+            resolve(event);
+        };
+        client.on("event", onEvent);
+    });
+
+    httpBackend.flush();
+
+    return eventPromise.then((event) => {
+        expect(event.isEncrypted()).toBeTruthy();
+
+        // it may still be being decrypted
+        return testUtils.awaitDecryption(event);
+    }).then((event) => {
         expect(event.getType()).toEqual("m.room.message");
         expect(event.getContent()).toEqual({
             msgtype: "m.text",
             body: "Hello, World",
         });
         expect(event.isEncrypted()).toBeTruthy();
-
-        client.removeListener("event", onEvent);
-        deferred.resolve();
-    };
-
-    client.on("event", onEvent);
-
-    httpBackend.flush();
-    return deferred.promise;
+    });
 }
 
 
@@ -375,9 +382,7 @@ function firstSync(testClient) {
     };
 
     testClient.httpBackend.when("GET", "/sync").respond(200, syncData);
-    return testClient.httpBackend.flush("/sync", 1).then(() => {
-        return testUtils.syncPromise(testClient.client);
-    });
+    return testClient.flushSync();
 }
 
 
@@ -386,11 +391,14 @@ describe("MatrixClient crypto", function() {
         return;
     }
 
-    beforeEach(function() {
+    beforeEach(async function() {
         testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
 
         aliTestClient = new TestClient(aliUserId, aliDeviceId, aliAccessToken);
+        await aliTestClient.client.initCrypto();
+
         bobTestClient = new TestClient(bobUserId, bobDeviceId, bobAccessToken);
+        await bobTestClient.client.initCrypto();
 
         aliMessages = [];
         bobMessages = [];
@@ -404,19 +412,19 @@ describe("MatrixClient crypto", function() {
     });
 
     it("Bob uploads device keys", function() {
-        return q()
+        return Promise.resolve()
             .then(bobUploadsDeviceKeys);
     });
 
     it("Ali downloads Bobs device keys", function(done) {
-        q()
+        Promise.resolve()
             .then(bobUploadsDeviceKeys)
             .then(aliDownloadsKeys)
-            .catch(testUtils.failTest).done(done);
+            .nodeify(done);
     });
 
     it("Ali gets keys with an invalid signature", function(done) {
-        q()
+        Promise.resolve()
             .then(bobUploadsDeviceKeys)
             .then(function() {
                 // tamper bob's keys
@@ -424,14 +432,17 @@ describe("MatrixClient crypto", function() {
                 expect(bobDeviceKeys.keys["curve25519:" + bobDeviceId]).toBeTruthy();
                 bobDeviceKeys.keys["curve25519:" + bobDeviceId] += "abc";
 
-                return q.all(aliTestClient.client.downloadKeys([bobUserId]),
-                             expectAliQueryKeys());
-            })
-            .then(function() {
+                return Promise.all([
+                    aliTestClient.client.downloadKeys([bobUserId]),
+                    expectAliQueryKeys(),
+                ]);
+            }).then(function() {
+                return aliTestClient.client.getStoredDevicesForUser(bobUserId);
+            }).then((devices) => {
                 // should get an empty list
-                expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([]);
+                expect(devices).toEqual([]);
             })
-            .catch(testUtils.failTest).done(done);
+            .nodeify(done);
     });
 
     it("Ali gets keys with an incorrect userId", function(done) {
@@ -463,14 +474,19 @@ describe("MatrixClient crypto", function() {
             return {device_keys: result};
         });
 
-        q.all(
+        Promise.all([
             aliTestClient.client.downloadKeys([bobUserId, eveUserId]),
             aliTestClient.httpBackend.flush("/keys/query", 1),
-        ).then(function() {
+        ]).then(function() {
+            return Promise.all([
+                aliTestClient.client.getStoredDevicesForUser(bobUserId),
+                aliTestClient.client.getStoredDevicesForUser(eveUserId),
+            ]);
+        }).spread((bobDevices, eveDevices) => {
             // should get an empty list
-            expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([]);
-            expect(aliTestClient.client.listDeviceKeys(eveUserId)).toEqual([]);
-        }).catch(testUtils.failTest).done(done);
+            expect(bobDevices).toEqual([]);
+            expect(eveDevices).toEqual([]);
+        }).nodeify(done);
     });
 
     it("Ali gets keys with an incorrect deviceId", function(done) {
@@ -500,18 +516,20 @@ describe("MatrixClient crypto", function() {
             return {device_keys: result};
         });
 
-        q.all(
+        Promise.all([
             aliTestClient.client.downloadKeys([bobUserId]),
             aliTestClient.httpBackend.flush("/keys/query", 1),
-        ).then(function() {
+        ]).then(function() {
+            return aliTestClient.client.getStoredDevicesForUser(bobUserId);
+        }).then((devices) => {
             // should get an empty list
-            expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([]);
-        }).catch(testUtils.failTest).done(done);
+            expect(devices).toEqual([]);
+        }).nodeify(done);
     });
 
 
     it("Bob starts his client and uploads device keys and one-time keys", function() {
-        return q()
+        return Promise.resolve()
             .then(() => bobTestClient.start())
             .then(() => bobTestClient.awaitOneTimeKeyUpload())
             .then((keys) => {
@@ -521,28 +539,27 @@ describe("MatrixClient crypto", function() {
     });
 
     it("Ali sends a message", function(done) {
-        q()
+        Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
             .then(() => firstSync(aliTestClient))
             .then(aliEnablesEncryption)
             .then(aliSendsFirstMessage)
-            .catch(testUtils.failTest).nodeify(done);
+            .nodeify(done);
     });
 
-    it("Bob receives a message", function(done) {
-        q()
+    it("Bob receives a message", function() {
+        return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
             .then(() => firstSync(aliTestClient))
             .then(aliEnablesEncryption)
             .then(aliSendsFirstMessage)
-            .then(bobRecvMessage)
-            .catch(testUtils.failTest).done(done);
+            .then(bobRecvMessage);
     });
 
-    it("Bob receives a message with a bogus sender", function(done) {
-        q()
+    it("Bob receives a message with a bogus sender", function() {
+        return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
             .then(() => firstSync(aliTestClient))
@@ -572,34 +589,30 @@ describe("MatrixClient crypto", function() {
                 };
                 bobTestClient.httpBackend.when("GET", "/sync").respond(200, syncData);
 
-                const deferred = q.defer();
-                const onEvent = function(event) {
-                    console.log(bobUserId + " received event",
-                                event);
-
-                    // ignore the m.room.member events
-                    if (event.getType() == "m.room.member") {
-                        return;
-                    }
-
-                    expect(event.getType()).toEqual("m.room.message");
-                    expect(event.getContent().msgtype).toEqual("m.bad.encrypted");
-                    expect(event.isEncrypted()).toBeTruthy();
-
-                    bobTestClient.client.removeListener("event", onEvent);
-                    deferred.resolve();
-                };
-
-                bobTestClient.client.on("event", onEvent);
+                const eventPromise = new Promise((resolve, reject) => {
+                    const onEvent = function(event) {
+                        console.log(bobUserId + " received event",
+                                    event);
+                        resolve(event);
+                    };
+                    bobTestClient.client.once("event", onEvent);
+                });
 
                 bobTestClient.httpBackend.flush();
-                return deferred.promise;
-            })
-            .catch(testUtils.failTest).done(done);
+                return eventPromise;
+            }).then((event) => {
+                expect(event.isEncrypted()).toBeTruthy();
+
+                // it may still be being decrypted
+                return testUtils.awaitDecryption(event);
+            }).then((event) => {
+                expect(event.getType()).toEqual("m.room.message");
+                expect(event.getContent().msgtype).toEqual("m.bad.encrypted");
+            });
     });
 
     it("Ali blocks Bob's device", function(done) {
-        q()
+        Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
             .then(() => firstSync(aliTestClient))
@@ -613,12 +626,12 @@ describe("MatrixClient crypto", function() {
                           // no unblocked devices, so the ciphertext should be empty
                           expect(sentContent.ciphertext).toEqual({});
                       });
-                return q.all([p1, p2]);
+                return Promise.all([p1, p2]);
             }).nodeify(done);
     });
 
     it("Bob receives two pre-key messages", function(done) {
-        q()
+        Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
             .then(() => firstSync(aliTestClient))
@@ -627,11 +640,11 @@ describe("MatrixClient crypto", function() {
             .then(bobRecvMessage)
             .then(aliSendsMessage)
             .then(bobRecvMessage)
-            .catch(testUtils.failTest).done(done);
+            .nodeify(done);
     });
 
     it("Bob replies to the message", function() {
-        return q()
+        return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
             .then(() => firstSync(aliTestClient))
@@ -645,64 +658,10 @@ describe("MatrixClient crypto", function() {
             }).then(aliRecvMessage);
     });
 
-
-    it("Ali does a key query when she gets a new_device event", function() {
-        return q()
-            .then(() => aliTestClient.start())
-            .then(() => firstSync(aliTestClient))
-
-            // ali will only care about bob's new_device if she is tracking
-            // bob's devices, which she will do if we enable encryption
-            .then(aliEnablesEncryption)
-
-            .then(() => {
-                aliTestClient.expectKeyQuery({
-                    device_keys: {
-                        [aliUserId]: {},
-                        [bobUserId]: {},
-                    },
-                });
-                return aliTestClient.httpBackend.flush('/keys/query', 1);
-            })
-
-            // make sure that the initial key download has completed
-            // (downloadKeys will wait until it does)
-            .then(() => {
-                return aliTestClient.client.downloadKeys([bobUserId]);
-            })
-
-            .then(function() {
-                const syncData = {
-                    next_batch: '2',
-                    to_device: {
-                        events: [
-                            testUtils.mkEvent({
-                                content: {
-                                    device_id: 'TEST_DEVICE',
-                                    rooms: [],
-                                },
-                                sender: bobUserId,
-                                type: 'm.new_device',
-                            }),
-                        ],
-                    },
-                };
-                aliTestClient.httpBackend.when('GET', '/sync').respond(200, syncData);
-                return aliTestClient.httpBackend.flush('/sync', 1);
-            }).then(() => {
-                aliTestClient.expectKeyQuery({
-                    device_keys: {
-                        [bobUserId]: {},
-                    },
-                });
-                return aliTestClient.httpBackend.flush('/keys/query', 1);
-            });
-    });
-
     it("Ali does a key query when encryption is enabled", function() {
         // enabling encryption in the room should make alice download devices
         // for both members.
-        return q()
+        return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => firstSync(aliTestClient))
             .then(() => {
@@ -736,7 +695,56 @@ describe("MatrixClient crypto", function() {
                         [bobUserId]: {},
                     },
                 });
-                return aliTestClient.httpBackend.flush('/keys/query', 1);
+                return aliTestClient.httpBackend.flushAllExpected();
             });
     });
+
+    it("Upload new oneTimeKeys based on a /sync request - no count-asking", function() {
+        // Send a response which causes a key upload
+        const httpBackend = aliTestClient.httpBackend;
+        const syncDataEmpty = {
+            next_batch: "a",
+            device_one_time_keys_count: {
+                signed_curve25519: 0,
+            },
+        };
+
+        // enqueue expectations:
+        // * Sync with empty one_time_keys => upload keys
+
+        return Promise.resolve()
+            .then(() => {
+                console.log(aliTestClient + ': starting');
+                httpBackend.when("GET", "/pushrules").respond(200, {});
+                httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+                aliTestClient.expectDeviceKeyUpload();
+
+                // we let the client do a very basic initial sync, which it needs before
+                // it will upload one-time keys.
+                httpBackend.when("GET", "/sync").respond(200, syncDataEmpty);
+
+                aliTestClient.client.startClient({});
+
+                return httpBackend.flushAllExpected().then(() => {
+                    console.log(aliTestClient + ': started');
+                });
+            })
+            .then(() => httpBackend.when("POST", "/keys/upload")
+                .respond(200, (path, content) => {
+                    expect(content.one_time_keys).toBeTruthy();
+                    expect(content.one_time_keys).toNotEqual({});
+                    expect(Object.keys(content.one_time_keys).length)
+                        .toBeGreaterThanOrEqualTo(1);
+                    console.log('received %i one-time keys',
+                                Object.keys(content.one_time_keys).length);
+                    // cancel futher calls by telling the client
+                    // we have more than we need
+                    return {
+                       one_time_key_counts: {
+                           signed_curve25519: 70,
+                       },
+                    };
+                }))
+            .then(() => httpBackend.flushAllExpected());
+    });
 });

spec/integ/matrix-client-event-emitter.spec.js

@@ -1,10 +1,11 @@
 "use strict";
 import 'source-map-support/register';
 const sdk = require("../..");
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const utils = require("../test-utils");
 
 import expect from 'expect';
+import Promise from 'bluebird';
 
 describe("MatrixClient events", function() {
     const baseUrl = "http://localhost.or.something";
@@ -101,7 +102,7 @@ describe("MatrixClient events", function() {
         };
 
         it("should emit events from both the first and subsequent /sync calls",
-        function(done) {
+        function() {
             httpBackend.when("GET", "/sync").respond(200, SYNC_DATA);
             httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
 
@@ -130,11 +131,16 @@ describe("MatrixClient events", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                // wait for two SYNCING events
+                utils.syncPromise(client).then(() => {
+                    return utils.syncPromise(client);
+                }),
+                httpBackend.flushAllExpected(),
+            ]).then(() => {
                 expect(expectedEvents.length).toEqual(
                     0, "Failed to see all events from /sync calls",
                 );
-                done();
             });
         });
 
@@ -157,13 +163,13 @@ describe("MatrixClient events", function() {
             });
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            httpBackend.flushAllExpected().done(function() {
                 expect(fired).toBe(true, "User.presence didn't fire.");
                 done();
             });
         });
 
-        it("should emit Room events", function(done) {
+        it("should emit Room events", function() {
             httpBackend.when("GET", "/sync").respond(200, SYNC_DATA);
             httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
             let roomInvokeCount = 0;
@@ -183,7 +189,10 @@ describe("MatrixClient events", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 2),
+            ]).then(function() {
                 expect(roomInvokeCount).toEqual(
                     1, "Room fired wrong number of times.",
                 );
@@ -193,11 +202,10 @@ describe("MatrixClient events", function() {
                 expect(timelineFireCount).toEqual(
                     3, "Room.timeline fired the wrong number of times",
                 );
-                done();
             });
         });
 
-        it("should emit RoomState events", function(done) {
+        it("should emit RoomState events", function() {
             httpBackend.when("GET", "/sync").respond(200, SYNC_DATA);
             httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
 
@@ -232,7 +240,10 @@ describe("MatrixClient events", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 2),
+            ]).then(function() {
                 expect(membersInvokeCount).toEqual(
                     1, "RoomState.members fired wrong number of times",
                 );
@@ -242,11 +253,10 @@ describe("MatrixClient events", function() {
                 expect(eventsInvokeCount).toEqual(
                     2, "RoomState.events fired wrong number of times",
                 );
-                done();
             });
         });
 
-        it("should emit RoomMember events", function(done) {
+        it("should emit RoomMember events", function() {
             httpBackend.when("GET", "/sync").respond(200, SYNC_DATA);
             httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
 
@@ -271,7 +281,10 @@ describe("MatrixClient events", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 2),
+            ]).then(function() {
                 expect(typingInvokeCount).toEqual(
                     1, "RoomMember.typing fired wrong number of times",
                 );
@@ -284,11 +297,10 @@ describe("MatrixClient events", function() {
                 expect(membershipInvokeCount).toEqual(
                     1, "RoomMember.membership fired wrong number of times",
                 );
-                done();
             });
         });
 
-        it("should emit Session.logged_out on M_UNKNOWN_TOKEN", function(done) {
+        it("should emit Session.logged_out on M_UNKNOWN_TOKEN", function() {
             httpBackend.when("GET", "/sync").respond(401, { errcode: 'M_UNKNOWN_TOKEN' });
 
             let sessionLoggedOutCount = 0;
@@ -298,11 +310,10 @@ describe("MatrixClient events", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return httpBackend.flushAllExpected().then(function() {
                 expect(sessionLoggedOutCount).toEqual(
                     1, "Session.logged_out fired wrong number of times",
                 );
-                done();
             });
         });
     });

spec/integ/matrix-client-event-timeline.spec.js

@@ -1,8 +1,8 @@
 "use strict";
 import 'source-map-support/register';
-const q = require("q");
+import Promise from 'bluebird';
 const sdk = require("../..");
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const utils = require("../test-utils");
 const EventTimeline = sdk.EventTimeline;
 
@@ -82,7 +82,7 @@ function startClient(httpBackend, client) {
     client.startClient();
 
     // set up a promise which will resolve once the client is initialised
-    const deferred = q.defer();
+    const deferred = Promise.defer();
     client.on("sync", function(state) {
         console.log("sync", state);
         if (state != "SYNCING") {
@@ -91,8 +91,10 @@ function startClient(httpBackend, client) {
         deferred.resolve();
     });
 
-    httpBackend.flush();
-    return deferred.promise;
+    return Promise.all([
+        httpBackend.flushAllExpected(),
+        deferred.promise,
+    ]);
 }
 
 describe("getEventTimeline support", function() {
@@ -125,10 +127,10 @@ describe("getEventTimeline support", function() {
             expect(function() {
                 client.getEventTimeline(timelineSet, "event");
             }).toThrow();
-        }).catch(utils.failTest).done(done);
+        }).nodeify(done);
     });
 
-    it("timeline support works when enabled", function(done) {
+    it("timeline support works when enabled", function() {
         client = sdk.createClient({
             baseUrl: baseUrl,
             userId: userId,
@@ -136,16 +138,13 @@ describe("getEventTimeline support", function() {
             timelineSupport: true,
         });
 
-        startClient(httpBackend, client,
-        ).then(function() {
+        return startClient(httpBackend, client).then(() => {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
             expect(function() {
                 client.getEventTimeline(timelineSet, "event");
             }).toNotThrow();
-        }).catch(utils.failTest).done(done);
-
-        httpBackend.flush().catch(utils.failTest);
+        });
     });
 
 
@@ -196,18 +195,19 @@ describe("getEventTimeline support", function() {
                 },
             });
 
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 2),
+            ]);
+        }).then(function() {
+            expect(room.timeline.length).toEqual(1);
+            expect(room.timeline[0].event).toEqual(EVENTS[1]);
+
             httpBackend.when("GET", "/messages").respond(200, {
                 chunk: [EVENTS[0]],
                 start: "pagin_start",
                 end: "pagin_end",
             });
-
-
-            return httpBackend.flush("/sync", 2);
-        }).then(function() {
-            expect(room.timeline.length).toEqual(1);
-            expect(room.timeline[0].event).toEqual(EVENTS[1]);
-
             httpBackend.flush("/messages", 1);
             return client.scrollback(room);
         }).then(function() {
@@ -215,7 +215,7 @@ describe("getEventTimeline support", function() {
             expect(room.timeline[0].event).toEqual(EVENTS[0]);
             expect(room.timeline[1].event).toEqual(EVENTS[1]);
             expect(room.oldState.paginationToken).toEqual("pagin_end");
-        }).catch(utils.failTest).done(done);
+        }).nodeify(done);
     });
 });
 
@@ -225,7 +225,7 @@ describe("MatrixClient event timelines", function() {
     let client = null;
     let httpBackend = null;
 
-    beforeEach(function(done) {
+    beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
         httpBackend = new HttpBackend();
         sdk.request(httpBackend.requestFn);
@@ -237,8 +237,7 @@ describe("MatrixClient event timelines", function() {
             timelineSupport: true,
         });
 
-        startClient(httpBackend, client)
-            .catch(utils.failTest).done(done);
+        return startClient(httpBackend, client);
     });
 
     afterEach(function() {
@@ -247,7 +246,7 @@ describe("MatrixClient event timelines", function() {
     });
 
     describe("getEventTimeline", function() {
-        it("should create a new timeline for new events", function(done) {
+        it("should create a new timeline for new events", function() {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
             httpBackend.when("GET", "/rooms/!foo%3Abar/context/event1%3Abar")
@@ -265,22 +264,23 @@ describe("MatrixClient event timelines", function() {
                     };
                 });
 
-            client.getEventTimeline(timelineSet, "event1:bar").then(function(tl) {
-                expect(tl.getEvents().length).toEqual(4);
-                for (let i = 0; i < 4; i++) {
-                    expect(tl.getEvents()[i].event).toEqual(EVENTS[i]);
-                    expect(tl.getEvents()[i].sender.name).toEqual(userName);
-                }
-                expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toEqual("start_token");
-                expect(tl.getPaginationToken(EventTimeline.FORWARDS))
-                    .toEqual("end_token");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            return Promise.all([
+                client.getEventTimeline(timelineSet, "event1:bar").then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(4);
+                    for (let i = 0; i < 4; i++) {
+                        expect(tl.getEvents()[i].event).toEqual(EVENTS[i]);
+                        expect(tl.getEvents()[i].sender.name).toEqual(userName);
+                    }
+                    expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toEqual("start_token");
+                    expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                        .toEqual("end_token");
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
         });
 
-        it("should return existing timeline for known events", function(done) {
+        it("should return existing timeline for known events", function() {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
             httpBackend.when("GET", "/sync").respond(200, {
@@ -299,17 +299,20 @@ describe("MatrixClient event timelines", function() {
                 },
             });
 
-            httpBackend.flush("/sync").then(function() {
+            return Promise.all([
+                httpBackend.flush("/sync"),
+                utils.syncPromise(client),
+            ]).then(function() {
                 return client.getEventTimeline(timelineSet, EVENTS[0].event_id);
             }).then(function(tl) {
                 expect(tl.getEvents().length).toEqual(2);
                 expect(tl.getEvents()[1].event).toEqual(EVENTS[0]);
                 expect(tl.getEvents()[1].sender.name).toEqual(userName);
-                expect(tl.getPaginationToken(EventTimeline.BACKWARDS)).toEqual("f_1_1");
-                // expect(tl.getPaginationToken(EventTimeline.FORWARDS)).toEqual("s_5_4");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+                expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                    .toEqual("f_1_1");
+                // expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                //    .toEqual("s_5_4");
+            });
         });
 
         it("should update timelines where they overlap a previous /sync", function() {
@@ -344,7 +347,7 @@ describe("MatrixClient event timelines", function() {
                     };
                 });
 
-            const deferred = q.defer();
+            const deferred = Promise.defer();
             client.on("sync", function() {
                 client.getEventTimeline(timelineSet, EVENTS[2].event_id,
                 ).then(function(tl) {
@@ -360,14 +363,14 @@ describe("MatrixClient event timelines", function() {
                         (e) => deferred.reject(e));
             });
 
-            return q.all([
-                httpBackend.flush(),
+            return Promise.all([
+                httpBackend.flushAllExpected(),
                 deferred.promise,
             ]);
         });
 
         it("should join timelines where they overlap a previous /context",
-          function(done) {
+          function() {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
 
@@ -427,41 +430,42 @@ describe("MatrixClient event timelines", function() {
 
             let tl0;
             let tl3;
-            client.getEventTimeline(timelineSet, EVENTS[0].event_id,
-            ).then(function(tl) {
-                expect(tl.getEvents().length).toEqual(1);
-                tl0 = tl;
-                return client.getEventTimeline(timelineSet, EVENTS[2].event_id);
-            }).then(function(tl) {
-                expect(tl.getEvents().length).toEqual(1);
-                return client.getEventTimeline(timelineSet, EVENTS[3].event_id);
-            }).then(function(tl) {
-                expect(tl.getEvents().length).toEqual(1);
-                tl3 = tl;
-                return client.getEventTimeline(timelineSet, EVENTS[1].event_id);
-            }).then(function(tl) {
-                // we expect it to get merged in with event 2
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[0].event).toEqual(EVENTS[1]);
-                expect(tl.getEvents()[1].event).toEqual(EVENTS[2]);
-                expect(tl.getNeighbouringTimeline(EventTimeline.BACKWARDS))
-                    .toBe(tl0);
-                expect(tl.getNeighbouringTimeline(EventTimeline.FORWARDS))
-                    .toBe(tl3);
-                expect(tl0.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toEqual("start_token0");
-                expect(tl0.getPaginationToken(EventTimeline.FORWARDS))
-                    .toBe(null);
-                expect(tl3.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toBe(null);
-                expect(tl3.getPaginationToken(EventTimeline.FORWARDS))
-                    .toEqual("end_token3");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            return Promise.all([
+                client.getEventTimeline(timelineSet, EVENTS[0].event_id,
+                ).then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(1);
+                    tl0 = tl;
+                    return client.getEventTimeline(timelineSet, EVENTS[2].event_id);
+                }).then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(1);
+                    return client.getEventTimeline(timelineSet, EVENTS[3].event_id);
+                }).then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(1);
+                    tl3 = tl;
+                    return client.getEventTimeline(timelineSet, EVENTS[1].event_id);
+                }).then(function(tl) {
+                    // we expect it to get merged in with event 2
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[0].event).toEqual(EVENTS[1]);
+                    expect(tl.getEvents()[1].event).toEqual(EVENTS[2]);
+                    expect(tl.getNeighbouringTimeline(EventTimeline.BACKWARDS))
+                        .toBe(tl0);
+                    expect(tl.getNeighbouringTimeline(EventTimeline.FORWARDS))
+                        .toBe(tl3);
+                    expect(tl0.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toEqual("start_token0");
+                    expect(tl0.getPaginationToken(EventTimeline.FORWARDS))
+                        .toBe(null);
+                    expect(tl3.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toBe(null);
+                    expect(tl3.getPaginationToken(EventTimeline.FORWARDS))
+                        .toEqual("end_token3");
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
         });
 
-        it("should fail gracefully if there is no event field", function(done) {
+        it("should fail gracefully if there is no event field", function() {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
             // we fetch event 0, then 2, then 3, and finally 1. 1 is returned
@@ -477,20 +481,21 @@ describe("MatrixClient event timelines", function() {
                     };
                 });
 
-            client.getEventTimeline(timelineSet, "event1",
-            ).then(function(tl) {
-                // could do with a fail()
-                expect(true).toBeFalsy();
-            }).catch(function(e) {
-                expect(String(e)).toMatch(/'event'/);
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            return Promise.all([
+                client.getEventTimeline(timelineSet, "event1",
+                ).then(function(tl) {
+                    // could do with a fail()
+                    expect(true).toBeFalsy();
+                }, function(e) {
+                    expect(String(e)).toMatch(/'event'/);
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
         });
     });
 
     describe("paginateEventTimeline", function() {
-        it("should allow you to paginate backwards", function(done) {
+        it("should allow you to paginate backwards", function() {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
 
@@ -521,27 +526,28 @@ describe("MatrixClient event timelines", function() {
                 });
 
             let tl;
-            client.getEventTimeline(timelineSet, EVENTS[0].event_id,
-            ).then(function(tl0) {
-                tl = tl0;
-                return client.paginateEventTimeline(tl, {backwards: true});
-            }).then(function(success) {
-                expect(success).toBeTruthy();
-                expect(tl.getEvents().length).toEqual(3);
-                expect(tl.getEvents()[0].event).toEqual(EVENTS[2]);
-                expect(tl.getEvents()[1].event).toEqual(EVENTS[1]);
-                expect(tl.getEvents()[2].event).toEqual(EVENTS[0]);
-                expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toEqual("start_token1");
-                expect(tl.getPaginationToken(EventTimeline.FORWARDS))
-                    .toEqual("end_token0");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            return Promise.all([
+                client.getEventTimeline(timelineSet, EVENTS[0].event_id,
+                ).then(function(tl0) {
+                    tl = tl0;
+                    return client.paginateEventTimeline(tl, {backwards: true});
+                }).then(function(success) {
+                    expect(success).toBeTruthy();
+                    expect(tl.getEvents().length).toEqual(3);
+                    expect(tl.getEvents()[0].event).toEqual(EVENTS[2]);
+                    expect(tl.getEvents()[1].event).toEqual(EVENTS[1]);
+                    expect(tl.getEvents()[2].event).toEqual(EVENTS[0]);
+                    expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toEqual("start_token1");
+                    expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                        .toEqual("end_token0");
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
         });
 
 
-        it("should allow you to paginate forwards", function(done) {
+        it("should allow you to paginate forwards", function() {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
 
@@ -572,24 +578,25 @@ describe("MatrixClient event timelines", function() {
                 });
 
             let tl;
-            client.getEventTimeline(timelineSet, EVENTS[0].event_id,
-            ).then(function(tl0) {
-                tl = tl0;
-                return client.paginateEventTimeline(
-                    tl, {backwards: false, limit: 20});
-            }).then(function(success) {
-                expect(success).toBeTruthy();
-                expect(tl.getEvents().length).toEqual(3);
-                expect(tl.getEvents()[0].event).toEqual(EVENTS[0]);
-                expect(tl.getEvents()[1].event).toEqual(EVENTS[1]);
-                expect(tl.getEvents()[2].event).toEqual(EVENTS[2]);
-                expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toEqual("start_token0");
-                expect(tl.getPaginationToken(EventTimeline.FORWARDS))
-                    .toEqual("end_token1");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            return Promise.all([
+                client.getEventTimeline(timelineSet, EVENTS[0].event_id,
+                ).then(function(tl0) {
+                    tl = tl0;
+                    return client.paginateEventTimeline(
+                        tl, {backwards: false, limit: 20});
+                }).then(function(success) {
+                    expect(success).toBeTruthy();
+                    expect(tl.getEvents().length).toEqual(3);
+                    expect(tl.getEvents()[0].event).toEqual(EVENTS[0]);
+                    expect(tl.getEvents()[1].event).toEqual(EVENTS[1]);
+                    expect(tl.getEvents()[2].event).toEqual(EVENTS[2]);
+                    expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toEqual("start_token0");
+                    expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                        .toEqual("end_token1");
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
         });
     });
 
@@ -624,60 +631,66 @@ describe("MatrixClient event timelines", function() {
             });
         });
 
-        it("should work when /send returns before /sync", function(done) {
+        it("should work when /send returns before /sync", function() {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
 
-            client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
-                expect(res.event_id).toEqual(event.event_id);
-                return client.getEventTimeline(timelineSet, event.event_id);
-            }).then(function(tl) {
-                // 2 because the initial sync contained an event
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[1].getContent().body).toEqual("a body");
+            return Promise.all([
+                client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
+                    expect(res.event_id).toEqual(event.event_id);
+                    return client.getEventTimeline(timelineSet, event.event_id);
+                }).then(function(tl) {
+                    // 2 because the initial sync contained an event
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[1].getContent().body).toEqual("a body");
 
-                // now let the sync complete, and check it again
-                return httpBackend.flush("/sync", 1);
-            }).then(function() {
-                return utils.syncPromise(client);
-            }).then(function() {
-                return client.getEventTimeline(timelineSet, event.event_id);
-            }).then(function(tl) {
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[1].event).toEqual(event);
-            }).catch(utils.failTest).done(done);
+                    // now let the sync complete, and check it again
+                    return Promise.all([
+                        httpBackend.flush("/sync", 1),
+                        utils.syncPromise(client),
+                    ]);
+                }).then(function() {
+                    return client.getEventTimeline(timelineSet, event.event_id);
+                }).then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[1].event).toEqual(event);
+                }),
 
-            httpBackend.flush("/send/m.room.message/" + TXN_ID, 1).catch(utils.failTest);
+                httpBackend.flush("/send/m.room.message/" + TXN_ID, 1),
+            ]);
         });
 
-        it("should work when /send returns after /sync", function(done) {
+        it("should work when /send returns after /sync", function() {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
 
-            // initiate the send, and set up checks to be done when it completes
-            // - but note that it won't complete until after the /sync does, below.
-            client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
-                console.log("sendTextMessage completed");
-                expect(res.event_id).toEqual(event.event_id);
-                return client.getEventTimeline(timelineSet, event.event_id);
-            }).then(function(tl) {
-                console.log("getEventTimeline completed (2)");
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[1].getContent().body).toEqual("a body");
-            }).catch(utils.failTest).done(done);
+            return Promise.all([
+                // initiate the send, and set up checks to be done when it completes
+                // - but note that it won't complete until after the /sync does, below.
+                client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
+                    console.log("sendTextMessage completed");
+                    expect(res.event_id).toEqual(event.event_id);
+                    return client.getEventTimeline(timelineSet, event.event_id);
+                }).then(function(tl) {
+                    console.log("getEventTimeline completed (2)");
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[1].getContent().body).toEqual("a body");
+                }),
 
-            httpBackend.flush("/sync", 1).then(function() {
-                return utils.syncPromise(client);
-            }).then(function() {
-                return client.getEventTimeline(timelineSet, event.event_id);
-            }).then(function(tl) {
-                console.log("getEventTimeline completed (1)");
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[1].event).toEqual(event);
+                Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
+                    return client.getEventTimeline(timelineSet, event.event_id);
+                }).then(function(tl) {
+                    console.log("getEventTimeline completed (1)");
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[1].event).toEqual(event);
 
-                // now let the send complete.
-                return httpBackend.flush("/send/m.room.message/" + TXN_ID, 1);
-            }).catch(utils.failTest);
+                    // now let the send complete.
+                    return httpBackend.flush("/send/m.room.message/" + TXN_ID, 1);
+                }),
+            ]);
         });
     });
 
@@ -714,7 +727,10 @@ describe("MatrixClient event timelines", function() {
         };
         httpBackend.when("GET", "/sync").respond(200, syncData);
 
-        httpBackend.flush().then(function() {
+        Promise.all([
+            httpBackend.flushAllExpected(),
+            utils.syncPromise(client),
+        ]).then(function() {
             const room = client.getRoom(roomId);
             const tl = room.getLiveTimeline();
             expect(tl.getEvents().length).toEqual(3);
@@ -739,11 +755,14 @@ describe("MatrixClient event timelines", function() {
             };
             httpBackend.when("GET", "/sync").respond(200, sync2);
 
-            return httpBackend.flush();
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client),
+            ]);
         }).then(function() {
             const room = client.getRoom(roomId);
             const tl = room.getLiveTimeline();
             expect(tl.getEvents().length).toEqual(1);
-        }).catch(utils.failTest).done(done);
+        }).nodeify(done);
     });
 });

spec/integ/matrix-client-methods.spec.js

@@ -1,7 +1,7 @@
 "use strict";
 import 'source-map-support/register';
 const sdk = require("../..");
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const publicGlobals = require("../../lib/matrix");
 const Room = publicGlobals.Room;
 const MatrixInMemoryStore = publicGlobals.MatrixInMemoryStore;
@@ -49,13 +49,16 @@ describe("MatrixClient", function() {
             httpBackend.when(
                 "POST", "/_matrix/media/v1/upload",
             ).check(function(req) {
-                expect(req.data).toEqual(buf);
+                expect(req.rawData).toEqual(buf);
                 expect(req.queryParams.filename).toEqual("hi.txt");
-                expect(req.queryParams.access_token).toEqual(accessToken);
+                if (!(req.queryParams.access_token == accessToken ||
+                        req.headers["Authorization"] == "Bearer " + accessToken)) {
+                    expect(true).toBe(false);
+                }
                 expect(req.headers["Content-Type"]).toEqual("text/plain");
                 expect(req.opts.json).toBeFalsy();
                 expect(req.opts.timeout).toBe(undefined);
-            }).respond(200, "content");
+            }).respond(200, "content", true);
 
             const prom = client.uploadContent({
                 stream: buf,
@@ -76,7 +79,7 @@ describe("MatrixClient", function() {
 
                 const uploads = client.getCurrentUploads();
                 expect(uploads.length).toEqual(0);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
 
             httpBackend.flush();
         });
@@ -86,7 +89,7 @@ describe("MatrixClient", function() {
                 "POST", "/_matrix/media/v1/upload",
             ).check(function(req) {
                 expect(req.opts.json).toBeFalsy();
-            }).respond(200, JSON.stringify({ "content_uri": "uri" }));
+            }).respond(200, { "content_uri": "uri" });
 
             client.uploadContent({
                 stream: buf,
@@ -96,22 +99,21 @@ describe("MatrixClient", function() {
                 rawResponse: false,
             }).then(function(response) {
                 expect(response.content_uri).toEqual("uri");
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
 
             httpBackend.flush();
         });
 
         it("should parse errors into a MatrixError", function(done) {
-            // opts.json is false, so request returns unparsed json.
             httpBackend.when(
                 "POST", "/_matrix/media/v1/upload",
             ).check(function(req) {
-                expect(req.data).toEqual(buf);
+                expect(req.rawData).toEqual(buf);
                 expect(req.opts.json).toBeFalsy();
-            }).respond(400, JSON.stringify({
+            }).respond(400, {
                 "errcode": "M_SNAFU",
                 "error": "broken",
-            }));
+            });
 
             client.uploadContent({
                 stream: buf,
@@ -123,7 +125,7 @@ describe("MatrixClient", function() {
                 expect(error.httpStatus).toEqual(400);
                 expect(error.errcode).toEqual("M_SNAFU");
                 expect(error.message).toEqual("broken");
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
 
             httpBackend.flush();
         });
@@ -147,7 +149,7 @@ describe("MatrixClient", function() {
 
                 const uploads = client.getCurrentUploads();
                 expect(uploads.length).toEqual(0);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
 
             const r = client.cancelUpload(prom);
             expect(r).toBe(true);
@@ -299,6 +301,14 @@ describe("MatrixClient", function() {
 
 
     describe("downloadKeys", function() {
+        if (!sdk.CRYPTO_ENABLED) {
+            return;
+        }
+
+        beforeEach(function() {
+            return client.initCrypto();
+        });
+
         it("should do an HTTP request and then store the keys", function(done) {
             const ed25519key = "7wG2lzAqbjcyEkOP7O4gU7ItYcn+chKzh5sT/5r2l78";
             // ed25519key = client.getDeviceEd25519Key();
@@ -375,7 +385,7 @@ describe("MatrixClient", function() {
                     algorithms: ["2"],
                     unsigned: { "ghi": "def" },
                 });
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
 
             httpBackend.flush();
         });
@@ -392,7 +402,7 @@ describe("MatrixClient", function() {
 
             client.deleteDevice(
                 "my_device", auth,
-            ).catch(utils.failTest).done(done);
+            ).nodeify(done);
 
             httpBackend.flush();
         });

spec/integ/matrix-client-opts.spec.js

@@ -2,10 +2,11 @@
 import 'source-map-support/register';
 const sdk = require("../..");
 const MatrixClient = sdk.MatrixClient;
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const utils = require("../test-utils");
 
 import expect from 'expect';
+import Promise from 'bluebird';
 
 describe("MatrixClient opts", function() {
     const baseUrl = "http://localhost.or.something";
@@ -113,9 +114,10 @@ describe("MatrixClient opts", function() {
             httpBackend.flush("/pushrules", 1).then(function() {
                 return httpBackend.flush("/filter", 1);
             }).then(function() {
-                return httpBackend.flush("/sync", 1);
-            }).then(function() {
-                return utils.syncPromise(client);
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]);
             }).done(function() {
                 expect(expectedEventTypes.length).toEqual(
                     0, "Expected to see event types: " + expectedEventTypes,

spec/integ/matrix-client-retrying.spec.js

@@ -1,7 +1,9 @@
 "use strict";
 import 'source-map-support/register';
+import Promise from 'bluebird';
+
 const sdk = require("../..");
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const utils = require("../test-utils");
 const EventStatus = sdk.EventStatus;
 
@@ -52,15 +54,20 @@ describe("MatrixClient retrying", function() {
 
     });
 
-    it("should mark events as EventStatus.CANCELLED when cancelled", function(done) {
+    it("should mark events as EventStatus.CANCELLED when cancelled", function() {
         // send a couple of events; the second will be queued
-        client.sendMessage(roomId, "m1").then(function(ev) {
-            expect(ev).toEqual(ev1);
-        });
-        client.sendMessage(roomId, "m2").then(function(ev) {
-            expect(ev).toEqual(ev2);
+        const p1 = client.sendMessage(roomId, "m1").then(function(ev) {
+            // we expect the first message to fail
+            throw new Error('Message 1 unexpectedly sent successfully');
+        }, (e) => {
+            // this is expected
         });
 
+        // XXX: it turns out that the promise returned by this message
+        // never gets resolved.
+        // https://github.com/matrix-org/matrix-js-sdk/issues/496
+        client.sendMessage(roomId, "m2");
+
         // both events should be in the timeline at this point
         const tl = room.getLiveTimeline().getEvents();
         expect(tl.length).toEqual(2);
@@ -86,7 +93,14 @@ describe("MatrixClient retrying", function() {
             }).toThrow();
         }).respond(400); // fail the first message
 
-        httpBackend.flush().then(function() {
+        // wait for the localecho of ev1 to be updated
+        const p3 = new Promise((resolve, reject) => {
+            room.on("Room.localEchoUpdated", (ev0) => {
+                if(ev0 === ev1) {
+                    resolve();
+                }
+            });
+        }).then(function() {
             expect(ev1.status).toEqual(EventStatus.NOT_SENT);
             expect(tl.length).toEqual(1);
 
@@ -94,7 +108,13 @@ describe("MatrixClient retrying", function() {
             client.cancelPendingEvent(ev1);
             expect(ev1.status).toEqual(EventStatus.CANCELLED);
             expect(tl.length).toEqual(0);
-        }).catch(utils.failTest).done(done);
+        });
+
+        return Promise.all([
+            p1,
+            p3,
+            httpBackend.flushAllExpected(),
+        ]);
     });
 
     describe("resending", function() {

spec/integ/matrix-client-room-timeline.spec.js

@@ -2,9 +2,10 @@
 import 'source-map-support/register';
 const sdk = require("../..");
 const EventStatus = sdk.EventStatus;
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const utils = require("../test-utils");
 
+import Promise from 'bluebird';
 import expect from 'expect';
 
 describe("MatrixClient room timelines", function() {
@@ -251,11 +252,14 @@ describe("MatrixClient room timelines", function() {
                 client.scrollback(room).done(function() {
                     expect(room.timeline.length).toEqual(1);
                     expect(room.oldState.paginationToken).toBe(null);
-                    done();
+
+                    // still have a sync to flush
+                    httpBackend.flush("/sync", 1).then(() => {
+                        done();
+                    });
                 });
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1);
             });
             httpBackend.flush("/sync", 1);
         });
@@ -317,11 +321,14 @@ describe("MatrixClient room timelines", function() {
                     expect(oldMsg.sender.name).toEqual("Old Alice");
                     const newMsg = room.timeline[3];
                     expect(newMsg.sender.name).toEqual(userName);
-                    done();
+
+                    // still have a sync to flush
+                    httpBackend.flush("/sync", 1).then(() => {
+                        done();
+                    });
                 });
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1);
             });
             httpBackend.flush("/sync", 1);
         });
@@ -348,11 +355,14 @@ describe("MatrixClient room timelines", function() {
                     expect(room.timeline.length).toEqual(3);
                     expect(room.timeline[0].event).toEqual(sbEvents[1]);
                     expect(room.timeline[1].event).toEqual(sbEvents[0]);
-                    done();
+
+                    // still have a sync to flush
+                    httpBackend.flush("/sync", 1).then(() => {
+                        done();
+                    });
                 });
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1);
             });
             httpBackend.flush("/sync", 1);
         });
@@ -376,9 +386,11 @@ describe("MatrixClient room timelines", function() {
                     expect(room.oldState.paginationToken).toEqual(sbEndTok);
                 });
 
-                httpBackend.flush("/sync", 1);
                 httpBackend.flush("/messages", 1).done(function() {
-                    done();
+                    // still have a sync to flush
+                    httpBackend.flush("/sync", 1).then(() => {
+                        done();
+                    });
                 });
             });
             httpBackend.flush("/sync", 1);
@@ -386,17 +398,17 @@ describe("MatrixClient room timelines", function() {
     });
 
     describe("new events", function() {
-        it("should be added to the right place in the timeline", function(done) {
+        it("should be added to the right place in the timeline", function() {
             const eventData = [
                 utils.mkMessage({user: userId, room: roomId}),
                 utils.mkMessage({user: userId, room: roomId}),
             ];
             setNextSyncData(eventData);
 
-            client.on("sync", function(state) {
-                if (state !== "PREPARED") {
-                    return;
-                }
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
                 const room = client.getRoom(roomId);
 
                 let index = 0;
@@ -408,9 +420,10 @@ describe("MatrixClient room timelines", function() {
                 });
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1).then(function() {
-                    return utils.syncPromise(client);
-                }).then(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                     expect(index).toEqual(2);
                     expect(room.timeline.length).toEqual(3);
                     expect(room.timeline[2].event).toEqual(
@@ -419,12 +432,11 @@ describe("MatrixClient room timelines", function() {
                     expect(room.timeline[1].event).toEqual(
                         eventData[0],
                     );
-                }).catch(utils.failTest).done(done);
+                });
             });
-            httpBackend.flush("/sync", 1);
         });
 
-        it("should set the right event.sender values", function(done) {
+        it("should set the right event.sender values", function() {
             const eventData = [
                 utils.mkMessage({user: userId, room: roomId}),
                 utils.mkMembership({
@@ -435,24 +447,24 @@ describe("MatrixClient room timelines", function() {
             eventData[1].__prev_event = USER_MEMBERSHIP_EVENT;
             setNextSyncData(eventData);
 
-            client.on("sync", function(state) {
-                if (state !== "PREPARED") {
-                    return;
-                }
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
                 const room = client.getRoom(roomId);
-                httpBackend.flush("/sync", 1).then(function() {
-                    return utils.syncPromise(client);
-                }).then(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                     const preNameEvent = room.timeline[room.timeline.length - 3];
                     const postNameEvent = room.timeline[room.timeline.length - 1];
                     expect(preNameEvent.sender.name).toEqual(userName);
                     expect(postNameEvent.sender.name).toEqual("New Name");
-                }).catch(utils.failTest).done(done);
+                });
             });
-            httpBackend.flush("/sync", 1);
         });
 
-        it("should set the right room.name", function(done) {
+        it("should set the right room.name", function() {
             const secondRoomNameEvent = utils.mkEvent({
                 user: userId, room: roomId, type: "m.room.name", content: {
                     name: "Room 2",
@@ -461,19 +473,20 @@ describe("MatrixClient room timelines", function() {
             secondRoomNameEvent.__prev_event = ROOM_NAME_EVENT;
             setNextSyncData([secondRoomNameEvent]);
 
-            client.on("sync", function(state) {
-                if (state !== "PREPARED") {
-                    return;
-                }
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
                 const room = client.getRoom(roomId);
                 let nameEmitCount = 0;
                 client.on("Room.name", function(rm) {
                     nameEmitCount += 1;
                 });
 
-                httpBackend.flush("/sync", 1).then(() => {
-                    return utils.syncPromise(client);
-                }).done(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                     expect(nameEmitCount).toEqual(1);
                     expect(room.name).toEqual("Room 2");
                     // do another round
@@ -485,19 +498,18 @@ describe("MatrixClient room timelines", function() {
                     thirdRoomNameEvent.__prev_event = secondRoomNameEvent;
                     setNextSyncData([thirdRoomNameEvent]);
                     httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
-                    httpBackend.flush("/sync", 1).then(() => {
-                        return utils.syncPromise(client);
-                    }).done(function() {
-                        expect(nameEmitCount).toEqual(2);
-                        expect(room.name).toEqual("Room 3");
-                        done();
-                    });
+                    return Promise.all([
+                        httpBackend.flush("/sync", 1),
+                        utils.syncPromise(client),
+                    ]);
+                }).then(function() {
+                    expect(nameEmitCount).toEqual(2);
+                    expect(room.name).toEqual("Room 3");
                 });
             });
-            httpBackend.flush("/sync", 1);
         });
 
-        it("should set the right room members", function(done) {
+        it("should set the right room members", function() {
             const userC = "@cee:bar";
             const userD = "@dee:bar";
             const eventData = [
@@ -512,14 +524,15 @@ describe("MatrixClient room timelines", function() {
             eventData[1].__prev_event = null;
             setNextSyncData(eventData);
 
-            client.on("sync", function(state) {
-                if (state !== "PREPARED") {
-                    return;
-                }
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
                 const room = client.getRoom(roomId);
-                httpBackend.flush("/sync", 1).then(function() {
-                    return utils.syncPromise(client);
-                }).then(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                     expect(room.currentState.getMembers().length).toEqual(4);
                     expect(room.currentState.getMember(userC).name).toEqual("C");
                     expect(room.currentState.getMember(userC).membership).toEqual(
@@ -529,30 +542,30 @@ describe("MatrixClient room timelines", function() {
                     expect(room.currentState.getMember(userD).membership).toEqual(
                         "invite",
                     );
-                }).catch(utils.failTest).done(done);
+                });
             });
-            httpBackend.flush("/sync", 1);
         });
     });
 
     describe("gappy sync", function() {
-        it("should copy the last known state to the new timeline", function(done) {
+        it("should copy the last known state to the new timeline", function() {
             const eventData = [
                 utils.mkMessage({user: userId, room: roomId}),
             ];
             setNextSyncData(eventData);
             NEXT_SYNC_DATA.rooms.join[roomId].timeline.limited = true;
 
-            client.on("sync", function(state) {
-                if (state !== "PREPARED") {
-                    return;
-                }
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
                 const room = client.getRoom(roomId);
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1).then(function() {
-                    return utils.syncPromise(client);
-                }).done(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                     expect(room.timeline.length).toEqual(1);
                     expect(room.timeline[0].event).toEqual(eventData[0]);
                     expect(room.currentState.getMembers().length).toEqual(2);
@@ -564,23 +577,21 @@ describe("MatrixClient room timelines", function() {
                     expect(room.currentState.getMember(otherUserId).membership).toEqual(
                         "join",
                     );
-                    done();
                 });
             });
-            httpBackend.flush("/sync", 1);
         });
 
-        it("should emit a 'Room.timelineReset' event", function(done) {
+        it("should emit a 'Room.timelineReset' event", function() {
             const eventData = [
                 utils.mkMessage({user: userId, room: roomId}),
             ];
             setNextSyncData(eventData);
             NEXT_SYNC_DATA.rooms.join[roomId].timeline.limited = true;
 
-            client.on("sync", function(state) {
-                if (state !== "PREPARED") {
-                    return;
-                }
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
                 const room = client.getRoom(roomId);
 
                 let emitCount = 0;
@@ -590,14 +601,13 @@ describe("MatrixClient room timelines", function() {
                 });
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1).then(function() {
-                    return utils.syncPromise(client);
-                }).done(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                     expect(emitCount).toEqual(1);
-                    done();
                 });
             });
-            httpBackend.flush("/sync", 1);
         });
     });
 });

spec/integ/matrix-client-syncing.spec.js

@@ -1,12 +1,13 @@
 "use strict";
 import 'source-map-support/register';
 const sdk = require("../..");
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const utils = require("../test-utils");
 const MatrixEvent = sdk.MatrixEvent;
 const EventTimeline = sdk.EventTimeline;
 
 import expect from 'expect';
+import Promise from 'bluebird';
 
 describe("MatrixClient syncing", function() {
     const baseUrl = "http://localhost.or.something";
@@ -51,7 +52,7 @@ describe("MatrixClient syncing", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            httpBackend.flushAllExpected().done(function() {
                 done();
             });
         });
@@ -65,7 +66,7 @@ describe("MatrixClient syncing", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            httpBackend.flushAllExpected().done(function() {
                 done();
             });
         });
@@ -113,7 +114,7 @@ describe("MatrixClient syncing", function() {
             };
         });
 
-        it("should resolve incoming invites from /sync", function(done) {
+        it("should resolve incoming invites from /sync", function() {
             syncData.rooms.join[roomOne].state.events.push(
                 utils.mkMembership({
                     room: roomOne, mship: "invite", user: userC,
@@ -132,17 +133,20 @@ describe("MatrixClient syncing", function() {
                 resolveInvitesToProfiles: true,
             });
 
-            httpBackend.flush().done(function() {
+
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                 const member = client.getRoom(roomOne).getMember(userC);
                 expect(member.name).toEqual("The Boss");
                 expect(
                     member.getAvatarUrl("home.server.url", null, null, null, false),
                 ).toBeTruthy();
-                done();
             });
         });
 
-        it("should use cached values from m.presence wherever possible", function(done) {
+        it("should use cached values from m.presence wherever possible", function() {
             syncData.presence.events = [
                 utils.mkPresence({
                     user: userC, presence: "online", name: "The Ghost",
@@ -160,14 +164,16 @@ describe("MatrixClient syncing", function() {
                 resolveInvitesToProfiles: true,
             });
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                 const member = client.getRoom(roomOne).getMember(userC);
                 expect(member.name).toEqual("The Ghost");
-                done();
             });
         });
 
-        it("should result in events on the room member firing", function(done) {
+        it("should result in events on the room member firing", function() {
             syncData.presence.events = [
                 utils.mkPresence({
                     user: userC, presence: "online", name: "The Ghost",
@@ -192,13 +198,15 @@ describe("MatrixClient syncing", function() {
                 resolveInvitesToProfiles: true,
             });
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                 expect(latestFiredName).toEqual("The Ghost");
-                done();
             });
         });
 
-        it("should no-op if resolveInvitesToProfiles is not set", function(done) {
+        it("should no-op if resolveInvitesToProfiles is not set", function() {
             syncData.rooms.join[roomOne].state.events.push(
                 utils.mkMembership({
                     room: roomOne, mship: "invite", user: userC,
@@ -209,13 +217,15 @@ describe("MatrixClient syncing", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                 const member = client.getRoom(roomOne).getMember(userC);
                 expect(member.name).toEqual(userC);
                 expect(
                     member.getAvatarUrl("home.server.url", null, null, null, false),
                 ).toBe(null);
-                done();
             });
         });
     });
@@ -236,15 +246,17 @@ describe("MatrixClient syncing", function() {
         };
 
         it("should create users for presence events from /sync",
-        function(done) {
+        function() {
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                 expect(client.getUser(userA).presence).toEqual("online");
                 expect(client.getUser(userB).presence).toEqual("unavailable");
-                done();
             });
         });
     });
@@ -355,57 +367,66 @@ describe("MatrixClient syncing", function() {
             },
         };
 
-        it("should continually recalculate the right room name.", function(done) {
+        it("should continually recalculate the right room name.", function() {
             httpBackend.when("GET", "/sync").respond(200, syncData);
             httpBackend.when("GET", "/sync").respond(200, nextSyncData);
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
                 const room = client.getRoom(roomOne);
                 // should have clobbered the name to the one from /events
                 expect(room.name).toEqual(
                     nextSyncData.rooms.join[roomOne].state.events[0].content.name,
                 );
-                done();
             });
         });
 
-        it("should store the right events in the timeline.", function(done) {
+        it("should store the right events in the timeline.", function() {
             httpBackend.when("GET", "/sync").respond(200, syncData);
             httpBackend.when("GET", "/sync").respond(200, nextSyncData);
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
                 const room = client.getRoom(roomTwo);
                 // should have added the message from /events
                 expect(room.timeline.length).toEqual(2);
                 expect(room.timeline[1].getContent().body).toEqual(msgText);
-                done();
             });
         });
 
-        it("should set the right room name.", function(done) {
+        it("should set the right room name.", function() {
             httpBackend.when("GET", "/sync").respond(200, syncData);
             httpBackend.when("GET", "/sync").respond(200, nextSyncData);
 
             client.startClient();
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
                 const room = client.getRoom(roomTwo);
                 // should use the display name of the other person.
                 expect(room.name).toEqual(otherDisplayName);
-                done();
             });
         });
 
-        it("should set the right user's typing flag.", function(done) {
+        it("should set the right user's typing flag.", function() {
             httpBackend.when("GET", "/sync").respond(200, syncData);
             httpBackend.when("GET", "/sync").respond(200, nextSyncData);
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
                 const room = client.getRoom(roomTwo);
                 let member = room.getMember(otherUserId);
                 expect(member).toBeTruthy();
@@ -413,7 +434,6 @@ describe("MatrixClient syncing", function() {
                 member = room.getMember(selfUserId);
                 expect(member).toBeTruthy();
                 expect(member.typing).toEqual(false);
-                done();
             });
         });
 
@@ -448,10 +468,13 @@ describe("MatrixClient syncing", function() {
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
             client.startClient();
-            httpBackend.flush();
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]);
         });
 
-        it("should set the back-pagination token on new rooms", function(done) {
+        it("should set the back-pagination token on new rooms", function() {
             const syncData = {
                 next_batch: "batch_token",
                 rooms: {
@@ -471,16 +494,19 @@ describe("MatrixClient syncing", function() {
 
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
-            httpBackend.flush().then(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                 const room = client.getRoom(roomTwo);
+                expect(room).toExist();
                 const tok = room.getLiveTimeline()
                     .getPaginationToken(EventTimeline.BACKWARDS);
                 expect(tok).toEqual("roomtwotok");
-                done();
-            }).catch(utils.failTest).done();
+            });
         });
 
-        it("should set the back-pagination token on gappy syncs", function(done) {
+        it("should set the back-pagination token on gappy syncs", function() {
             const syncData = {
                 next_batch: "batch_token",
                 rooms: {
@@ -511,13 +537,15 @@ describe("MatrixClient syncing", function() {
                 expect(tok).toEqual("newerTok");
             });
 
-            httpBackend.flush().then(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                 const room = client.getRoom(roomOne);
                 const tl = room.getLiveTimeline();
                 expect(tl.getEvents().length).toEqual(1);
                 expect(resetCallCount).toEqual(1);
-                done();
-            }).catch(utils.failTest).done();
+            });
         });
     });
 
@@ -570,7 +598,7 @@ describe("MatrixClient syncing", function() {
             };
         });
 
-        it("should sync receipts from /sync.", function(done) {
+        it("should sync receipts from /sync.", function() {
             const ackEvent = syncData.rooms.join[roomOne].timeline.events[0];
             const receipt = {};
             receipt[ackEvent.event_id] = {
@@ -588,7 +616,10 @@ describe("MatrixClient syncing", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                 const room = client.getRoom(roomOne);
                 expect(room.getReceiptsForEvent(new MatrixEvent(ackEvent))).toEqual([{
                     type: "m.read",
@@ -597,7 +628,6 @@ describe("MatrixClient syncing", function() {
                         ts: 176592842636,
                     },
                 }]);
-                done();
             });
         });
     });
@@ -617,7 +647,7 @@ describe("MatrixClient syncing", function() {
         beforeEach(function(done) {
             client.startClient();
 
-            httpBackend.flush().then(function() {
+            httpBackend.flushAllExpected().then(function() {
                 // the /sync call from syncLeftRooms ends up in the request
                 // queue behind the call from the running client; add a response
                 // to flush the client's one out.
@@ -627,29 +657,34 @@ describe("MatrixClient syncing", function() {
             });
         });
 
-        it("should create and use an appropriate filter", function(done) {
+        it("should create and use an appropriate filter", function() {
             httpBackend.when("POST", "/filter").check(function(req) {
                 expect(req.data).toEqual({
                     room: { timeline: {limit: 1},
                             include_leave: true }});
             }).respond(200, { filter_id: "another_id" });
 
+            const defer = Promise.defer();
+
             httpBackend.when("GET", "/sync").check(function(req) {
                 expect(req.queryParams.filter).toEqual("another_id");
-                done();
+                defer.resolve();
             }).respond(200, {});
 
             client.syncLeftRooms();
 
             // first flush the filter request; this will make syncLeftRooms
             // make its /sync call
-            httpBackend.flush("/filter").then(function() {
-                // flush the syncs
-                return httpBackend.flush();
-            }).catch(utils.failTest);
+            return Promise.all([
+                httpBackend.flush("/filter").then(function() {
+                    // flush the syncs
+                    return httpBackend.flushAllExpected();
+                }),
+                defer.promise,
+            ]);
         });
 
-        it("should set the back-pagination token on left rooms", function(done) {
+        it("should set the back-pagination token on left rooms", function() {
             const syncData = {
                 next_batch: "batch_token",
                 rooms: {
@@ -674,20 +709,31 @@ describe("MatrixClient syncing", function() {
 
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
-            client.syncLeftRooms().then(function() {
-                const room = client.getRoom(roomTwo);
-                const tok = room.getLiveTimeline().getPaginationToken(
-                    EventTimeline.BACKWARDS);
+            return Promise.all([
+                client.syncLeftRooms().then(function() {
+                    const room = client.getRoom(roomTwo);
+                    const tok = room.getLiveTimeline().getPaginationToken(
+                        EventTimeline.BACKWARDS);
 
-                expect(tok).toEqual("pagTok");
-                done();
-            }).catch(utils.failTest).done();
+                    expect(tok).toEqual("pagTok");
+                }),
 
-            // first flush the filter request; this will make syncLeftRooms
-            // make its /sync call
-            httpBackend.flush("/filter").then(function() {
-                return httpBackend.flush();
-            }).catch(utils.failTest);
+                // first flush the filter request; this will make syncLeftRooms
+                // make its /sync call
+                httpBackend.flush("/filter").then(function() {
+                    return httpBackend.flushAllExpected();
+                }),
+            ]);
         });
     });
+
+    /**
+     * waits for the MatrixClient to emit one or more 'sync' events.
+     *
+     * @param {Number?} numSyncs number of syncs to wait for
+     * @returns {Promise} promise which resolves after the sync events have happened
+     */
+    function awaitSyncEvent(numSyncs) {
+        return utils.syncPromise(client, numSyncs);
+    }
 });

spec/integ/megolm-integ.spec.js

@@ -16,18 +16,10 @@ limitations under the License.
 
 "use strict";
 
-import 'source-map-support/register';
-
-let Olm = null;
-try {
-    Olm = require('olm');
-} catch (e) {}
-
 const anotherjson = require('another-json');
-const q = require('q');
+import Promise from 'bluebird';
 import expect from 'expect';
 
-const sdk = require('../..');
 const utils = require('../../lib/utils');
 const testUtils = require('../test-utils');
 const TestClient = require('../TestClient').default;
@@ -46,7 +38,7 @@ function createOlmSession(olmAccount, recipientTestClient) {
         const otkId = utils.keys(keys)[0];
         const otk = keys[otkId];
 
-        const session = new Olm.Session();
+        const session = new global.Olm.Session();
         session.create_outbound(
             olmAccount, recipientTestClient.getDeviceKey(), otk.key,
         );
@@ -127,6 +119,7 @@ function encryptMegolmEvent(opts) {
     }
 
     return {
+        event_id: 'test_megolm_event',
         content: {
             algorithm: "m.megolm.v1.aes-sha2",
             ciphertext: opts.groupSession.encrypt(JSON.stringify(plaintext)),
@@ -209,9 +202,11 @@ function getSyncResponse(roomMembers) {
 
 
 describe("megolm", function() {
-    if (!sdk.CRYPTO_ENABLED) {
+    if (!global.Olm) {
+        console.warn('not running megolm tests: Olm not present');
         return;
     }
+    const Olm = global.Olm;
 
     let testOlmAccount;
     let testSenderKey;
@@ -286,12 +281,13 @@ describe("megolm", function() {
         return claimResponse;
     }
 
-    beforeEach(function() {
+    beforeEach(async function() {
         testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
 
         aliceTestClient = new TestClient(
             "@alice:localhost", "xzcvb", "akjgkrgjs",
         );
+        await aliceTestClient.client.initCrypto();
 
         testOlmAccount = new Olm.Account();
         testOlmAccount.create();
@@ -303,7 +299,7 @@ describe("megolm", function() {
         aliceTestClient.stop();
     });
 
-    it("Alice receives a megolm message", function(done) {
+    it("Alice receives a megolm message", function() {
         return aliceTestClient.start().then(() => {
             return createOlmSession(testOlmAccount, aliceTestClient);
         }).then((p2pSession) => {
@@ -343,17 +339,18 @@ describe("megolm", function() {
             };
 
             aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush("/sync", 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
         }).then(function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
             const event = room.getLiveTimeline().getEvents()[0];
+            expect(event.isEncrypted()).toBe(true);
+            return testUtils.awaitDecryption(event);
+        }).then((event) => {
             expect(event.getContent().body).toEqual('42');
-        }).nodeify(done);
+        });
     });
 
-    it("Alice receives a megolm message before the session keys", function(done) {
+    it("Alice receives a megolm message before the session keys", function() {
         // https://github.com/vector-im/riot-web/issues/2273
         let roomKeyEncrypted;
 
@@ -393,9 +390,7 @@ describe("megolm", function() {
             };
 
             aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush("/sync", 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
         }).then(function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
             const event = room.getLiveTimeline().getEvents()[0];
@@ -410,17 +405,27 @@ describe("megolm", function() {
             };
 
             aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush("/sync", 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
         }).then(function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
             const event = room.getLiveTimeline().getEvents()[0];
+
+            if (event.getContent().msgtype != 'm.bad.encrypted') {
+                return event;
+            }
+
+            return new Promise((resolve, reject) => {
+                event.once('Event.decrypted', (ev) => {
+                    console.log(`${Date.now()} event ${event.getId()} now decrypted`);
+                    resolve(ev);
+                });
+            });
+        }).then((event) => {
             expect(event.getContent().body).toEqual('42');
-        }).nodeify(done);
+        });
     });
 
-    it("Alice gets a second room_key message", function(done) {
+    it("Alice gets a second room_key message", function() {
         return aliceTestClient.start().then(() => {
             return createOlmSession(testOlmAccount, aliceTestClient);
         }).then((p2pSession) => {
@@ -480,17 +485,18 @@ describe("megolm", function() {
             };
             aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse2);
 
-            return aliceTestClient.httpBackend.flush("/sync", 2);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            // flush both syncs
+            return aliceTestClient.flushSync().then(() => {
+                return aliceTestClient.flushSync();
+            });
         }).then(function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
             const event = room.getLiveTimeline().getEvents()[0];
             expect(event.getContent().body).toEqual('42');
-        }).nodeify(done);
+        });
     });
 
-    it('Alice sends a megolm message', function(done) {
+    it('Alice sends a megolm message', function() {
         let p2pSession;
 
         return aliceTestClient.start().then(() => {
@@ -510,22 +516,20 @@ describe("megolm", function() {
             syncResponse.to_device = { events: [olmEvent] };
 
             aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush('/sync', 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
         }).then(function() {
             // start out with the device unknown - the send should be rejected.
             aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
                 200, getTestKeysQueryResponse('@bob:xyz'),
             );
 
-            return q.all([
+            return Promise.all([
                 aliceTestClient.client.sendTextMessage(ROOM_ID, 'test').then(() => {
                     throw new Error("sendTextMessage failed on an unknown device");
                 }, (e) => {
                     expect(e.name).toEqual("UnknownDeviceError");
                 }),
-                aliceTestClient.httpBackend.flush(),
+                aliceTestClient.httpBackend.flushAllExpected(),
             ]);
         }).then(function() {
             // mark the device as known, and resend.
@@ -565,52 +569,18 @@ describe("megolm", function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
             const pendingMsg = room.getPendingEvents()[0];
 
-            return q.all([
+            return Promise.all([
                 aliceTestClient.client.resendEvent(pendingMsg, room),
-                aliceTestClient.httpBackend.flush(),
+
+                // the crypto stuff can take a while, so give the requests a whole second.
+                aliceTestClient.httpBackend.flushAllExpected({
+                    timeout: 1000,
+                }),
             ]);
-        }).nodeify(done);
+        });
     });
 
-    it("Alice shouldn't do a second /query for non-e2e-capable devices", function(done) {
-        return aliceTestClient.start().then(function() {
-            const syncResponse = getSyncResponse(['@bob:xyz']);
-            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
-
-            return aliceTestClient.httpBackend.flush('/sync', 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
-        }).then(function() {
-            console.log("Forcing alice to download our device keys");
-
-            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(200, {
-                device_keys: {
-                    '@bob:xyz': {},
-                },
-            });
-
-            return q.all([
-                aliceTestClient.client.downloadKeys(['@bob:xyz']),
-                aliceTestClient.httpBackend.flush('/keys/query', 1),
-            ]);
-        }).then(function() {
-            console.log("Telling alice to send a megolm message");
-
-            aliceTestClient.httpBackend.when(
-                'PUT', '/send/',
-            ).respond(200, {
-                    event_id: '$event_id',
-            });
-
-            return q.all([
-                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
-                aliceTestClient.httpBackend.flush(),
-            ]);
-        }).nodeify(done);
-    });
-
-
-    it("We shouldn't attempt to send to blocked devices", function(done) {
+    it("We shouldn't attempt to send to blocked devices", function() {
         return aliceTestClient.start().then(() => {
             // establish an olm session with alice
             return createOlmSession(testOlmAccount, aliceTestClient);
@@ -626,9 +596,7 @@ describe("megolm", function() {
             syncResponse.to_device = { events: [olmEvent] };
             aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
 
-            return aliceTestClient.httpBackend.flush('/sync', 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
         }).then(function() {
             console.log('Forcing alice to download our device keys');
 
@@ -636,7 +604,7 @@ describe("megolm", function() {
                 200, getTestKeysQueryResponse('@bob:xyz'),
             );
 
-            return q.all([
+            return Promise.all([
                 aliceTestClient.client.downloadKeys(['@bob:xyz']),
                 aliceTestClient.httpBackend.flush('/keys/query', 1),
             ]);
@@ -651,14 +619,18 @@ describe("megolm", function() {
                 event_id: '$event_id',
             });
 
-            return q.all([
+            return Promise.all([
                 aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
-                aliceTestClient.httpBackend.flush(),
+
+                // the crypto stuff can take a while, so give the requests a whole second.
+                aliceTestClient.httpBackend.flushAllExpected({
+                    timeout: 1000,
+                }),
             ]);
-        }).nodeify(done);
+        });
     });
 
-    it("We should start a new megolm session when a device is blocked", function(done) {
+    it("We should start a new megolm session when a device is blocked", function() {
         let p2pSession;
         let megolmSessionId;
 
@@ -679,9 +651,7 @@ describe("megolm", function() {
             syncResponse.to_device = { events: [olmEvent] };
             aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
 
-            return aliceTestClient.httpBackend.flush('/sync', 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
         }).then(function() {
             console.log("Fetching bob's devices and marking known");
 
@@ -689,9 +659,9 @@ describe("megolm", function() {
                 200, getTestKeysQueryResponse('@bob:xyz'),
             );
 
-            return q.all([
+            return Promise.all([
                 aliceTestClient.client.downloadKeys(['@bob:xyz']),
-                aliceTestClient.httpBackend.flush(),
+                aliceTestClient.httpBackend.flushAllExpected(),
             ]).then((keys) => {
                 aliceTestClient.client.setDeviceKnown('@bob:xyz', 'DEVICE_ID');
             });
@@ -722,9 +692,13 @@ describe("megolm", function() {
                 };
             });
 
-            return q.all([
+            return Promise.all([
                 aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
-                aliceTestClient.httpBackend.flush(),
+
+                // the crypto stuff can take a while, so give the requests a whole second.
+                aliceTestClient.httpBackend.flushAllExpected({
+                    timeout: 1000,
+                }),
             ]);
         }).then(function() {
             console.log('Telling alice to block our device');
@@ -741,15 +715,15 @@ describe("megolm", function() {
                 };
             });
 
-            return q.all([
+            return Promise.all([
                 aliceTestClient.client.sendTextMessage(ROOM_ID, 'test2'),
-                aliceTestClient.httpBackend.flush(),
+                aliceTestClient.httpBackend.flushAllExpected(),
             ]);
-        }).nodeify(done);
+        });
     });
 
     // https://github.com/vector-im/riot-web/issues/2676
-    it("Alice should send to her other devices", function(done) {
+    it("Alice should send to her other devices", function() {
         // for this test, we make the testOlmAccount be another of Alice's devices.
         // it ought to get included in messages Alice sends.
 
@@ -791,20 +765,17 @@ describe("megolm", function() {
                 getTestKeysQueryResponse(aliceTestClient.userId),
             );
 
-            return aliceTestClient.httpBackend.flush();
+            return aliceTestClient.httpBackend.flushAllExpected();
         }).then(function() {
             // start out with the device unknown - the send should be rejected.
-            return q.all([
-                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test').then(() => {
-                    throw new Error("sendTextMessage failed on an unknown device");
-                }, (e) => {
-                    expect(e.name).toEqual("UnknownDeviceError");
-                    expect(Object.keys(e.devices)).toEqual([aliceTestClient.userId]);
-                    expect(Object.keys(e.devices[aliceTestClient.userId])).
-                        toEqual(['DEVICE_ID']);
-                }),
-                aliceTestClient.httpBackend.flush(),
-            ]);
+            return aliceTestClient.client.sendTextMessage(ROOM_ID, 'test').then(() => {
+                throw new Error("sendTextMessage failed on an unknown device");
+            }, (e) => {
+                expect(e.name).toEqual("UnknownDeviceError");
+                expect(Object.keys(e.devices)).toEqual([aliceTestClient.userId]);
+                expect(Object.keys(e.devices[aliceTestClient.userId])).
+                    toEqual(['DEVICE_ID']);
+            });
         }).then(function() {
             // mark the device as known, and resend.
             aliceTestClient.client.setDeviceKnown(aliceTestClient.userId, 'DEVICE_ID');
@@ -846,55 +817,26 @@ describe("megolm", function() {
                 };
             });
 
-            return q.all([
+            return Promise.all([
                 aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
-                aliceTestClient.httpBackend.flush(),
+
+                // the crypto stuff can take a while, so give the requests a whole second.
+                aliceTestClient.httpBackend.flushAllExpected({
+                    timeout: 1000,
+                }),
             ]);
         }).then(function() {
             expect(decrypted.type).toEqual('m.room.message');
             expect(decrypted.content.body).toEqual('test');
-        }).nodeify(done);
+        });
     });
 
 
     it('Alice should wait for device list to complete when sending a megolm message',
-    function(done) {
-        let p2pSession;
-        let inboundGroupSession;
-
+    function() {
         let downloadPromise;
         let sendPromise;
 
-        aliceTestClient.httpBackend.when(
-            'PUT', '/sendToDevice/m.room.encrypted/',
-        ).respond(200, function(path, content) {
-            const m = content.messages['@bob:xyz'].DEVICE_ID;
-            const ct = m.ciphertext[testSenderKey];
-            const decrypted = JSON.parse(p2pSession.decrypt(ct.type, ct.body));
-
-            expect(decrypted.type).toEqual('m.room_key');
-            inboundGroupSession = new Olm.InboundGroupSession();
-            inboundGroupSession.create(decrypted.content.session_key);
-            return {};
-        });
-
-        aliceTestClient.httpBackend.when(
-            'PUT', '/send/',
-        ).respond(200, function(path, content) {
-            const ct = content.ciphertext;
-            const r = inboundGroupSession.decrypt(ct);
-            console.log('Decrypted received megolm message', r);
-
-            expect(r.message_index).toEqual(0);
-            const decrypted = JSON.parse(r.plaintext);
-            expect(decrypted.type).toEqual('m.room.message');
-            expect(decrypted.content.body).toEqual('test');
-
-            return {
-                event_id: '$event_id',
-            };
-        });
-
         return aliceTestClient.start().then(() => {
             // establish an olm session with alice
             return createOlmSession(testOlmAccount, aliceTestClient);
@@ -910,13 +852,10 @@ describe("megolm", function() {
             syncResponse.to_device = { events: [olmEvent] };
 
             aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush('/sync', 1);
+            return aliceTestClient.flushSync();
         }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
-        }).then(function() {
-            console.log('Forcing alice to download our device keys');
-
             // this will block
+            console.log('Forcing alice to download our device keys');
             downloadPromise = aliceTestClient.client.downloadKeys(['@bob:xyz']);
 
             // so will this.
@@ -931,137 +870,14 @@ describe("megolm", function() {
                 200, getTestKeysQueryResponse('@bob:xyz'),
             );
 
-            return aliceTestClient.httpBackend.flush();
+            return aliceTestClient.httpBackend.flushAllExpected();
         }).then(function() {
-            return q.all([downloadPromise, sendPromise]);
-        }).nodeify(done);
+            return Promise.all([downloadPromise, sendPromise]);
+        });
     });
 
 
-    it("We should not get confused by out-of-order device query responses",
-       () => {
-           // https://github.com/vector-im/riot-web/issues/3126
-           return aliceTestClient.start().then(() => {
-               aliceTestClient.httpBackend.when('GET', '/sync').respond(
-                   200, getSyncResponse(['@bob:xyz', '@chris:abc']));
-               return aliceTestClient.httpBackend.flush('/sync', 1);
-           }).then(() => {
-               return testUtils.syncPromise(aliceTestClient.client);
-           }).then(() => {
-               // to make sure the initial device queries are flushed out, we
-               // attempt to send a message.
-
-               aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
-                   200, {
-                       device_keys: {
-                           '@bob:xyz': {},
-                           '@chris:abc': {},
-                       },
-                   },
-               );
-
-               aliceTestClient.httpBackend.when('PUT', '/send/').respond(
-                   200, {event_id: '$event1'});
-
-               return q.all([
-                   aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
-                   aliceTestClient.httpBackend.flush('/keys/query', 1).then(
-                       () => aliceTestClient.httpBackend.flush('/send/', 1, 20),
-                   ),
-               ]);
-           }).then(() => {
-               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(1);
-
-               // invalidate bob's and chris's device lists in separate syncs
-               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
-                   next_batch: '2',
-                   device_lists: {
-                       changed: ['@bob:xyz'],
-                   },
-               });
-               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
-                   next_batch: '3',
-                   device_lists: {
-                       changed: ['@chris:abc'],
-                   },
-               });
-               return aliceTestClient.httpBackend.flush('/sync', 2);
-           }).then(() => {
-               return testUtils.syncPromise(aliceTestClient.client);
-           }).then(() => {
-               // check that we don't yet have a request for chris's devices.
-               aliceTestClient.httpBackend.when('POST', '/keys/query', {
-                   device_keys: {
-                       '@chris:abc': {},
-                   },
-                   token: '3',
-               }).respond(200, {
-                   device_keys: {'@chris:abc': {}},
-               });
-               return aliceTestClient.httpBackend.flush('/keys/query', 1);
-           }).then((flushed) => {
-               expect(flushed).toEqual(0);
-               const bobStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-               if (bobStat != 1 && bobStat != 2) {
-                   throw new Error('Unexpected status for bob: wanted 1 or 2, got ' +
-                                   bobStat);
-               }
-
-               const chrisStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
-               if (chrisStat != 1 && chrisStat != 2) {
-                   throw new Error('Unexpected status for chris: wanted 1 or 2, got ' +
-                                   bobStat);
-               }
-
-               // now add an expectation for a query for bob's devices, and let
-               // it complete.
-               aliceTestClient.httpBackend.when('POST', '/keys/query', {
-                   device_keys: {
-                       '@bob:xyz': {},
-                   },
-                   token: '2',
-               }).respond(200, {
-                   device_keys: {'@bob:xyz': {}},
-               });
-               return aliceTestClient.httpBackend.flush('/keys/query', 1);
-           }).then((flushed) => {
-               expect(flushed).toEqual(1);
-
-               // wait for the client to stop processing the response
-               return aliceTestClient.client.downloadKeys(['@bob:xyz']);
-           }).then(() => {
-               const bobStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-               expect(bobStat).toEqual(3);
-               const chrisStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
-               if (chrisStat != 1 && chrisStat != 2) {
-                   throw new Error('Unexpected status for chris: wanted 1 or 2, got ' +
-                                   bobStat);
-               }
-
-               // now let the query for chris's devices complete.
-               return aliceTestClient.httpBackend.flush('/keys/query', 1);
-           }).then((flushed) => {
-               expect(flushed).toEqual(1);
-
-               // wait for the client to stop processing the response
-               return aliceTestClient.client.downloadKeys(['@chris:abc']);
-           }).then(() => {
-               const bobStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-               const chrisStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
-
-               expect(bobStat).toEqual(3);
-               expect(chrisStat).toEqual(3);
-               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(3);
-           });
-       });
-
-    it("Alice exports megolm keys and imports them to a new device", function(done) {
+    it("Alice exports megolm keys and imports them to a new device", function() {
         let messageEncrypted;
 
         return aliceTestClient.start().then(() => {
@@ -1104,9 +920,7 @@ describe("megolm", function() {
             };
 
             aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush("/sync", 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
         }).then(function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
             const event = room.getLiveTimeline().getEvents()[0];
@@ -1120,10 +934,10 @@ describe("megolm", function() {
             aliceTestClient = new TestClient(
                 "@alice:localhost", "device2", "access_token2",
             );
-
-            aliceTestClient.client.importRoomKeys(exported);
-
-            return aliceTestClient.start();
+            return aliceTestClient.client.initCrypto().then(() => {
+                aliceTestClient.client.importRoomKeys(exported);
+                return aliceTestClient.start();
+            });
         }).then(function() {
             const syncResponse = {
                 next_batch: 1,
@@ -1138,13 +952,11 @@ describe("megolm", function() {
             };
 
             aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush("/sync", 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
         }).then(function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
             const event = room.getLiveTimeline().getEvents()[0];
             expect(event.getContent().body).toEqual('42');
-        }).nodeify(done);
+        });
     });
 });

spec/mock-request.js

@@ -1,295 +0,0 @@
-"use strict";
-const q = require("q");
-import expect from 'expect';
-
-/**
- * Construct a mock HTTP backend, heavily inspired by Angular.js.
- * @constructor
- */
-function HttpBackend() {
-    this.requests = [];
-    this.expectedRequests = [];
-    const self = this;
-    // the request function dependency that the SDK needs.
-    this.requestFn = function(opts, callback) {
-        const req = new Request(opts, callback);
-        console.log("HTTP backend received request: %s", req);
-        self.requests.push(req);
-
-        const abort = function() {
-            const idx = self.requests.indexOf(req);
-            if (idx >= 0) {
-                console.log("Aborting HTTP request: %s %s", opts.method,
-                            opts.uri);
-                self.requests.splice(idx, 1);
-                req.callback("aborted");
-            }
-        };
-
-        return {
-            abort: abort,
-        };
-    };
-}
-HttpBackend.prototype = {
-    /**
-     * Respond to all of the requests (flush the queue).
-     * @param {string} path The path to flush (optional) default: all.
-     * @param {integer} numToFlush The number of things to flush (optional), default: all.
-     * @param {integer=} waitTime  The time (in ms) to wait for a request to happen.
-     *    default: 5
-     *
-     * @return {Promise} resolves when there is nothing left to flush, with the
-     *    number of requests flushed
-     */
-    flush: function(path, numToFlush, waitTime) {
-        const defer = q.defer();
-        const self = this;
-        let flushed = 0;
-        let triedWaiting = false;
-        if (waitTime === undefined) {
-            waitTime = 5;
-        }
-        console.log(
-            "HTTP backend flushing... (path=%s numToFlush=%s waitTime=%s)",
-            path, numToFlush, waitTime,
-        );
-        const tryFlush = function() {
-            // if there's more real requests and more expected requests, flush 'em.
-            console.log(
-                "  trying to flush queue => reqs=[%s] expected=[%s]",
-                self.requests, self.expectedRequests.map((er) => er.path),
-            );
-            if (self._takeFromQueue(path)) {
-                // try again on the next tick.
-                flushed += 1;
-                if (numToFlush && flushed === numToFlush) {
-                    console.log("  Flushed assigned amount: %s", numToFlush);
-                    defer.resolve(flushed);
-                } else {
-                    console.log("  flushed. Trying for more.");
-                    setTimeout(tryFlush, 0);
-                }
-            } else if (flushed === 0 && !triedWaiting) {
-                // we may not have made the request yet, wait a generous amount of
-                // time before giving up.
-                setTimeout(tryFlush, waitTime);
-                triedWaiting = true;
-            } else {
-                console.log("  no more flushes.");
-                defer.resolve(flushed);
-            }
-        };
-
-        setTimeout(tryFlush, 0);
-
-        return defer.promise;
-    },
-
-    /**
-     * Attempts to resolve requests/expected requests.
-     * @param {string} path The path to flush (optional) default: all.
-     * @return {boolean} true if something was resolved.
-     */
-    _takeFromQueue: function(path) {
-        let req = null;
-        let i;
-        let j;
-        let matchingReq = null;
-        let expectedReq = null;
-        let testResponse = null;
-        for (i = 0; i < this.requests.length; i++) {
-            req = this.requests[i];
-            for (j = 0; j < this.expectedRequests.length; j++) {
-                expectedReq = this.expectedRequests[j];
-                if (path && path !== expectedReq.path) {
-                    continue;
-                }
-                if (expectedReq.method === req.method &&
-                        req.path.indexOf(expectedReq.path) !== -1) {
-                    if (!expectedReq.data || (JSON.stringify(expectedReq.data) ===
-                            JSON.stringify(req.data))) {
-                        matchingReq = expectedReq;
-                        this.expectedRequests.splice(j, 1);
-                        break;
-                    }
-                }
-            }
-
-            if (matchingReq) {
-                // remove from request queue
-                this.requests.splice(i, 1);
-                i--;
-
-                for (j = 0; j < matchingReq.checks.length; j++) {
-                    matchingReq.checks[j](req);
-                }
-                testResponse = matchingReq.response;
-                console.log("    responding to %s", matchingReq.path);
-                let body = testResponse.body;
-                if (Object.prototype.toString.call(body) == "[object Function]") {
-                    body = body(req.path, req.data);
-                }
-                req.callback(
-                    testResponse.err, testResponse.response, body,
-                );
-                matchingReq = null;
-            }
-        }
-        if (testResponse) {  // flushed something
-            return true;
-        }
-        return false;
-    },
-
-    /**
-     * Makes sure that the SDK hasn't sent any more requests to the backend.
-     */
-    verifyNoOutstandingRequests: function() {
-        const firstOutstandingReq = this.requests[0] || {};
-        expect(this.requests.length).toEqual(0,
-            "Expected no more HTTP requests but received request to " +
-            firstOutstandingReq.path,
-        );
-    },
-
-    /**
-     * Makes sure that the test doesn't have any unresolved requests.
-     */
-    verifyNoOutstandingExpectation: function() {
-        const firstOutstandingExpectation = this.expectedRequests[0] || {};
-        expect(this.expectedRequests.length).toEqual(0,
-            "Expected to see HTTP request for " + firstOutstandingExpectation.path,
-        );
-    },
-
-    /**
-     * Create an expected request.
-     * @param {string} method The HTTP method
-     * @param {string} path The path (which can be partial)
-     * @param {Object} data The expected data.
-     * @return {Request} An expected request.
-     */
-    when: function(method, path, data) {
-        const pendingReq = new ExpectedRequest(method, path, data);
-        this.expectedRequests.push(pendingReq);
-        return pendingReq;
-    },
-};
-
-/**
- * Represents the expectation of a request.
- *
- * <p>Includes the conditions to be matched against, the checks to be made,
- * and the response to be returned.
- *
- * @constructor
- * @param {string} method
- * @param {string} path
- * @param {object?} data
- */
-function ExpectedRequest(method, path, data) {
-    this.method = method;
-    this.path = path;
-    this.data = data;
-    this.response = null;
-    this.checks = [];
-}
-
-ExpectedRequest.prototype = {
-    /**
-     * Execute a check when this request has been satisfied.
-     * @param {Function} fn The function to execute.
-     * @return {Request} for chaining calls.
-     */
-    check: function(fn) {
-        this.checks.push(fn);
-        return this;
-    },
-
-    /**
-     * Respond with the given data when this request is satisfied.
-     * @param {Number} code The HTTP status code.
-     * @param {Object|Function} data The HTTP JSON body. If this is a function,
-     * it will be invoked when the JSON body is required (which should be returned).
-     */
-    respond: function(code, data) {
-        this.response = {
-            response: {
-                statusCode: code,
-                headers: {},
-            },
-            body: data,
-            err: null,
-        };
-    },
-
-    /**
-     * Fail with an Error when this request is satisfied.
-     * @param {Number} code The HTTP status code.
-     * @param {Error} err The error to throw (e.g. Network Error)
-     */
-    fail: function(code, err) {
-        this.response = {
-            response: {
-                statusCode: code,
-                headers: {},
-            },
-            body: null,
-            err: err,
-        };
-    },
-};
-
-/**
- * Represents a request made by the app.
- *
- * @constructor
- * @param {object} opts opts passed to request()
- * @param {function} callback
- */
-function Request(opts, callback) {
-    this.opts = opts;
-    this.callback = callback;
-
-    Object.defineProperty(this, 'method', {
-        get: function() {
-            return opts.method;
-        },
-    });
-
-    Object.defineProperty(this, 'path', {
-        get: function() {
-            return opts.uri;
-        },
-    });
-
-    Object.defineProperty(this, 'data', {
-        get: function() {
-            return opts.body;
-        },
-    });
-
-    Object.defineProperty(this, 'queryParams', {
-        get: function() {
-            return opts.qs;
-        },
-    });
-
-    Object.defineProperty(this, 'headers', {
-        get: function() {
-            return opts.headers || {};
-        },
-    });
-}
-
-Request.prototype = {
-    toString: function() {
-        return this.method + " " + this.path;
-    },
-};
-
-/**
- * The HttpBackend class.
- */
-module.exports = HttpBackend;

spec/olm-loader.js

@@ -0,0 +1,24 @@
+/*
+Copyright 2017 Vector creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// try to load the olm library.
+
+try {
+    global.Olm = require('olm');
+    console.log('loaded libolm');
+} catch (e) {
+    console.warn("unable to run crypto tests: libolm not available");
+}

spec/test-utils.js

@@ -1,27 +1,43 @@
 "use strict";
 import expect from 'expect';
-import q from 'q';
+import Promise from 'bluebird';
 
-const sdk = require("..");
+// load olm before the sdk if possible
+import './olm-loader';
+
+import sdk from '..';
 const MatrixEvent = sdk.MatrixEvent;
 
 /**
  * Return a promise that is resolved when the client next emits a
  * SYNCING event.
  * @param {Object} client The client
+ * @param {Number=} count Number of syncs to wait for (default 1)
  * @return {Promise} Resolves once the client has emitted a SYNCING event
  */
-module.exports.syncPromise = function(client) {
-    const def = q.defer();
-    const cb = (state) => {
-        if (state == 'SYNCING') {
-            def.resolve();
-        } else {
-            client.once('sync', cb);
-        }
-    };
-    client.once('sync', cb);
-    return def.promise;
+module.exports.syncPromise = function(client, count) {
+    if (count === undefined) {
+        count = 1;
+    }
+    if (count <= 0) {
+        return Promise.resolve();
+    }
+
+    const p = new Promise((resolve, reject) => {
+        const cb = (state) => {
+            console.log(`${Date.now()} syncPromise(${count}): ${state}`);
+            if (state == 'SYNCING') {
+                resolve();
+            } else {
+                client.once('sync', cb);
+            }
+        };
+        client.once('sync', cb);
+    });
+
+    return p.then(() => {
+        return module.exports.syncPromise(client, count-1);
+    });
 };
 
 /**
@@ -178,35 +194,6 @@ module.exports.mkMessage = function(opts) {
 };
 
 
-/**
- * make the test fail, with the given exception
- *
- * <p>This is useful for use with integration tests which use asyncronous
- * methods: it can be added as a 'catch' handler in a promise chain.
- *
- * @param {Error} err   exception to be reported
- *
- * @deprecated
- * It turns out there are easier ways of doing this. Just use nodeify():
- *
- * it("should not throw", function(done) {
- *    asynchronousMethod().then(function() {
- *       // some tests
- *    }).nodeify(done);
- * });
- *
- * @example
- * it("should not throw", function(done) {
- *    asynchronousMethod().then(function() {
- *       // some tests
- *    }).catch(utils.failTest).done(done);
- * });
- */
-module.exports.failTest = function(err) {
-    expect(true).toBe(false, "Testfunc threw: " + err.stack);
-};
-
-
 /**
  * A mock implementation of webstorage
  *
@@ -232,3 +219,25 @@ module.exports.MockStorageApi.prototype = {
         delete this.data[k];
     },
 };
+
+
+/**
+ * If an event is being decrypted, wait for it to finish being decrypted.
+ *
+ * @param {MatrixEvent} event
+ * @returns {Promise} promise which resolves (to `event`) when the event has been decrypted
+ */
+module.exports.awaitDecryption = function(event) {
+    if (!event.isBeingDecrypted()) {
+        return Promise.resolve(event);
+    }
+
+    console.log(`${Date.now()} event ${event.getId()} is being decrypted; waiting`);
+
+    return new Promise((resolve, reject) => {
+        event.once('Event.decrypted', (ev) => {
+            console.log(`${Date.now()} event ${event.getId()} now decrypted`);
+            resolve(ev);
+        });
+    });
+};

spec/unit/crypto/DeviceList.spec.js

@@ -5,7 +5,7 @@ import testUtils from '../../test-utils';
 import utils from '../../../lib/utils';
 
 import expect from 'expect';
-import q from 'q';
+import Promise from 'bluebird';
 
 const signedDeviceList = {
     "failures": {},
@@ -64,7 +64,7 @@ describe('DeviceList', function() {
 
         dl.startTrackingDeviceList('@test1:sw1v.org');
 
-        const queryDefer1 = q.defer();
+        const queryDefer1 = Promise.defer();
         downloadSpy.andReturn(queryDefer1.promise);
 
         const prom1 = dl.refreshOutdatedDeviceLists();
@@ -83,7 +83,7 @@ describe('DeviceList', function() {
 
         dl.startTrackingDeviceList('@test1:sw1v.org');
 
-        const queryDefer1 = q.defer();
+        const queryDefer1 = Promise.defer();
         downloadSpy.andReturn(queryDefer1.promise);
 
         const prom1 = dl.refreshOutdatedDeviceLists();
@@ -91,7 +91,7 @@ describe('DeviceList', function() {
         downloadSpy.reset();
 
         // outdated notif arrives while the request is in flight.
-        const queryDefer2 = q.defer();
+        const queryDefer2 = Promise.defer();
         downloadSpy.andReturn(queryDefer2.promise);
 
         dl.invalidateUserDeviceList('@test1:sw1v.org');
@@ -110,7 +110,7 @@ describe('DeviceList', function() {
             console.log("Creating new devicelist to simulate app reload");
             downloadSpy.reset();
             const dl2 = createTestDeviceList();
-            const queryDefer3 = q.defer();
+            const queryDefer3 = Promise.defer();
             downloadSpy.andReturn(queryDefer3.promise);
 
             const prom3 = dl2.refreshOutdatedDeviceLists();

spec/unit/crypto/algorithms/megolm.spec.js

@@ -0,0 +1,268 @@
+try {
+    global.Olm = require('olm');
+} catch (e) {
+    console.warn("unable to run megolm tests: libolm not available");
+}
+
+import expect from 'expect';
+import Promise from 'bluebird';
+
+import sdk from '../../../..';
+import algorithms from '../../../../lib/crypto/algorithms';
+import WebStorageSessionStore from '../../../../lib/store/session/webstorage';
+import MockStorageApi from '../../../MockStorageApi';
+import testUtils from '../../../test-utils';
+
+// Crypto and OlmDevice won't import unless we have global.Olm
+let OlmDevice;
+let Crypto;
+if (global.Olm) {
+    OlmDevice = require('../../../../lib/crypto/OlmDevice');
+    Crypto = require('../../../../lib/crypto');
+}
+
+const MatrixEvent = sdk.MatrixEvent;
+const MegolmDecryption = algorithms.DECRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
+
+const ROOM_ID = '!ROOM:ID';
+
+describe("MegolmDecryption", function() {
+    if (!global.Olm) {
+        console.warn('Not running megolm unit tests: libolm not present');
+        return;
+    }
+
+    let megolmDecryption;
+    let mockOlmLib;
+    let mockCrypto;
+    let mockBaseApis;
+
+    beforeEach(function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+
+        mockCrypto = testUtils.mock(Crypto, 'Crypto');
+        mockBaseApis = {};
+
+        const mockStorage = new MockStorageApi();
+        const sessionStore = new WebStorageSessionStore(mockStorage);
+
+        const olmDevice = new OlmDevice(sessionStore);
+
+        megolmDecryption = new MegolmDecryption({
+            userId: '@user:id',
+            crypto: mockCrypto,
+            olmDevice: olmDevice,
+            baseApis: mockBaseApis,
+            roomId: ROOM_ID,
+        });
+
+
+        // we stub out the olm encryption bits
+        mockOlmLib = {};
+        mockOlmLib.ensureOlmSessionsForDevices = expect.createSpy();
+        mockOlmLib.encryptMessageForDevice =
+            expect.createSpy().andReturn(Promise.resolve());
+        megolmDecryption.olmlib = mockOlmLib;
+    });
+
+    describe('receives some keys:', function() {
+        let groupSession;
+        beforeEach(function() {
+            groupSession = new global.Olm.OutboundGroupSession();
+            groupSession.create();
+
+            // construct a fake decrypted key event via the use of a mocked
+            // 'crypto' implementation.
+            const event = new MatrixEvent({
+                type: 'm.room.encrypted',
+            });
+            const decryptedData = {
+                clearEvent: {
+                    type: 'm.room_key',
+                    content: {
+                        algorithm: 'm.megolm.v1.aes-sha2',
+                        room_id: ROOM_ID,
+                        session_id: groupSession.session_id(),
+                        session_key: groupSession.session_key(),
+                    },
+                },
+                senderCurve25519Key: "SENDER_CURVE25519",
+                claimedEd25519Key: "SENDER_ED25519",
+            };
+
+            const mockCrypto = {
+                decryptEvent: function() {
+                    return Promise.resolve(decryptedData);
+                },
+            };
+
+            return event.attemptDecryption(mockCrypto).then(() => {
+                megolmDecryption.onRoomKeyEvent(event);
+            });
+        });
+
+        it('can decrypt an event', function() {
+            const event = new MatrixEvent({
+                type: 'm.room.encrypted',
+                room_id: ROOM_ID,
+                content: {
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: groupSession.session_id(),
+                    ciphertext: groupSession.encrypt(JSON.stringify({
+                        room_id: ROOM_ID,
+                        content: 'testytest',
+                    })),
+                },
+            });
+
+            return megolmDecryption.decryptEvent(event).then((res) => {
+                expect(res.clearEvent.content).toEqual('testytest');
+            });
+        });
+
+        it('can respond to a key request event', function() {
+            const keyRequest = {
+                userId: '@alice:foo',
+                deviceId: 'alidevice',
+                requestBody: {
+                    room_id: ROOM_ID,
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: groupSession.session_id(),
+                },
+            };
+
+            return megolmDecryption.hasKeysForKeyRequest(
+                keyRequest,
+            ).then((hasKeys) => {
+                expect(hasKeys).toBe(true);
+
+                // set up some pre-conditions for the share call
+                const deviceInfo = {};
+                mockCrypto.getStoredDevice.andReturn(deviceInfo);
+
+                mockOlmLib.ensureOlmSessionsForDevices.andReturn(
+                    Promise.resolve({'@alice:foo': {'alidevice': {
+                        sessionId: 'alisession',
+                    }}}),
+                );
+
+                const awaitEncryptForDevice = new Promise((res, rej) => {
+                    mockOlmLib.encryptMessageForDevice.andCall(() => {
+                        res();
+                        return Promise.resolve();
+                    });
+                });
+
+                mockBaseApis.sendToDevice = expect.createSpy();
+
+                // do the share
+                megolmDecryption.shareKeysWithDevice(keyRequest);
+
+                // it's asynchronous, so we have to wait a bit
+                return awaitEncryptForDevice;
+            }).then(() => {
+                // check that it called encryptMessageForDevice with
+                // appropriate args.
+                expect(mockOlmLib.encryptMessageForDevice.calls.length)
+                    .toEqual(1);
+
+                const call = mockOlmLib.encryptMessageForDevice.calls[0];
+                const payload = call.arguments[6];
+
+                expect(payload.type).toEqual("m.forwarded_room_key");
+                expect(payload.content).toInclude({
+                    sender_key: "SENDER_CURVE25519",
+                    sender_claimed_ed25519_key: "SENDER_ED25519",
+                    session_id: groupSession.session_id(),
+                    chain_index: 0,
+                    forwarding_curve25519_key_chain: [],
+                });
+                expect(payload.content.session_key).toExist();
+            });
+        });
+
+        it("can detect replay attacks", function() {
+            // trying to decrypt two different messages (marked by different
+            // event IDs or timestamps) using the same (sender key, session id,
+            // message index) triple should result in an exception being thrown
+            // as it should be detected as a replay attack.
+            const sessionId = groupSession.session_id();
+            const cipherText = groupSession.encrypt(JSON.stringify({
+                room_id: ROOM_ID,
+                content: 'testytest',
+            }));
+            const event1 = new MatrixEvent({
+                type: 'm.room.encrypted',
+                room_id: ROOM_ID,
+                content: {
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: sessionId,
+                    ciphertext: cipherText,
+                },
+                event_id: "$event1",
+                origin_server_ts: 1507753886000,
+            });
+
+            const successHandler = expect.createSpy();
+            const failureHandler = expect.createSpy()
+                .andCall((err) => {
+                    expect(err.toString()).toMatch(
+                        /Duplicate message index, possible replay attack/,
+                    );
+                });
+
+            return megolmDecryption.decryptEvent(event1).then((res) => {
+                const event2 = new MatrixEvent({
+                    type: 'm.room.encrypted',
+                    room_id: ROOM_ID,
+                    content: {
+                        algorithm: 'm.megolm.v1.aes-sha2',
+                        sender_key: "SENDER_CURVE25519",
+                        session_id: sessionId,
+                        ciphertext: cipherText,
+                    },
+                    event_id: "$event2",
+                    origin_server_ts: 1507754149000,
+                });
+
+                return megolmDecryption.decryptEvent(event2);
+            }).then(
+                successHandler,
+                failureHandler,
+            ).then(() => {
+                expect(successHandler).toNotHaveBeenCalled();
+                expect(failureHandler).toHaveBeenCalled();
+            });
+        });
+
+        it("allows re-decryption of the same event", function() {
+            // in contrast with the previous test, if the event ID and
+            // timestamp are the same, then it should not be considered a
+            // replay attack
+            const sessionId = groupSession.session_id();
+            const cipherText = groupSession.encrypt(JSON.stringify({
+                room_id: ROOM_ID,
+                content: 'testytest',
+            }));
+            const event = new MatrixEvent({
+                type: 'm.room.encrypted',
+                room_id: ROOM_ID,
+                content: {
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: sessionId,
+                    ciphertext: cipherText,
+                },
+                event_id: "$event1",
+                origin_server_ts: 1507753886000,
+            });
+
+            return megolmDecryption.decryptEvent(event).then((res) => {
+                return megolmDecryption.decryptEvent(event);
+                // test is successful if no exception is thrown
+            });
+        });
+    });
+});

spec/unit/event.spec.js

@@ -0,0 +1,82 @@
+/*
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import sdk from '../..';
+const MatrixEvent = sdk.MatrixEvent;
+
+import testUtils from '../test-utils';
+
+import expect from 'expect';
+import Promise from 'bluebird';
+
+describe("MatrixEvent", () => {
+    beforeEach(function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+    });
+
+    describe(".attemptDecryption", () => {
+        let encryptedEvent;
+
+        beforeEach(() => {
+            encryptedEvent = new MatrixEvent({
+                id: 'test_encrypted_event',
+                type: 'm.room.encrypted',
+                content: {
+                    ciphertext: 'secrets',
+                },
+            });
+        });
+
+        it('should retry decryption if a retry is queued', () => {
+            let callCount = 0;
+
+            let prom2;
+
+            const crypto = {
+                decryptEvent: function() {
+                    ++callCount;
+                    console.log(`decrypt: ${callCount}`);
+                    if (callCount == 1) {
+                        // schedule a second decryption attempt while
+                        // the first one is still running.
+                        prom2 = encryptedEvent.attemptDecryption(crypto);
+
+                        const error = new Error("nope");
+                        error.name = 'DecryptionError';
+                        return Promise.reject(error);
+                    } else {
+                        expect(prom2.isFulfilled()).toBe(
+                            false, 'second attemptDecryption resolved too soon');
+
+                        return Promise.resolve({
+                            clearEvent: {
+                                type: 'm.room.message',
+                            },
+                        });
+                    }
+                },
+            };
+
+            return encryptedEvent.attemptDecryption(crypto).then(() => {
+                expect(callCount).toEqual(2);
+                expect(encryptedEvent.getType()).toEqual('m.room.message');
+
+                // make sure the second attemptDecryption resolves
+                return prom2;
+            });
+        });
+    });
+});

spec/unit/interactive-auth.spec.js

@@ -16,7 +16,7 @@ limitations under the License.
 "use strict";
 
 import 'source-map-support/register';
-const q = require("q");
+import Promise from 'bluebird';
 const sdk = require("../..");
 const utils = require("../test-utils");
 
@@ -81,14 +81,14 @@ describe("InteractiveAuth", function() {
                 type: "logintype",
                 foo: "bar",
             });
-            return q(requestRes);
+            return Promise.resolve(requestRes);
         });
 
         ia.attemptAuth().then(function(res) {
             expect(res).toBe(requestRes);
             expect(doRequest.calls.length).toEqual(1);
             expect(stateUpdated.calls.length).toEqual(1);
-        }).catch(utils.failTest).done(done);
+        }).nodeify(done);
     });
 
     it("should make a request if no authdata is provided", function(done) {
@@ -138,7 +138,7 @@ describe("InteractiveAuth", function() {
                     type: "logintype",
                     foo: "bar",
                 });
-                return q(requestRes);
+                return Promise.resolve(requestRes);
             });
 
             ia.submitAuthDict({
@@ -151,6 +151,6 @@ describe("InteractiveAuth", function() {
             expect(res).toBe(requestRes);
             expect(doRequest.calls.length).toEqual(2);
             expect(stateUpdated.calls.length).toEqual(1);
-        }).catch(utils.failTest).done(done);
+        }).nodeify(done);
     });
 });

spec/unit/matrix-client.spec.js

@@ -1,6 +1,6 @@
 "use strict";
 import 'source-map-support/register';
-const q = require("q");
+import Promise from 'bluebird';
 const sdk = require("../..");
 const MatrixClient = sdk.MatrixClient;
 const utils = require("../test-utils");
@@ -62,7 +62,7 @@ describe("MatrixClient", function() {
     let pendingLookup = null;
     function httpReq(cb, method, path, qp, data, prefix) {
         if (path === KEEP_ALIVE_PATH && acceptKeepalives) {
-            return q();
+            return Promise.resolve();
         }
         const next = httpLookups.shift();
         const logLine = (
@@ -84,7 +84,7 @@ describe("MatrixClient", function() {
                 );
             }
             pendingLookup = {
-                promise: q.defer().promise,
+                promise: Promise.defer().promise,
                 method: method,
                 path: path,
             };
@@ -109,7 +109,7 @@ describe("MatrixClient", function() {
             }
 
             if (next.error) {
-                return q.reject({
+                return Promise.reject({
                     errcode: next.error.errcode,
                     httpStatus: next.error.httpStatus,
                     name: next.error.errcode,
@@ -117,10 +117,10 @@ describe("MatrixClient", function() {
                     data: next.error,
                 });
             }
-            return q(next.data);
+            return Promise.resolve(next.data);
         }
         expect(true).toBe(false, "Expected different request. " + logLine);
-        return q.defer().promise;
+        return Promise.defer().promise;
     }
 
     beforeEach(function() {
@@ -136,8 +136,8 @@ describe("MatrixClient", function() {
             "getFilterIdByName", "setFilterIdByName", "getFilter", "storeFilter",
             "getSyncAccumulator", "startup", "deleteAllData",
         ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
-        store.getSavedSync = expect.createSpy().andReturn(q(null));
-        store.setSyncData = expect.createSpy().andReturn(q(null));
+        store.getSavedSync = expect.createSpy().andReturn(Promise.resolve(null));
+        store.setSyncData = expect.createSpy().andReturn(Promise.resolve(null));
         client = new MatrixClient({
             baseUrl: "https://my.home.server",
             idBaseUrl: identityServerUrl,
@@ -174,10 +174,10 @@ describe("MatrixClient", function() {
         // a DIFFERENT test (pollution between tests!) - we return unresolved
         // promises to stop the client from continuing to run.
         client._http.authedRequest.andCall(function() {
-            return q.defer().promise;
+            return Promise.defer().promise;
         });
         client._http.authedRequestWithPrefix.andCall(function() {
-            return q.defer().promise;
+            return Promise.defer().promise;
         });
     });
 

spec/unit/room.spec.js

@@ -404,7 +404,7 @@ describe("Room", function() {
         it("should copy state from previous timeline", function() {
             room.addLiveEvents([events[0], events[1]]);
             expect(room.getLiveTimeline().getEvents().length).toEqual(2);
-            room.resetLiveTimeline();
+            room.resetLiveTimeline('sometoken', 'someothertoken');
 
             room.addLiveEvents([events[2]]);
             const oldState = room.getLiveTimeline().getState(EventTimeline.BACKWARDS);
@@ -417,7 +417,7 @@ describe("Room", function() {
         it("should reset the legacy timeline fields", function() {
             room.addLiveEvents([events[0], events[1]]);
             expect(room.timeline.length).toEqual(2);
-            room.resetLiveTimeline();
+            room.resetLiveTimeline('sometoken', 'someothertoken');
 
             room.addLiveEvents([events[2]]);
             const newLiveTimeline = room.getLiveTimeline();
@@ -451,7 +451,7 @@ describe("Room", function() {
             room.addLiveEvents([events[0]]);
             expect(room.timeline.length).toEqual(1);
             const firstLiveTimeline = room.getLiveTimeline();
-            room.resetLiveTimeline();
+            room.resetLiveTimeline('sometoken', 'someothertoken');
 
             const tl = room.getTimelineForEvent(events[0].getId());
             expect(tl).toBe(timelineSupport ? firstLiveTimeline : null);
@@ -886,7 +886,7 @@ describe("Room", function() {
                 expect(name).toEqual("Empty room");
             });
 
-            it("should return 'Invite from [inviter display name] if state event " +
+            it("should return '[inviter display name] if state event " +
                "available",
             function() {
                 setJoinRule("invite");
@@ -894,7 +894,7 @@ describe("Room", function() {
                 addMember(userB, "invite", {user: userA});
                 room.recalculate(userB);
                 const name = room.name;
-                expect(name).toEqual("Invite from Alice");
+                expect(name).toEqual("Alice");
             });
 
             it("should return inviter mxid if display name not available",
@@ -904,7 +904,7 @@ describe("Room", function() {
                 addMember(userB, "invite", {user: userA});
                 room.recalculate(userB);
                 const name = room.name;
-                expect(name).toEqual("Invite from " + userA);
+                expect(name).toEqual(userA);
             });
         });
     });

spec/unit/scheduler.spec.js

@@ -2,7 +2,7 @@
 /* eslint new-cap: "off" */
 
 import 'source-map-support/register';
-const q = require("q");
+import Promise from 'bluebird';
 const sdk = require("../..");
 const MatrixScheduler = sdk.MatrixScheduler;
 const MatrixError = sdk.MatrixError;
@@ -41,7 +41,7 @@ describe("MatrixScheduler", function() {
         });
         retryFn = null;
         queueFn = null;
-        defer = q.defer();
+        defer = Promise.defer();
     });
 
     afterEach(function() {
@@ -55,8 +55,8 @@ describe("MatrixScheduler", function() {
         queueFn = function() {
             return "one_big_queue";
         };
-        const deferA = q.defer();
-        const deferB = q.defer();
+        const deferA = Promise.defer();
+        const deferB = Promise.defer();
         let resolvedA = false;
         scheduler.setProcessFunction(function(event) {
             if (resolvedA) {
@@ -80,7 +80,7 @@ describe("MatrixScheduler", function() {
     it("should invoke the retryFn on failure and wait the amount of time specified",
     function(done) {
         const waitTimeMs = 1500;
-        const retryDefer = q.defer();
+        const retryDefer = Promise.defer();
         retryFn = function() {
             retryDefer.resolve();
             return waitTimeMs;
@@ -97,7 +97,7 @@ describe("MatrixScheduler", function() {
                 return defer.promise;
             } else if (procCount === 2) {
                 // don't care about this defer
-                return q.defer().promise;
+                return Promise.defer().promise;
             }
             expect(procCount).toBeLessThan(3);
         });
@@ -125,8 +125,8 @@ describe("MatrixScheduler", function() {
  return "yep";
 };
 
-        const deferA = q.defer();
-        const deferB = q.defer();
+        const deferA = Promise.defer();
+        const deferB = Promise.defer();
         let procCount = 0;
         scheduler.setProcessFunction(function(ev) {
             procCount += 1;
@@ -177,7 +177,7 @@ describe("MatrixScheduler", function() {
         const expectOrder = [
             eventA.getId(), eventB.getId(), eventD.getId(),
         ];
-        const deferA = q.defer();
+        const deferA = Promise.defer();
         scheduler.setProcessFunction(function(event) {
             const id = expectOrder.shift();
             expect(id).toEqual(event.getId());

spec/unit/timeline-window.spec.js

@@ -1,6 +1,6 @@
 "use strict";
 import 'source-map-support/register';
-const q = require("q");
+import Promise from 'bluebird';
 const sdk = require("../..");
 const EventTimeline = sdk.EventTimeline;
 const TimelineWindow = sdk.TimelineWindow;
@@ -157,7 +157,7 @@ describe("TimelineWindow", function() {
         client = {};
         client.getEventTimeline = function(timelineSet0, eventId0) {
             expect(timelineSet0).toBe(timelineSet);
-            return q(timeline);
+            return Promise.resolve(timeline);
         };
 
         return new TimelineWindow(client, timelineSet, opts);
@@ -179,7 +179,7 @@ describe("TimelineWindow", function() {
             timelineWindow.load(undefined, 2).then(function() {
                 const expectedEvents = liveTimeline.getEvents().slice(1);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
         it("should initialise from a specific event", function(done) {
@@ -191,14 +191,14 @@ describe("TimelineWindow", function() {
             client.getEventTimeline = function(timelineSet0, eventId0) {
                 expect(timelineSet0).toBe(timelineSet);
                 expect(eventId0).toEqual(eventId);
-                return q(timeline);
+                return Promise.resolve(timeline);
             };
 
             const timelineWindow = new TimelineWindow(client, timelineSet);
             timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
         it("canPaginate should return false until load has returned",
@@ -219,7 +219,7 @@ describe("TimelineWindow", function() {
                     .toBe(false);
                 expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
                     .toBe(false);
-                return q(timeline);
+                return Promise.resolve(timeline);
             };
 
             timelineWindow.load(eventId, 3).then(function() {
@@ -229,7 +229,7 @@ describe("TimelineWindow", function() {
                     .toBe(true);
                 expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
                     .toBe(true);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
     });
 
@@ -277,7 +277,7 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(false);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
         it("should advance into next timeline", function(done) {
@@ -322,7 +322,7 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(false);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
         it("should retreat into previous timeline", function(done) {
@@ -367,7 +367,7 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(false);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
         it("should make forward pagination requests", function(done) {
@@ -383,7 +383,7 @@ describe("TimelineWindow", function() {
                 expect(opts.limit).toEqual(2);
 
                 addEventsToTimeline(timeline, 3, false);
-                return q(true);
+                return Promise.resolve(true);
             };
 
             timelineWindow.load(eventId, 3).then(function() {
@@ -399,7 +399,7 @@ describe("TimelineWindow", function() {
                 expect(success).toBe(true);
                 const expectedEvents = timeline.getEvents().slice(0, 5);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
 
@@ -416,7 +416,7 @@ describe("TimelineWindow", function() {
                 expect(opts.limit).toEqual(2);
 
                 addEventsToTimeline(timeline, 3, true);
-                return q(true);
+                return Promise.resolve(true);
             };
 
             timelineWindow.load(eventId, 3).then(function() {
@@ -432,7 +432,7 @@ describe("TimelineWindow", function() {
                 expect(success).toBe(true);
                 const expectedEvents = timeline.getEvents().slice(1, 6);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
         it("should limit the number of unsuccessful pagination requests",
@@ -449,7 +449,7 @@ describe("TimelineWindow", function() {
                 expect(opts.backwards).toBe(false);
                 expect(opts.limit).toEqual(2);
                 paginateCount += 1;
-                return q(true);
+                return Promise.resolve(true);
             };
 
             timelineWindow.load(eventId, 3).then(function() {
@@ -471,7 +471,7 @@ describe("TimelineWindow", function() {
                     .toBe(false);
                 expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
                     .toBe(true);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
     });
 });

src/ReEmitter.js

@@ -0,0 +1,46 @@
+/*
+Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module
+ */
+
+export default class Reemitter {
+    constructor(target) {
+        this.target = target;
+
+        // We keep one bound event handler for each event name so we know
+        // what event is arriving
+        this.boundHandlers = {};
+    }
+
+    _handleEvent(eventName, ...args) {
+        this.target.emit(eventName, ...args);
+    }
+
+    reEmit(source, eventNames) {
+        for (const eventName of eventNames) {
+            if (this.boundHandlers[eventName] === undefined) {
+                this.boundHandlers[eventName] = this._handleEvent.bind(this, eventName);
+            }
+            const boundHandler = this.boundHandlers[eventName];
+
+            source.on(eventName, boundHandler);
+        }
+    }
+}

src/base-apis.js

@@ -54,6 +54,8 @@ const utils = require("./utils");
  * to all requests with this client. Useful for application services which require
  * <code>?user_id=</code>.
  *
+ * @param {boolean} [opts.useAuthorizationHeader = false] Set to true to use
+ * Authorization header instead of query param to send the access token to the server.
  */
 function MatrixBaseApis(opts) {
     utils.checkObjectHasKeys(opts, ["baseUrl", "request"]);
@@ -70,6 +72,7 @@ function MatrixBaseApis(opts) {
         onlyData: true,
         extraParams: opts.queryParams,
         localTimeoutMs: opts.localTimeoutMs,
+        useAuthorizationHeader: opts.useAuthorizationHeader,
     };
     this._http = new httpApi.MatrixHttpApi(this, httpOpts);
 
@@ -398,6 +401,304 @@ MatrixBaseApis.prototype.roomState = function(roomId, callback) {
     return this._http.authedRequest(callback, "GET", path);
 };
 
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group summary object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupSummary = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/summary", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group profile object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupProfile = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/profile", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {string} groupId
+ * @param {Object} profile The group profile object
+ * @param {string=} profile.name Name of the group
+ * @param {string=} profile.avatar_url MXC avatar URL
+ * @param {string=} profile.short_description A short description of the room
+ * @param {string=} profile.long_description A longer HTML description of the room
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.setGroupProfile = function(groupId, profile) {
+    const path = utils.encodeUri("/groups/$groupId/profile", {$groupId: groupId});
+    return this._http.authedRequest(
+        undefined, "POST", path, undefined, profile,
+    );
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group users list object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupUsers = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/users", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group users list object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupInvitedUsers = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/invited_users", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group rooms list object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupRooms = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/rooms", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} userId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.inviteUserToGroup = function(groupId, userId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/users/invite/$userId",
+        {$groupId: groupId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} userId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeUserFromGroup = function(groupId, userId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/users/remove/$userId",
+        {$groupId: groupId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} userId
+ * @param {string} roleId Optional.
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.addUserToGroupSummary = function(groupId, userId, roleId) {
+    const path = utils.encodeUri(
+        roleId ?
+            "/groups/$groupId/summary/$roleId/users/$userId" :
+            "/groups/$groupId/summary/users/$userId",
+        {$groupId: groupId, $roleId: roleId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} userId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeUserFromGroupSummary = function(groupId, userId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/summary/users/$userId",
+        {$groupId: groupId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "DELETE", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @param {string} categoryId Optional.
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.addRoomToGroupSummary = function(groupId, roomId, categoryId) {
+    const path = utils.encodeUri(
+        categoryId ?
+            "/groups/$groupId/summary/$categoryId/rooms/$roomId" :
+            "/groups/$groupId/summary/rooms/$roomId",
+        {$groupId: groupId, $categoryId: categoryId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeRoomFromGroupSummary = function(groupId, roomId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/summary/rooms/$roomId",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "DELETE", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @param {bool} isPublic Whether the room-group association is visible to non-members
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.addRoomToGroup = function(groupId, roomId, isPublic) {
+    if (isPublic === undefined) {
+        isPublic = true;
+    }
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/rooms/$roomId",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined,
+        { "m.visibility": { type: isPublic ? "public" : "private" } },
+    );
+};
+
+/**
+ * Configure the visibility of a room-group association.
+ * @param {string} groupId
+ * @param {string} roomId
+ * @param {bool} isPublic Whether the room-group association is visible to non-members
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.updateGroupRoomVisibility = function(groupId, roomId, isPublic) {
+    // NB: The /config API is generic but there's not much point in exposing this yet as synapse
+    //     is the only server to implement this. In future we should consider an API that allows
+    //     arbitrary configuration, i.e. "config/$configKey".
+
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/rooms/$roomId/config/m.visibility",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined,
+        { type: isPublic ? "public" : "private" },
+    );
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeRoomFromGroup = function(groupId, roomId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/rooms/$roomId",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "DELETE", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.acceptGroupInvite = function(groupId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/self/accept_invite",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.leaveGroup = function(groupId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/self/leave",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @return {module:client.Promise} Resolves: The groups to which the user is joined
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getJoinedGroups = function() {
+    const path = utils.encodeUri("/joined_groups");
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {Object} content Request content
+ * @param {string} content.localpart The local part of the desired group ID
+ * @param {Object} content.profile Group profile object
+ * @return {module:client.Promise} Resolves: Object with key group_id: id of the created group
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.createGroup = function(content) {
+    const path = utils.encodeUri("/create_group");
+    return this._http.authedRequest(
+        undefined, "POST", path, undefined, content,
+    );
+};
+
+/**
+ * @param {string[]} userIds List of user IDs
+ * @return {module:client.Promise} Resolves: Object as exmaple below
+ *
+ *     {
+ *         "users": {
+ *             "@bob:example.com": {
+ *                 "+example:example.com"
+ *             }
+ *         }
+ *     }
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getPublicisedGroups = function(userIds) {
+    const path = utils.encodeUri("/publicised_groups");
+    return this._http.authedRequest(
+        undefined, "POST", path, undefined, { user_ids: userIds },
+    );
+};
+
+/**
+ * @param {string} groupId
+ * @param {bool} isPublic Whether the user's membership of this group is made public
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.setGroupPublicity = function(groupId, isPublic) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/self/update_publicity",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {
+        publicise: isPublic,
+    });
+};
+
 /**
  * Retrieve a state event.
  * @param {string} roomId
@@ -678,6 +979,31 @@ MatrixBaseApis.prototype.setRoomDirectoryVisibilityAppService =
     );
 };
 
+// User Directory Operations
+// =========================
+
+/**
+ * Query the user directory with a term matching user IDs, display names and domains.
+ * @param {object} opts options
+ * @param {string} opts.term the term with which to search.
+ * @param {number} opts.limit the maximum number of results to return. The server will
+ *                 apply a limit if unspecified.
+ * @return {module:client.Promise} Resolves: an array of results.
+ */
+MatrixBaseApis.prototype.searchUserDirectory = function(opts) {
+    const body = {
+        search_term: opts.term,
+    };
+
+    if (opts.limit !== undefined) {
+        body.limit = opts.limit;
+    }
+
+    return this._http.authedRequest(
+        undefined, "POST", "/user_directory/search", undefined, body,
+    );
+};
+
 
 // Media operations
 // ================
@@ -710,6 +1036,10 @@ MatrixBaseApis.prototype.setRoomDirectoryVisibilityAppService =
  *    invoke on success/failure. See the promise return values for more
  *    information.
  *
+ * @param {Function=} opts.progressHandler Optional. Called when a chunk of
+ *    data has been uploaded, with an object containing the fields `loaded`
+ *    (number of bytes transferred) and `total` (total size, if known).
+ *
  * @return {module:client.Promise} Resolves to response object, as
  *    determined by this.opts.onlyData, opts.rawResponse, and
  *    opts.onlyContentUri.  Rejects with an error (usually a MatrixError).
@@ -1301,7 +1631,15 @@ MatrixBaseApis.prototype.getThirdpartyProtocols = function() {
     return this._http.authedRequestWithPrefix(
         undefined, "GET", "/thirdparty/protocols", undefined, undefined,
         httpApi.PREFIX_UNSTABLE,
-    );
+    ).then((response) => {
+        // sanity check
+        if (!response || typeof(response) !== 'object') {
+            throw new Error(
+                `/thirdparty/protocols did not return an object: ${response}`,
+            );
+        }
+        return response;
+    });
 };
 
 /**

src/client.js

@@ -23,7 +23,7 @@ const PushProcessor = require('./pushprocessor');
  * @module client
  */
 const EventEmitter = require("events").EventEmitter;
-const q = require("q");
+import Promise from 'bluebird';
 const url = require('url');
 
 const httpApi = require("./http-api");
@@ -40,6 +40,8 @@ const SyncApi = require("./sync");
 const MatrixBaseApis = require("./base-apis");
 const MatrixError = httpApi.MatrixError;
 
+import ReEmitter from './ReEmitter';
+
 const SCROLLBACK_DELAY_MS = 3000;
 let CRYPTO_ENABLED = false;
 
@@ -47,8 +49,7 @@ try {
     var Crypto = require("./crypto");
     CRYPTO_ENABLED = true;
 } catch (e) {
-    console.error("olm load error", e);
-    // Olm not installed.
+    console.warn("Unable to load crypto module: crypto will be disabled: " + e);
 }
 
 /**
@@ -98,16 +99,34 @@ try {
  * @param {Number=} opts.localTimeoutMs Optional. The default maximum amount of
  * time to wait before timing out HTTP requests. If not specified, there is no timeout.
  *
+ * @param {boolean} [opts.useAuthorizationHeader = false] Set to true to use
+ * Authorization header instead of query param to send the access token to the server.
+ *
  * @param {boolean} [opts.timelineSupport = false] Set to true to enable
  * improved timeline support ({@link
  * module:client~MatrixClient#getEventTimeline getEventTimeline}). It is
  * disabled by default for compatibility with older clients - in particular to
  * maintain support for back-paginating the live timeline after a '/sync'
  * result with a gap.
+ *
+ * @param {module:crypto.store.base~CryptoStore} opts.cryptoStore
+ *    crypto store implementation.
  */
 function MatrixClient(opts) {
+    // Allow trailing slash in HS url
+    if (opts.baseUrl && opts.baseUrl.endsWith("/")) {
+        opts.baseUrl = opts.baseUrl.substr(0, opts.baseUrl.length - 1);
+    }
+
+    // Allow trailing slash in IS url
+    if (opts.idBaseUrl && opts.idBaseUrl.endsWith("/")) {
+        opts.idBaseUrl = opts.idBaseUrl.substr(0, opts.idBaseUrl.length - 1);
+    }
+
     MatrixBaseApis.call(this, opts);
 
+    this.reEmitter = new ReEmitter(this);
+
     this.store = opts.store || new StubStore();
 
     this.deviceId = opts.deviceId || null;
@@ -153,21 +172,49 @@ function MatrixClient(opts) {
     this._notifTimelineSet = null;
 
     this._crypto = null;
-    if (CRYPTO_ENABLED && Boolean(opts.sessionStore) &&
-            userId !== null && this.deviceId !== null) {
-        this._crypto = new Crypto(
-            this, this,
-            opts.sessionStore,
-            userId, this.deviceId,
-            this.store,
-        );
+    this._cryptoStore = opts.cryptoStore;
+    this._sessionStore = opts.sessionStore;
 
+    this._forceTURN = opts.forceTURN || false;
+
+    if (CRYPTO_ENABLED) {
         this.olmVersion = Crypto.getOlmVersion();
     }
 }
 utils.inherits(MatrixClient, EventEmitter);
 utils.extend(MatrixClient.prototype, MatrixBaseApis.prototype);
 
+/**
+ * Clear any data out of the persistent stores used by the client.
+ *
+ * @returns {Promise} Promise which resolves when the stores have been cleared.
+ */
+MatrixClient.prototype.clearStores = function() {
+    if (this._clientRunning) {
+        throw new Error("Cannot clear stores while client is running");
+    }
+
+    const promises = [];
+
+    promises.push(this.store.deleteAllData());
+    if (this._cryptoStore) {
+        promises.push(this._cryptoStore.deleteAllData());
+    }
+    return Promise.all(promises);
+};
+
+/**
+ * Get the user-id of the logged-in user
+ *
+ * @return {?string} MXID for the logged-in user, or null if not logged in
+ */
+MatrixClient.prototype.getUserId = function() {
+    if (this.credentials && this.credentials.userId) {
+        return this.credentials.userId;
+    }
+    return null;
+};
+
 /**
  * Get the domain for this client's MXID
  * @return {?string} Domain of this MXID
@@ -207,6 +254,16 @@ MatrixClient.prototype.supportsVoip = function() {
     return this._supportsVoip;
 };
 
+/**
+ * Set whether VoIP calls are forced to use only TURN
+ * candidates. This is the same as the forceTURN option
+ * when creating the client.
+ * @param {bool} forceTURN True to force use of TURN servers
+ */
+MatrixClient.prototype.setForceTURN = function(forceTURN) {
+    this._forceTURN = forceTURN;
+};
+
 /**
  * Get the current sync state.
  * @return {?string} the sync state, which may be null.
@@ -278,6 +335,73 @@ MatrixClient.prototype.setNotifTimelineSet = function(notifTimelineSet) {
 // Crypto bits
 // ===========
 
+/**
+ * Initialise support for end-to-end encryption in this client
+ *
+ * You should call this method after creating the matrixclient, but *before*
+ * calling `startClient`, if you want to support end-to-end encryption.
+ *
+ * It will return a Promise which will resolve when the crypto layer has been
+ * successfully initialised.
+ */
+MatrixClient.prototype.initCrypto = async function() {
+    if (this._crypto) {
+        console.warn("Attempt to re-initialise e2e encryption on MatrixClient");
+        return;
+    }
+
+    if (!CRYPTO_ENABLED) {
+        throw new Error(
+            `End-to-end encryption not supported in this js-sdk build: did ` +
+                `you remember to load the olm library?`,
+        );
+    }
+
+    if (!this._sessionStore) {
+        // this is temporary, the sessionstore is supposed to be going away
+        throw new Error(`Cannot enable encryption: no sessionStore provided`);
+    }
+    if (!this._cryptoStore) {
+        // the cryptostore is provided by sdk.createClient, so this shouldn't happen
+        throw new Error(`Cannot enable encryption: no cryptoStore provided`);
+    }
+
+    const userId = this.getUserId();
+    if (userId === null) {
+        throw new Error(
+            `Cannot enable encryption on MatrixClient with unknown userId: ` +
+                `ensure userId is passed in createClient().`,
+        );
+    }
+    if (this.deviceId === null) {
+        throw new Error(
+            `Cannot enable encryption on MatrixClient with unknown deviceId: ` +
+                `ensure deviceId is passed in createClient().`,
+        );
+    }
+
+    const crypto = new Crypto(
+        this,
+        this._sessionStore,
+        userId, this.deviceId,
+        this.store,
+        this._cryptoStore,
+    );
+
+    this.reEmitter.reEmit(crypto, [
+        "crypto.roomKeyRequest",
+        "crypto.roomKeyRequestCancellation",
+    ]);
+
+    await crypto.init();
+
+    // if crypto initialisation was successful, tell it to attach its event
+    // handlers.
+    crypto.registerEventHandlers(this);
+    this._crypto = crypto;
+};
+
+
 /**
  * Is end-to-end crypto enabled for this client.
  * @return {boolean} True if end-to-end is enabled.
@@ -323,42 +447,40 @@ MatrixClient.prototype.uploadKeys = function() {
  */
 MatrixClient.prototype.downloadKeys = function(userIds, forceDownload) {
     if (this._crypto === null) {
-        return q.reject(new Error("End-to-end encryption disabled"));
+        return Promise.reject(new Error("End-to-end encryption disabled"));
     }
     return this._crypto.downloadKeys(userIds, forceDownload);
 };
 
-/**
- * List the stored device keys for a user id
- *
- * @deprecated prefer {@link module:client#getStoredDevicesForUser}
- *
- * @param {string} userId the user to list keys for.
- *
- * @return {object[]} list of devices with "id", "verified", "blocked",
- *    "key", and "display_name" parameters.
- */
-MatrixClient.prototype.listDeviceKeys = function(userId) {
-    if (this._crypto === null) {
-        throw new Error("End-to-end encryption disabled");
-    }
-    return this._crypto.listDeviceKeys(userId);
-};
-
 /**
  * Get the stored device keys for a user id
  *
  * @param {string} userId the user to list keys for.
  *
- * @return {module:crypto-deviceinfo[]} list of devices
+ * @return {Promise<module:crypto-deviceinfo[]>} list of devices
  */
-MatrixClient.prototype.getStoredDevicesForUser = function(userId) {
+MatrixClient.prototype.getStoredDevicesForUser = async function(userId) {
     if (this._crypto === null) {
         throw new Error("End-to-end encryption disabled");
     }
     return this._crypto.getStoredDevicesForUser(userId) || [];
 };
 
+/**
+ * Get the stored device key for a user id and device id
+ *
+ * @param {string} userId the user to list keys for.
+ * @param {string} deviceId unique identifier for the device
+ *
+ * @return {Promise<?module:crypto-deviceinfo>} device or null
+ */
+MatrixClient.prototype.getStoredDevice = async function(userId, deviceId) {
+    if (this._crypto === null) {
+        throw new Error("End-to-end encryption disabled");
+    }
+    return this._crypto.getStoredDevice(userId, deviceId) || null;
+};
+
 /**
  * Mark the given device as verified
  *
@@ -368,13 +490,15 @@ MatrixClient.prototype.getStoredDevicesForUser = function(userId) {
  * @param {boolean=} verified whether to mark the device as verified. defaults
  *   to 'true'.
  *
+ * @returns {Promise}
+ *
  * @fires module:client~event:MatrixClient"deviceVerificationChanged"
  */
 MatrixClient.prototype.setDeviceVerified = function(userId, deviceId, verified) {
     if (verified === undefined) {
         verified = true;
     }
-    _setDeviceVerification(this, userId, deviceId, verified, null);
+    return _setDeviceVerification(this, userId, deviceId, verified, null);
 };
 
 /**
@@ -386,13 +510,15 @@ MatrixClient.prototype.setDeviceVerified = function(userId, deviceId, verified)
  * @param {boolean=} blocked whether to mark the device as blocked. defaults
  *   to 'true'.
  *
+ * @returns {Promise}
+ *
  * @fires module:client~event:MatrixClient"deviceVerificationChanged"
  */
 MatrixClient.prototype.setDeviceBlocked = function(userId, deviceId, blocked) {
     if (blocked === undefined) {
         blocked = true;
     }
-    _setDeviceVerification(this, userId, deviceId, null, blocked);
+    return _setDeviceVerification(this, userId, deviceId, null, blocked);
 };
 
 /**
@@ -404,20 +530,24 @@ MatrixClient.prototype.setDeviceBlocked = function(userId, deviceId, blocked) {
  * @param {boolean=} known whether to mark the device as known. defaults
  *   to 'true'.
  *
+ * @returns {Promise}
+ *
  * @fires module:client~event:MatrixClient"deviceVerificationChanged"
  */
 MatrixClient.prototype.setDeviceKnown = function(userId, deviceId, known) {
     if (known === undefined) {
         known = true;
     }
-    _setDeviceVerification(this, userId, deviceId, null, null, known);
+    return _setDeviceVerification(this, userId, deviceId, null, null, known);
 };
 
-function _setDeviceVerification(client, userId, deviceId, verified, blocked, known) {
+async function _setDeviceVerification(
+    client, userId, deviceId, verified, blocked, known,
+) {
     if (!client._crypto) {
         throw new Error("End-to-End encryption disabled");
     }
-    const dev = client._crypto.setDeviceVerification(
+    const dev = await client._crypto.setDeviceVerification(
         userId, deviceId, verified, blocked, known,
     );
     client.emit("deviceVerificationChanged", userId, deviceId, dev);
@@ -454,9 +584,9 @@ MatrixClient.prototype.getGlobalBlacklistUnverifiedDevices = function() {
  *
  * @param {MatrixEvent} event event to be checked
  *
- * @return {module:crypto/deviceinfo?}
+ * @return {Promise<module:crypto/deviceinfo?>}
  */
-MatrixClient.prototype.getEventSenderDeviceInfo = function(event) {
+MatrixClient.prototype.getEventSenderDeviceInfo = async function(event) {
     if (!this._crypto) {
         return null;
     }
@@ -472,8 +602,8 @@ MatrixClient.prototype.getEventSenderDeviceInfo = function(event) {
  * @return {boolean} true if the sender of this event has been verified using
  * {@link module:client~MatrixClient#setDeviceVerified|setDeviceVerified}.
  */
-MatrixClient.prototype.isEventSenderVerified = function(event) {
-    const device = this.getEventSenderDeviceInfo(event);
+MatrixClient.prototype.isEventSenderVerified = async function(event) {
+    const device = await this.getEventSenderDeviceInfo(event);
     if (!device) {
         return false;
     }
@@ -484,14 +614,13 @@ MatrixClient.prototype.isEventSenderVerified = function(event) {
  * Enable end-to-end encryption for a room.
  * @param {string} roomId The room ID to enable encryption in.
  * @param {object} config The encryption config for the room.
- * @return {Object} A promise that will resolve when encryption is setup.
+ * @return {Promise} A promise that will resolve when encryption is set up.
  */
 MatrixClient.prototype.setRoomEncryption = function(roomId, config) {
     if (!this._crypto) {
         throw new Error("End-to-End encryption disabled");
     }
-    this._crypto.setRoomEncryption(roomId, config);
-    return q();
+    return this._crypto.setRoomEncryption(roomId, config);
 };
 
 /**
@@ -500,11 +629,28 @@ MatrixClient.prototype.setRoomEncryption = function(roomId, config) {
  * @return {bool} whether encryption is enabled.
  */
 MatrixClient.prototype.isRoomEncrypted = function(roomId) {
-    if (!this._crypto) {
+    const room = this.getRoom(roomId);
+    if (!room) {
+        // we don't know about this room, so can't determine if it should be
+        // encrypted. Let's assume not.
         return false;
     }
 
-    return this._crypto.isRoomEncrypted(roomId);
+    // if there is an 'm.room.encryption' event in this room, it should be
+    // encrypted (independently of whether we actually support encryption)
+    const ev = room.currentState.getStateEvents("m.room.encryption", "");
+    if (ev) {
+        return true;
+    }
+
+    // we don't have an m.room.encrypted event, but that might be because
+    // the server is hiding it from us. Check the store to see if it was
+    // previously encrypted.
+    if (!this._sessionStore) {
+        return false;
+    }
+
+    return Boolean(this._sessionStore.getEndToEndRoom(roomId));
 };
 
 /**
@@ -517,7 +663,7 @@ MatrixClient.prototype.isRoomEncrypted = function(roomId) {
  */
 MatrixClient.prototype.exportRoomKeys = function() {
     if (!this._crypto) {
-        return q.reject(new Error("End-to-end encryption disabled"));
+        return Promise.reject(new Error("End-to-end encryption disabled"));
     }
     return this._crypto.exportRoomKeys();
 };
@@ -526,49 +672,40 @@ MatrixClient.prototype.exportRoomKeys = function() {
  * Import a list of room keys previously exported by exportRoomKeys
  *
  * @param {Object[]} keys a list of session export objects
+ *
+ * @return {module:client.Promise} a promise which resolves when the keys
+ *    have been imported
  */
 MatrixClient.prototype.importRoomKeys = function(keys) {
     if (!this._crypto) {
         throw new Error("End-to-end encryption disabled");
     }
-    this._crypto.importRoomKeys(keys);
+    return this._crypto.importRoomKeys(keys);
+};
+
+// Group ops
+// =========
+// Operations on groups that come down the sync stream (ie. ones the
+// user is a member of or invited to)
+
+/**
+ * Get the group for the given group ID.
+ * This function will return a valid group for any group for which a Group event
+ * has been emitted.
+ * @param {string} groupId The group ID
+ * @return {Group} The Group or null if the group is not known or there is no data store.
+ */
+MatrixClient.prototype.getGroup = function(groupId) {
+    return this.store.getGroup(groupId);
 };
 
 /**
- * Decrypt a received event according to the algorithm specified in the event.
- *
- * @param {MatrixClient} client
- * @param {MatrixEvent} event
+ * Retrieve all known groups.
+ * @return {Groups[]} A list of groups, or an empty list if there is no data store.
  */
-function _decryptEvent(client, event) {
-    if (!client._crypto) {
-        _badEncryptedMessage(event, "Encryption not enabled");
-        return;
-    }
-
-    try {
-        client._crypto.decryptEvent(event);
-    } catch (e) {
-        console.warn(
-            `Error decrypting event (id=${event.getId()}): ${e}`,
-        );
-        if (!(e instanceof Crypto.DecryptionError)) {
-            throw e;
-        }
-        _badEncryptedMessage(event, e.message);
-        return;
-    }
-}
-
-function _badEncryptedMessage(event, reason) {
-    event.setClearData({
-        type: "m.room.message",
-        content: {
-            msgtype: "m.bad.encrypted",
-            body: "** Unable to decrypt: " + reason + " **",
-        },
-    });
-}
+MatrixClient.prototype.getGroups = function() {
+    return this.store.getGroups();
+};
 
 // Room ops
 // ========
@@ -642,6 +779,38 @@ MatrixClient.prototype.getAccountData = function(eventType) {
     return this.store.getAccountData(eventType);
 };
 
+/**
+ * Gets the users that are ignored by this client
+ * @returns {string[]} The array of users that are ignored (empty if none)
+ */
+MatrixClient.prototype.getIgnoredUsers = function() {
+    const event = this.getAccountData("m.ignored_user_list");
+    if (!event || !event.getContent() || !event.getContent()["ignored_users"]) return [];
+    return Object.keys(event.getContent()["ignored_users"]);
+};
+
+/**
+ * Sets the users that the current user should ignore.
+ * @param {string[]} userIds the user IDs to ignore
+ * @param {module:client.callback} [callback] Optional.
+ * @return {module:client.Promise} Resolves: Account data event
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixClient.prototype.setIgnoredUsers = function(userIds, callback) {
+    const content = {ignored_users: {}};
+    userIds.map((u) => content.ignored_users[u] = {});
+    return this.setAccountData("m.ignored_user_list", content, callback);
+};
+
+/**
+ * Gets whether or not a specific user is being ignored by this client.
+ * @param {string} userId the user ID to check
+ * @returns {boolean} true if the user is ignored, false otherwise
+ */
+MatrixClient.prototype.isUserIgnored = function(userId) {
+    return this.getIgnoredUsers().indexOf(userId) !== -1;
+};
+
 // Room operations
 // ===============
 
@@ -670,10 +839,10 @@ MatrixClient.prototype.joinRoom = function(roomIdOrAlias, opts, callback) {
 
     const room = this.getRoom(roomIdOrAlias);
     if (room && room.hasMembershipState(this.credentials.userId, "join")) {
-        return q(room);
+        return Promise.resolve(room);
     }
 
-    let sign_promise = q();
+    let sign_promise = Promise.resolve();
 
     if (opts.inviteSignUrl) {
         sign_promise = this._http.requestOtherUrl(
@@ -682,7 +851,7 @@ MatrixClient.prototype.joinRoom = function(roomIdOrAlias, opts, callback) {
         );
     }
 
-    const defer = q.defer();
+    const defer = Promise.defer();
 
     const self = this;
     sign_promise.then(function(signed_invite_object) {
@@ -701,7 +870,7 @@ MatrixClient.prototype.joinRoom = function(roomIdOrAlias, opts, callback) {
             // v2 will do this for us
             // return syncApi.syncRoom(room);
         }
-        return q(room);
+        return Promise.resolve(room);
     }).done(function(room) {
         _resolve(callback, defer, room);
     }, function(err) {
@@ -890,6 +1059,8 @@ MatrixClient.prototype.sendEvent = function(roomId, eventType, content, txnId,
         txnId = this.makeTxnId();
     }
 
+    console.log(`sendEvent of type ${eventType} in ${roomId} with txnId ${txnId}`);
+
     // we always construct a MatrixEvent when sending because the store and
     // scheduler use them. We'll extract the params back out if it turns out
     // the client has no scheduler or store.
@@ -919,21 +1090,20 @@ MatrixClient.prototype.sendEvent = function(roomId, eventType, content, txnId,
 // marks the event as sent/unsent
 // returns a promise which resolves with the result of the send request
 function _sendEvent(client, room, event, callback) {
-    // Add an extra q() to turn synchronous exceptions into promise rejections,
+    // Add an extra Promise.resolve() to turn synchronous exceptions into promise rejections,
     // so that we can handle synchronous and asynchronous exceptions with the
     // same code path.
-    return q().then(function() {
-        let encryptionPromise = null;
-        if (client._crypto) {
-            encryptionPromise = client._crypto.encryptEventIfNeeded(event, room);
+    return Promise.resolve().then(function() {
+        const encryptionPromise = _encryptEventIfNeeded(client, event, room);
+
+        if (!encryptionPromise) {
+            return null;
         }
-        if (encryptionPromise) {
-            _updatePendingEventStatus(room, event, EventStatus.ENCRYPTING);
-            encryptionPromise = encryptionPromise.then(function() {
-                _updatePendingEventStatus(room, event, EventStatus.SENDING);
-            });
-        }
-        return encryptionPromise;
+
+        _updatePendingEventStatus(room, event, EventStatus.ENCRYPTING);
+        return encryptionPromise.then(() => {
+            _updatePendingEventStatus(room, event, EventStatus.SENDING);
+        });
     }).then(function() {
         let promise;
         // this event may be queued
@@ -980,6 +1150,43 @@ function _sendEvent(client, room, event, callback) {
     });
 }
 
+/**
+ * Encrypt an event according to the configuration of the room, if necessary.
+ *
+ * @param {MatrixClient} client
+ *
+ * @param {module:models/event.MatrixEvent} event  event to be sent
+ *
+ * @param {module:models/room?} room destination room. Null if the destination
+ *     is not a room we have seen over the sync pipe.
+ *
+ * @return {module:client.Promise?} Promise which resolves when the event has been
+ *     encrypted, or null if nothing was needed
+ */
+
+function _encryptEventIfNeeded(client, event, room) {
+    if (event.isEncrypted()) {
+        // this event has already been encrypted; this happens if the
+        // encryption step succeeded, but the send step failed on the first
+        // attempt.
+        return null;
+    }
+
+    if (!client.isRoomEncrypted(event.getRoomId())) {
+        // looks like this room isn't encrypted.
+        return null;
+    }
+
+    if (!client._crypto) {
+        throw new Error(
+            "This room is configured to use encryption, but your client does " +
+            "not support encryption.",
+        );
+    }
+
+    return client._crypto.encryptEvent(event, room);
+}
+
 function _updatePendingEventStatus(room, event, newStatus) {
     if (room) {
         room.updatePendingEvent(event, newStatus);
@@ -1014,7 +1221,12 @@ function _sendEventHttpRequest(client, event) {
 
     return client._http.authedRequest(
         undefined, "PUT", path, undefined, event.getWireContent(),
-    );
+    ).then((res) => {
+        console.log(
+            `Event sent to ${event.getRoomId()} with event id ${res.event_id}`,
+        );
+        return res;
+    });
 }
 
 /**
@@ -1171,7 +1383,7 @@ MatrixClient.prototype.sendHtmlEmote = function(roomId, body, htmlBody, callback
  */
 MatrixClient.prototype.sendReceipt = function(event, receiptType, callback) {
     if (this.isGuest()) {
-        return q({}); // guests cannot send receipts so don't bother.
+        return Promise.resolve({}); // guests cannot send receipts so don't bother.
     }
 
     const path = utils.encodeUri("/rooms/$roomId/receipt/$receiptType/$eventId", {
@@ -1248,7 +1460,7 @@ MatrixClient.prototype.getUrlPreview = function(url, ts, callback) {
     const key = ts + "_" + url;
     const og = this.urlPreviewCache[key];
     if (og) {
-        return q(og);
+        return Promise.resolve(og);
     }
 
     const self = this;
@@ -1274,7 +1486,7 @@ MatrixClient.prototype.getUrlPreview = function(url, ts, callback) {
  */
 MatrixClient.prototype.sendTyping = function(roomId, isTyping, timeoutMs, callback) {
     if (this.isGuest()) {
-        return q({}); // guests cannot send typing notifications so don't bother.
+        return Promise.resolve({}); // guests cannot send typing notifications so don't bother.
     }
 
     const path = utils.encodeUri("/rooms/$roomId/typing/$userId", {
@@ -1335,7 +1547,7 @@ MatrixClient.prototype.inviteByThreePid = function(roomId, medium, address, call
 
     let identityServerUrl = this.getIdentityServerUrl();
     if (!identityServerUrl) {
-        return q.reject(new MatrixError({
+        return Promise.reject(new MatrixError({
             error: "No supplied identity server URL",
             errcode: "ORG.MATRIX.JSSDK_MISSING_PARAM",
         }));
@@ -1675,13 +1887,13 @@ MatrixClient.prototype.scrollback = function(room, limit, callback) {
     }
 
     if (room.oldState.paginationToken === null) {
-        return q(room); // already at the start.
+        return Promise.resolve(room); // already at the start.
     }
     // attempt to grab more events from the store first
     const numAdded = this.store.scrollback(room, limit).length;
     if (numAdded === limit) {
         // store contained everything we needed.
-        return q(room);
+        return Promise.resolve(room);
     }
     // reduce the required number of events appropriately
     limit = limit - numAdded;
@@ -1694,7 +1906,7 @@ MatrixClient.prototype.scrollback = function(room, limit, callback) {
         limit: limit,
         dir: 'b',
     };
-    const defer = q.defer();
+    const defer = Promise.defer();
     info = {
         promise: defer.promise,
         errorTs: null,
@@ -1702,7 +1914,7 @@ MatrixClient.prototype.scrollback = function(room, limit, callback) {
     const self = this;
     // wait for a time before doing this request
     // (which may be 0 in order not to special case the code paths)
-    q.delay(timeToWaitMs).then(function() {
+    Promise.delay(timeToWaitMs).then(function() {
         return self._http.authedRequest(callback, "GET", path, params);
     }).done(function(res) {
         const matrixEvents = utils.map(res.chunk, _PojoToMatrixEventMapper(self));
@@ -1745,7 +1957,7 @@ MatrixClient.prototype.paginateEventContext = function(eventContext, opts) {
     const token = eventContext.getPaginateToken(backwards);
     if (!token) {
         // no more results.
-        return q.reject(new Error("No paginate token"));
+        return Promise.reject(new Error("No paginate token"));
     }
 
     const dir = backwards ? 'b' : 'f';
@@ -1814,7 +2026,7 @@ MatrixClient.prototype.getEventTimeline = function(timelineSet, eventId) {
     }
 
     if (timelineSet.getTimelineForEvent(eventId)) {
-        return q(timelineSet.getTimelineForEvent(eventId));
+        return Promise.resolve(timelineSet.getTimelineForEvent(eventId));
     }
 
     const path = utils.encodeUri(
@@ -1902,7 +2114,7 @@ MatrixClient.prototype.paginateEventTimeline = function(eventTimeline, opts) {
     const token = eventTimeline.getPaginationToken(dir);
     if (!token) {
         // no token - no results.
-        return q(false);
+        return Promise.resolve(false);
     }
 
     const pendingRequest = eventTimeline._paginationRequests[dir];
@@ -2022,7 +2234,7 @@ MatrixClient.prototype.resetNotifTimelineSet = function() {
     // know about /notifications, so we have no choice but to start paginating
     // from the current point in time.  This may well overlap with historical
     // notifs which are then inserted into the timeline by /sync responses.
-    this._notifTimelineSet.resetLiveTimeline('end', true);
+    this._notifTimelineSet.resetLiveTimeline('end', null);
 
     // we could try to paginate a single event at this point in order to get
     // a more valid pagination token, but it just ends up with an out of order
@@ -2079,14 +2291,14 @@ MatrixClient.prototype.setGuestAccess = function(roomId, opts) {
         guest_access: opts.allowJoin ? "can_join" : "forbidden",
     });
 
-    let readPromise = q();
+    let readPromise = Promise.resolve();
     if (opts.allowRead) {
         readPromise = this.sendStateEvent(roomId, "m.room.history_visibility", {
             history_visibility: "world_readable",
         });
     }
 
-    return q.all(readPromise, writePromise);
+    return Promise.all([readPromise, writePromise]);
 };
 
 // Registration/Login operations
@@ -2354,7 +2566,7 @@ MatrixClient.prototype.setRoomMutePushRule = function(scope, roomId, mute) {
         } else if (!hasDontNotifyRule) {
             // Remove the existing one before setting the mute push rule
             // This is a workaround to SYN-590 (Push rule update fails)
-            deferred = q.defer();
+            deferred = Promise.defer();
             this.deletePushRule(scope, "room", roomPushRule.rule_id)
             .done(function() {
                 self.addPushRule(scope, "room", roomId, {
@@ -2374,7 +2586,7 @@ MatrixClient.prototype.setRoomMutePushRule = function(scope, roomId, mute) {
 
     if (deferred) {
         // Update this.pushRules when the operation completes
-        const ruleRefreshDeferred = q.defer();
+        const ruleRefreshDeferred = Promise.defer();
         deferred.done(function() {
             self.getPushRules().done(function(result) {
                 self.pushRules = result;
@@ -2488,7 +2700,7 @@ MatrixClient.prototype.backPaginateRoomEventsSearch = function(searchResults) {
     // nicely with HTTP errors.
 
     if (!searchResults.next_batch) {
-        return q.reject(new Error("Cannot backpaginate event search any further"));
+        return Promise.reject(new Error("Cannot backpaginate event search any further"));
     }
 
     if (searchResults.pendingRequest) {
@@ -2557,7 +2769,7 @@ MatrixClient.prototype._processRoomEventsSearch = function(searchResults, respon
 MatrixClient.prototype.syncLeftRooms = function() {
     // Guard against multiple calls whilst ongoing and multiple calls post success
     if (this._syncedLeftRooms) {
-        return q([]); // don't call syncRooms again if it succeeded.
+        return Promise.resolve([]); // don't call syncRooms again if it succeeded.
     }
     if (this._syncLeftRoomsPromise) {
         return this._syncLeftRoomsPromise; // return the ongoing request
@@ -2616,7 +2828,7 @@ MatrixClient.prototype.getFilter = function(userId, filterId, allowCached) {
     if (allowCached) {
         const filter = this.store.getFilter(userId, filterId);
         if (filter) {
-            return q(filter);
+            return Promise.resolve(filter);
         }
     }
 
@@ -2645,7 +2857,7 @@ MatrixClient.prototype.getFilter = function(userId, filterId, allowCached) {
  */
 MatrixClient.prototype.getOrCreateFilter = function(filterName, filter) {
     const filterId = this.store.getFilterIdByName(filterName);
-    let promise = q();
+    let promise = Promise.resolve();
     const self = this;
 
     if (filterId) {
@@ -2660,7 +2872,7 @@ MatrixClient.prototype.getOrCreateFilter = function(filterName, filter) {
                 // super, just use that.
                 // debuglog("Using existing filter ID %s: %s", filterId,
                 //          JSON.stringify(oldDef));
-                return q(filterId);
+                return Promise.resolve(filterId);
             }
             // debuglog("Existing filter ID %s: %s; new filter: %s",
             //          filterId, JSON.stringify(oldDef), JSON.stringify(newDef));
@@ -2799,6 +3011,7 @@ MatrixClient.prototype.startClient = function(opts) {
 
     if (this._crypto) {
         this._crypto.uploadDeviceKeys().done();
+        this._crypto.start();
     }
 
     // periodically poll for turn servers if we support voip
@@ -2831,12 +3044,20 @@ MatrixClient.prototype.startClient = function(opts) {
  * clean shutdown.
  */
 MatrixClient.prototype.stopClient = function() {
+    console.log('stopping MatrixClient');
+
     this.clientRunning = false;
     // TODO: f.e. Room => self.store.storeRoom(room) ?
     if (this._syncApi) {
         this._syncApi.stop();
         this._syncApi = null;
     }
+    if (this._crypto) {
+        this._crypto.stop();
+    }
+    if (this._peekSync) {
+        this._peekSync.stopPeeking();
+    }
     global.clearTimeout(this._checkTurnServersTimeoutID);
 };
 
@@ -2888,6 +3109,10 @@ function setupCallEventHandler(client) {
             // now loop through the buffer chronologically and inject them
             callEventBuffer.forEach(function(e) {
                 if (ignoreCallIds[e.getContent().call_id]) {
+                    console.log(
+                        'Ignoring previously answered/hungup call ' +
+                            e.getContent().call_id,
+                    );
                     return;
                 }
                 callEventHandler(e);
@@ -2896,20 +3121,25 @@ function setupCallEventHandler(client) {
         }
     });
 
-    client.on("event", function(event) {
-        if (!isClientPrepared) {
-            if (event.getType().indexOf("m.call.") === 0) {
-                callEventBuffer.push(event);
+    client.on("event", onEvent);
+
+    function onEvent(event) {
+        if (event.getType().indexOf("m.call.") !== 0) {
+            // not a call event
+            if (event.isBeingDecrypted() || event.isDecryptionFailure()) {
+                // not *yet* a call event, but might become one...
+                event.once("Event.decrypted", onEvent);
             }
             return;
         }
+        if (!isClientPrepared) {
+            callEventBuffer.push(event);
+            return;
+        }
         callEventHandler(event);
-    });
+    }
 
     function callEventHandler(event) {
-        if (event.getType().indexOf("m.call.") !== 0) {
-            return; // not a call event
-        }
         const content = event.getContent();
         let call = content.call_id ? client.callList[content.call_id] : undefined;
         let i;
@@ -2935,7 +3165,9 @@ function setupCallEventHandler(client) {
                 );
             }
 
-            call = webRtcCall.createNewMatrixCall(client, event.getRoomId());
+            call = webRtcCall.createNewMatrixCall(client, event.getRoomId(), {
+                forceTURN: client._forceTURN,
+            });
             if (!call) {
                 console.log(
                     "Incoming call ID " + content.call_id + " but this client " +
@@ -3101,7 +3333,10 @@ function _PojoToMatrixEventMapper(client) {
     function mapper(plainOldJsObject) {
         const event = new MatrixEvent(plainOldJsObject);
         if (event.isEncrypted()) {
-            _decryptEvent(client, event);
+            client.reEmitter.reEmit(event, [
+                "Event.decrypted",
+            ]);
+            event.attemptDecryption(client._crypto);
         }
         return event;
     }
@@ -3282,6 +3517,17 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
  * });
  */
 
+ /**
+ * Fires whenever the sdk learns about a new group. <strong>This event
+ * is experimental and may change.</strong>
+ * @event module:client~MatrixClient#"Group"
+ * @param {Group} group The newly created, fully populated group.
+ * @example
+ * matrixClient.on("Group", function(group){
+ *   var groupId = group.groupId;
+ * });
+ */
+
  /**
  * Fires whenever a new Room is added. This will fire when you are invited to a
  * room, as well as when you join a room. <strong>This event is experimental and

src/crypto/DeviceList.js

@@ -21,13 +21,35 @@ limitations under the License.
  * Manages the list of other users' devices
  */
 
-import q from 'q';
+import Promise from 'bluebird';
 
 import DeviceInfo from './deviceinfo';
 import olmlib from './olmlib';
 
+
+/* State transition diagram for DeviceList._deviceTrackingStatus
+ *
+ *                                |
+ *     stopTrackingDeviceList     V
+ *   +---------------------> NOT_TRACKED
+ *   |                            |
+ *   +<--------------------+      | startTrackingDeviceList
+ *   |                     |      V
+ *   |   +-------------> PENDING_DOWNLOAD <--------------------+-+
+ *   |   |                      ^ |                            | |
+ *   |   | restart     download | |  start download            | | invalidateUserDeviceList
+ *   |   | client        failed | |                            | |
+ *   |   |                      | V                            | |
+ *   |   +------------ DOWNLOAD_IN_PROGRESS -------------------+ |
+ *   |                    |       |                              |
+ *   +<-------------------+       |  download successful         |
+ *   ^                            V                              |
+ *   +----------------------- UP_TO_DATE ------------------------+
+ */
+
+
 // constants for DeviceList._deviceTrackingStatus
-// const TRACKING_STATUS_NOT_TRACKED = 0;
+const TRACKING_STATUS_NOT_TRACKED = 0;
 const TRACKING_STATUS_PENDING_DOWNLOAD = 1;
 const TRACKING_STATUS_DOWNLOAD_IN_PROGRESS = 2;
 const TRACKING_STATUS_UP_TO_DATE = 3;
@@ -76,6 +98,10 @@ export default class DeviceList {
             if (this._keyDownloadsInProgressByUser[u]) {
                 // already a key download in progress/queued for this user; its results
                 // will be good enough for us.
+                console.log(
+                    `downloadKeys: already have a download in progress for ` +
+                    `${u}: awaiting its result`,
+                );
                 promises.push(this._keyDownloadsInProgressByUser[u]);
             } else if (forceDownload || trackingStatus != TRACKING_STATUS_UP_TO_DATE) {
                 usersToDownload.push(u);
@@ -88,7 +114,11 @@ export default class DeviceList {
             promises.push(downloadPromise);
         }
 
-        return q.all(promises).then(() => {
+        if (promises.length === 0) {
+            console.log("downloadKeys: already have all necessary keys");
+        }
+
+        return Promise.all(promises).then(() => {
             return this._getDevicesFromStore(userIds);
         });
     }
@@ -228,6 +258,26 @@ export default class DeviceList {
         // refreshOutdatedDeviceLists.
     }
 
+    /**
+     * Mark the given user as no longer being tracked for device-list updates.
+     *
+     * This won't affect any in-progress downloads, which will still go on to
+     * complete; it will just mean that we don't think that we have an up-to-date
+     * list for future calls to downloadKeys.
+     *
+     * @param {String} userId
+     */
+    stopTrackingDeviceList(userId) {
+        if (this._deviceTrackingStatus[userId]) {
+            console.log('No longer tracking device list for ' + userId);
+            this._deviceTrackingStatus[userId] = TRACKING_STATUS_NOT_TRACKED;
+        }
+        // we don't yet persist the tracking status, since there may be a lot
+        // of calls; instead we wait for the forthcoming
+        // refreshOutdatedDeviceLists.
+    }
+
+
     /**
      * Mark the cached device list for the given user outdated.
      *
@@ -275,9 +325,6 @@ export default class DeviceList {
                 usersToDownload.push(userId);
             }
         }
-        if (usersToDownload.length == 0) {
-            return;
-        }
 
         // we didn't persist the tracking status during
         // invalidateUserDeviceList, so do it now.
@@ -301,7 +348,7 @@ export default class DeviceList {
     _doKeyDownload(users) {
         if (users.length === 0) {
             // nothing to do
-            return q();
+            return Promise.resolve();
         }
 
         const prom = this._serialiser.updateDevicesForUsers(
@@ -408,7 +455,7 @@ class DeviceListUpdateSerialiser {
         this._nextSyncToken = syncToken;
 
         if (!this._queuedQueryDeferred) {
-            this._queuedQueryDeferred = q.defer();
+            this._queuedQueryDeferred = Promise.defer();
         }
 
         if (this._downloadInProgress) {
@@ -451,10 +498,10 @@ class DeviceListUpdateSerialiser {
             //
             // of course we ought to do this in a web worker or similar, but
             // this serves as an easy solution for now.
-            let prom = q();
+            let prom = Promise.resolve();
             for (const userId of downloadUsers) {
                 prom = prom.delay(5).then(() => {
-                    this._processQueryResponseForUser(userId, dk[userId]);
+                    return this._processQueryResponseForUser(userId, dk[userId]);
                 });
             }
 
@@ -471,14 +518,14 @@ class DeviceListUpdateSerialiser {
             }
         }, (e) => {
             console.warn('Error downloading keys for ' + downloadUsers + ':', e);
-            this._downloadInProgressInProgress = false;
+            this._downloadInProgress = false;
             deferred.reject(e);
         });
 
         return deferred.promise;
     }
 
-    _processQueryResponseForUser(userId, response) {
+    async _processQueryResponseForUser(userId, response) {
         console.log('got keys for ' + userId + ':', response);
 
         // map from deviceid -> deviceinfo for this user
@@ -491,7 +538,7 @@ class DeviceListUpdateSerialiser {
             });
         }
 
-        _updateStoredDeviceKeysForUser(
+        await _updateStoredDeviceKeysForUser(
             this._olmDevice, userId, userStore, response || {},
         );
 
@@ -508,7 +555,7 @@ class DeviceListUpdateSerialiser {
 }
 
 
-function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
+async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
         userResult) {
     let updated = false;
 
@@ -546,7 +593,7 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
             continue;
         }
 
-        if (_storeDeviceKeys(_olmDevice, userStore, deviceResult)) {
+        if (await _storeDeviceKeys(_olmDevice, userStore, deviceResult)) {
             updated = true;
         }
     }
@@ -557,9 +604,9 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
 /*
  * Process a device in a /query response, and add it to the userStore
  *
- * returns true if a change was made, else false
+ * returns (a promise for) true if a change was made, else false
  */
- function _storeDeviceKeys(_olmDevice, userStore, deviceResult) {
+async function _storeDeviceKeys(_olmDevice, userStore, deviceResult) {
     if (!deviceResult.keys) {
         // no keys?
         return false;
@@ -579,7 +626,7 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
     const unsigned = deviceResult.unsigned || {};
 
     try {
-        olmlib.verifySignature(_olmDevice, deviceResult, userId, deviceId, signKey);
+        await olmlib.verifySignature(_olmDevice, deviceResult, userId, deviceId, signKey);
     } catch (e) {
         console.warn("Unable to verify signature on device " +
             userId + ":" + deviceId + ":" + e);

src/crypto/OlmDevice.js

@@ -20,11 +20,9 @@ limitations under the License.
  *
  * @module crypto/OlmDevice
  */
-const Olm = require("olm");
+const Olm = global.Olm;
 if (!Olm) {
-    // this happens if we were loaded via browserify and the Olm module was not
-    // loaded.
-    throw new Error("Olm is not defined");
+    throw new Error("global.Olm is not defined");
 }
 const utils = require("../utils");
 
@@ -57,6 +55,8 @@ function checkPayloadLength(payloadString) {
  *
  * @typedef {Object} module:crypto/OlmDevice.MegolmSessionData
  * @property {String} sender_key  Sender's Curve25519 device key
+ * @property {String[]} forwarding_curve25519_key_chain Devices which forwarded
+ *     this session to us (normally empty).
  * @property {Object<string, string>} sender_claimed_keys Other keys the sender claims.
  * @property {String} room_id     Room this session is used in
  * @property {String} session_id  Unique id for the session
@@ -83,17 +83,10 @@ function OlmDevice(sessionStore) {
     this._sessionStore = sessionStore;
     this._pickleKey = "DEFAULT_KEY";
 
-    let e2eKeys;
-    const account = new Olm.Account();
-    try {
-        _initialise_account(this._sessionStore, this._pickleKey, account);
-        e2eKeys = JSON.parse(account.identity_keys());
-    } finally {
-        account.free();
-    }
-
-    this.deviceCurve25519Key = e2eKeys.curve25519;
-    this.deviceEd25519Key = e2eKeys.ed25519;
+    // don't know these until we load the account from storage in init()
+    this.deviceCurve25519Key = null;
+    this.deviceEd25519Key = null;
+    this._maxOneTimeKeys = null;
 
     // we don't bother stashing outboundgroupsessions in the sessionstore -
     // instead we keep them here.
@@ -103,15 +96,47 @@ function OlmDevice(sessionStore) {
     // This partially mitigates a replay attack where a MITM resends a group
     // message into the room.
     //
-    // TODO: If we ever remove an event from memory we will also need to remove
-    // it from this map. Otherwise if we download the event from the server we
-    // will think that it is a duplicate.
+    // When we decrypt a message and the message index matches a previously
+    // decrypted message, one possible cause of that is that we are decrypting
+    // the same event, and may not indicate an actual replay attack.  For
+    // example, this could happen if we receive events, forget about them, and
+    // then re-fetch them when we backfill.  So we store the event ID and
+    // timestamp corresponding to each message index when we first decrypt it,
+    // and compare these against the event ID and timestamp every time we use
+    // that same index.  If they match, then we're probably decrypting the same
+    // event and we don't consider it a replay attack.
     //
     // Keys are strings of form "<senderKey>|<session_id>|<message_index>"
-    // Values are true.
+    // Values are objects of the form "{id: <event id>, timestamp: <ts>}"
     this._inboundGroupSessionMessageIndexes = {};
 }
 
+/**
+ * Initialise the OlmAccount. This must be called before any other operations
+ * on the OlmDevice.
+ *
+ * Attempts to load the OlmAccount from localStorage, or creates one if none is
+ * found.
+ *
+ * Reads the device keys from the OlmAccount object.
+ */
+OlmDevice.prototype.init = async function() {
+    let e2eKeys;
+    const account = new Olm.Account();
+    try {
+        _initialise_account(this._sessionStore, this._pickleKey, account);
+        e2eKeys = JSON.parse(account.identity_keys());
+
+        this._maxOneTimeKeys = account.max_number_of_one_time_keys();
+    } finally {
+        account.free();
+    }
+
+    this.deviceCurve25519Key = e2eKeys.curve25519;
+    this.deviceEd25519Key = e2eKeys.ed25519;
+};
+
+
 function _initialise_account(sessionStore, pickleKey, account) {
     const e2eAccount = sessionStore.getEndToEndAccount();
     if (e2eAccount !== null) {
@@ -222,9 +247,9 @@ OlmDevice.prototype._getUtility = function(func) {
  * Signs a message with the ed25519 key for this account.
  *
  * @param {string} message  message to be signed
- * @return {string} base64-encoded signature
+ * @return {Promise<string>} base64-encoded signature
  */
-OlmDevice.prototype.sign = function(message) {
+OlmDevice.prototype.sign = async function(message) {
     return this._getAccount(function(account) {
         return account.sign(message);
     });
@@ -237,7 +262,7 @@ OlmDevice.prototype.sign = function(message) {
  * <tt>curve25519</tt>, which is itself an object mapping key id to Curve25519
  * key.
  */
-OlmDevice.prototype.getOneTimeKeys = function() {
+OlmDevice.prototype.getOneTimeKeys = async function() {
     return this._getAccount(function(account) {
         return JSON.parse(account.one_time_keys());
     });
@@ -250,15 +275,13 @@ OlmDevice.prototype.getOneTimeKeys = function() {
  * @return {number} number of keys
  */
 OlmDevice.prototype.maxNumberOfOneTimeKeys = function() {
-    return this._getAccount(function(account) {
-        return account.max_number_of_one_time_keys();
-    });
+    return this._maxOneTimeKeys;
 };
 
 /**
  * Marks all of the one-time keys as published.
  */
-OlmDevice.prototype.markKeysAsPublished = function() {
+OlmDevice.prototype.markKeysAsPublished = async function() {
     const self = this;
     this._getAccount(function(account) {
         account.mark_keys_as_published();
@@ -271,7 +294,7 @@ OlmDevice.prototype.markKeysAsPublished = function() {
  *
  * @param {number} numKeys number of keys to generate
  */
-OlmDevice.prototype.generateOneTimeKeys = function(numKeys) {
+OlmDevice.prototype.generateOneTimeKeys = async function(numKeys) {
     const self = this;
     this._getAccount(function(account) {
         account.generate_one_time_keys(numKeys);
@@ -288,7 +311,7 @@ OlmDevice.prototype.generateOneTimeKeys = function(numKeys) {
  * @param {string} theirOneTimeKey  remote user's one-time Curve25519 key
  * @return {string} sessionId for the outbound session.
  */
-OlmDevice.prototype.createOutboundSession = function(
+OlmDevice.prototype.createOutboundSession = async function(
     theirIdentityKey, theirOneTimeKey,
 ) {
     const self = this;
@@ -318,7 +341,7 @@ OlmDevice.prototype.createOutboundSession = function(
  * @raises {Error} if the received message was not valid (for instance, it
  *     didn't use a valid one-time key).
  */
-OlmDevice.prototype.createInboundSession = function(
+OlmDevice.prototype.createInboundSession = async function(
     theirDeviceIdentityKey, message_type, ciphertext,
 ) {
     if (message_type !== 0) {
@@ -353,9 +376,9 @@ OlmDevice.prototype.createInboundSession = function(
  *
  * @param {string} theirDeviceIdentityKey Curve25519 identity key for the
  *     remote device
- * @return {string[]}  a list of known session ids for the device
+ * @return {Promise<string[]>}  a list of known session ids for the device
  */
-OlmDevice.prototype.getSessionIdsForDevice = function(theirDeviceIdentityKey) {
+OlmDevice.prototype.getSessionIdsForDevice = async function(theirDeviceIdentityKey) {
     const sessions = this._sessionStore.getEndToEndSessions(
         theirDeviceIdentityKey,
     );
@@ -367,10 +390,10 @@ OlmDevice.prototype.getSessionIdsForDevice = function(theirDeviceIdentityKey) {
  *
  * @param {string} theirDeviceIdentityKey Curve25519 identity key for the
  *     remote device
- * @return {string?}  session id, or null if no established session
+ * @return {Promise<?string>}  session id, or null if no established session
  */
-OlmDevice.prototype.getSessionIdForDevice = function(theirDeviceIdentityKey) {
-    const sessionIds = this.getSessionIdsForDevice(theirDeviceIdentityKey);
+OlmDevice.prototype.getSessionIdForDevice = async function(theirDeviceIdentityKey) {
+    const sessionIds = await this.getSessionIdsForDevice(theirDeviceIdentityKey);
     if (sessionIds.length === 0) {
         return null;
     }
@@ -390,8 +413,8 @@ OlmDevice.prototype.getSessionIdForDevice = function(theirDeviceIdentityKey) {
  * @param {string} deviceIdentityKey Curve25519 identity key for the device
  * @return {Array.<{sessionId: string, hasReceivedMessage: Boolean}>}
  */
-OlmDevice.prototype.getSessionInfoForDevice = function(deviceIdentityKey) {
-    const sessionIds = this.getSessionIdsForDevice(deviceIdentityKey);
+OlmDevice.prototype.getSessionInfoForDevice = async function(deviceIdentityKey) {
+    const sessionIds = await this.getSessionIdsForDevice(deviceIdentityKey);
     sessionIds.sort();
 
     const info = [];
@@ -419,9 +442,9 @@ OlmDevice.prototype.getSessionInfoForDevice = function(deviceIdentityKey) {
  * @param {string} sessionId  the id of the active session
  * @param {string} payloadString  payload to be encrypted and sent
  *
- * @return {string} ciphertext
+ * @return {Promise<string>} ciphertext
  */
-OlmDevice.prototype.encryptMessage = function(
+OlmDevice.prototype.encryptMessage = async function(
     theirDeviceIdentityKey, sessionId, payloadString,
 ) {
     const self = this;
@@ -444,9 +467,9 @@ OlmDevice.prototype.encryptMessage = function(
  * @param {number} message_type  message_type field from the received message
  * @param {string} ciphertext base64-encoded body from the received message
  *
- * @return {string} decrypted payload.
+ * @return {Promise<string>} decrypted payload.
  */
-OlmDevice.prototype.decryptMessage = function(
+OlmDevice.prototype.decryptMessage = async function(
     theirDeviceIdentityKey, sessionId, message_type, ciphertext,
 ) {
     const self = this;
@@ -468,10 +491,10 @@ OlmDevice.prototype.decryptMessage = function(
  * @param {number} message_type  message_type field from the received message
  * @param {string} ciphertext base64-encoded body from the received message
  *
- * @return {boolean} true if the received message is a prekey message which matches
+ * @return {Promise<boolean>} true if the received message is a prekey message which matches
  *    the given session.
  */
-OlmDevice.prototype.matchesSession = function(
+OlmDevice.prototype.matchesSession = async function(
     theirDeviceIdentityKey, sessionId, message_type, ciphertext,
 ) {
     if (message_type !== 0) {
@@ -582,27 +605,30 @@ OlmDevice.prototype.getOutboundGroupSessionKey = function(sessionId) {
 // Inbound group session
 // =====================
 
+/**
+ * data stored in the session store about an inbound group session
+ *
+ * @typedef {Object} InboundGroupSessionData
+ * @property {string} room_Id
+ * @property {string} session   pickled Olm.InboundGroupSession
+ * @property {Object<string, string>} keysClaimed
+ * @property {Array<string>} forwardingCurve25519KeyChain  Devices involved in forwarding
+ *     this session to us (normally empty).
+ */
+
 /**
  * store an InboundGroupSession in the session store
  *
- * @param {string} roomId
  * @param {string} senderCurve25519Key
  * @param {string} sessionId
- * @param {Olm.InboundGroupSession} session
- * @param {object} keysClaimed Other keys the sender claims.
+ * @param {InboundGroupSessionData} sessionData
  * @private
  */
 OlmDevice.prototype._saveInboundGroupSession = function(
-    roomId, senderCurve25519Key, sessionId, session, keysClaimed,
+    senderCurve25519Key, sessionId, sessionData,
 ) {
-    const r = {
-        room_id: roomId,
-        session: session.pickle(this._pickleKey),
-        keysClaimed: keysClaimed,
-    };
-
     this._sessionStore.storeEndToEndInboundGroupSession(
-        senderCurve25519Key, sessionId, JSON.stringify(r),
+        senderCurve25519Key, sessionId, JSON.stringify(sessionData),
     );
 };
 
@@ -612,8 +638,8 @@ OlmDevice.prototype._saveInboundGroupSession = function(
  * @param {string} roomId
  * @param {string} senderKey
  * @param {string} sessionId
- * @param {function(Olm.InboundGroupSession, Object<string, string>): T} func
- *   function to call. Second argument is the map of keys claimed by the session.
+ * @param {function(Olm.InboundGroupSession, InboundGroupSessionData): T} func
+ *   function to call.
  *
  * @return {null} the sessionId is unknown
  *
@@ -647,7 +673,7 @@ OlmDevice.prototype._getInboundGroupSession = function(
     const session = new Olm.InboundGroupSession();
     try {
         session.unpickle(this._pickleKey, r.session);
-        return func(session, r.keysClaimed || {});
+        return func(session, r);
     } finally {
         session.free();
     }
@@ -658,17 +684,23 @@ OlmDevice.prototype._getInboundGroupSession = function(
  *
  * @param {string} roomId     room in which this session will be used
  * @param {string} senderKey  base64-encoded curve25519 key of the sender
+ * @param {Array<string>} forwardingCurve25519KeyChain  Devices involved in forwarding
+ *     this session to us.
  * @param {string} sessionId  session identifier
  * @param {string} sessionKey base64-encoded secret key
  * @param {Object<string, string>} keysClaimed Other keys the sender claims.
+ * @param {boolean} exportFormat true if the megolm keys are in export format
+ *    (ie, they lack an ed25519 signature)
  */
-OlmDevice.prototype.addInboundGroupSession = function(
-    roomId, senderKey, sessionId, sessionKey, keysClaimed,
+OlmDevice.prototype.addInboundGroupSession = async function(
+    roomId, senderKey, forwardingCurve25519KeyChain,
+    sessionId, sessionKey, keysClaimed,
+    exportFormat,
 ) {
     const self = this;
 
     /* if we already have this session, consider updating it */
-    function updateSession(session) {
+    function updateSession(session, sessionData) {
         console.log("Update for megolm session " + senderKey + "/" + sessionId);
         // for now we just ignore updates. TODO: implement something here
 
@@ -686,14 +718,26 @@ OlmDevice.prototype.addInboundGroupSession = function(
     // new session.
     const session = new Olm.InboundGroupSession();
     try {
-        session.create(sessionKey);
+        if (exportFormat) {
+            session.import_session(sessionKey);
+        } else {
+            session.create(sessionKey);
+        }
         if (sessionId != session.session_id()) {
             throw new Error(
                 "Mismatched group session ID from senderKey: " + senderKey,
             );
         }
+
+        const sessionData = {
+            room_id: roomId,
+            session: session.pickle(this._pickleKey),
+            keysClaimed: keysClaimed,
+            forwardingCurve25519KeyChain: forwardingCurve25519KeyChain,
+        };
+
         self._saveInboundGroupSession(
-            roomId, senderKey, sessionId, session, keysClaimed,
+            senderKey, sessionId, sessionData,
         );
     } finally {
         session.free();
@@ -706,9 +750,9 @@ OlmDevice.prototype.addInboundGroupSession = function(
  *
  * @param {module:crypto/OlmDevice.MegolmSessionData} data  session data
  */
-OlmDevice.prototype.importInboundGroupSession = function(data) {
+OlmDevice.prototype.importInboundGroupSession = async function(data) {
     /* if we already have this session, consider updating it */
-    function updateSession(session) {
+    function updateSession(session, sessionData) {
         console.log("Update for megolm session " + data.sender_key + "|" +
                     data.session_id);
         // for now we just ignore updates. TODO: implement something here
@@ -733,9 +777,16 @@ OlmDevice.prototype.importInboundGroupSession = function(data) {
                 "Mismatched group session ID from senderKey: " + data.sender_key,
             );
         }
+
+        const sessionData = {
+            room_id: data.room_id,
+            session: session.pickle(this._pickleKey),
+            keysClaimed: data.sender_claimed_keys,
+            forwardingCurve25519KeyChain: data.forwarding_curve25519_key_chain,
+        };
+
         this._saveInboundGroupSession(
-            data.room_id, data.sender_key, data.session_id, session,
-            data.sender_claimed_keys,
+            data.sender_key, data.session_id, sessionData,
         );
     } finally {
         session.free();
@@ -749,18 +800,21 @@ OlmDevice.prototype.importInboundGroupSession = function(data) {
  * @param {string} senderKey base64-encoded curve25519 key of the sender
  * @param {string} sessionId session identifier
  * @param {string} body      base64-encoded body of the encrypted message
+ * @param {string} eventId   ID of the event being decrypted
+ * @param {Number} timestamp timestamp of the event being decrypted
  *
  * @return {null} the sessionId is unknown
  *
- * @return {{result: string, keysProved: Object<string, string>, keysClaimed:
- *    Object<string, string>}} result
+ * @return {Promise<{result: string, senderKey: string,
+ *    forwardingCurve25519KeyChain: Array<string>,
+ *    keysClaimed: Object<string, string>}>}
  */
-OlmDevice.prototype.decryptGroupMessage = function(
-    roomId, senderKey, sessionId, body,
+OlmDevice.prototype.decryptGroupMessage = async function(
+    roomId, senderKey, sessionId, body, eventId, timestamp,
 ) {
     const self = this;
 
-    function decrypt(session, keysClaimed) {
+    function decrypt(session, sessionData) {
         const res = session.decrypt(body);
 
         let plaintext = res.plaintext;
@@ -769,27 +823,34 @@ OlmDevice.prototype.decryptGroupMessage = function(
             plaintext = res;
         } else {
             // Check if we have seen this message index before to detect replay attacks.
+            // If the event ID and timestamp are specified, and the match the event ID
+            // and timestamp from the last time we used this message index, then we
+            // don't consider it a replay attack.
             const messageIndexKey = senderKey + "|" + sessionId + "|" + res.message_index;
             if (messageIndexKey in self._inboundGroupSessionMessageIndexes) {
-                throw new Error(
-                    "Duplicate message index, possible replay attack: " +
-                    messageIndexKey,
-                );
+                const msgInfo = self._inboundGroupSessionMessageIndexes[messageIndexKey];
+                if (msgInfo.id !== eventId || msgInfo.timestamp !== timestamp) {
+                    throw new Error(
+                        "Duplicate message index, possible replay attack: " +
+                        messageIndexKey,
+                    );
+                }
             }
-            self._inboundGroupSessionMessageIndexes[messageIndexKey] = true;
+            self._inboundGroupSessionMessageIndexes[messageIndexKey] = {
+                id: eventId,
+                timestamp: timestamp,
+            };
         }
 
-        // the sender must have had the senderKey to persuade us to save the
-        // session.
-        const keysProved = {curve25519: senderKey};
-
+        sessionData.session = session.pickle(self._pickleKey);
         self._saveInboundGroupSession(
-            roomId, senderKey, sessionId, session, keysClaimed,
+            senderKey, sessionId, sessionData,
         );
         return {
             result: plaintext,
-            keysClaimed: keysClaimed,
-            keysProved: keysProved,
+            keysClaimed: sessionData.keysClaimed || {},
+            senderKey: senderKey,
+            forwardingCurve25519KeyChain: sessionData.forwardingCurve25519KeyChain || [],
         };
     }
 
@@ -798,14 +859,82 @@ OlmDevice.prototype.decryptGroupMessage = function(
     );
 };
 
+/**
+ * Determine if we have the keys for a given megolm session
+ *
+ * @param {string} roomId    room in which the message was received
+ * @param {string} senderKey base64-encoded curve25519 key of the sender
+ * @param {sring} sessionId session identifier
+ *
+ * @returns {Promise<boolean>} true if we have the keys to this session
+ */
+OlmDevice.prototype.hasInboundSessionKeys = async function(roomId, senderKey, sessionId) {
+    const s = this._sessionStore.getEndToEndInboundGroupSession(
+        senderKey, sessionId,
+    );
+
+    if (s === null) {
+        return false;
+    }
+
+    const r = JSON.parse(s);
+    if (roomId !== r.room_id) {
+        console.warn(
+            `requested keys for inbound group session ${senderKey}|` +
+            `${sessionId}, with incorrect room_id (expected ${r.room_id}, ` +
+            `was ${roomId})`,
+        );
+        return false;
+    }
+
+    return true;
+};
+
+/**
+ * Extract the keys to a given megolm session, for sharing
+ *
+ * @param {string} roomId    room in which the message was received
+ * @param {string} senderKey base64-encoded curve25519 key of the sender
+ * @param {string} sessionId session identifier
+ *
+ * @returns {Promise<{chain_index: number, key: string,
+ *        forwarding_curve25519_key_chain: Array<string>,
+ *        sender_claimed_ed25519_key: string
+ *    }>}
+ *    details of the session key. The key is a base64-encoded megolm key in
+ *    export format.
+ */
+OlmDevice.prototype.getInboundGroupSessionKey = async function(
+    roomId, senderKey, sessionId,
+) {
+    function getKey(session, sessionData) {
+        const messageIndex = session.first_known_index();
+
+        const claimedKeys = sessionData.keysClaimed || {};
+        const senderEd25519Key = claimedKeys.ed25519 || null;
+
+        return {
+            "chain_index": messageIndex,
+            "key": session.export_session(messageIndex),
+            "forwarding_curve25519_key_chain":
+                sessionData.forwardingCurve25519KeyChain || [],
+            "sender_claimed_ed25519_key": senderEd25519Key,
+        };
+    }
+
+    return this._getInboundGroupSession(
+        roomId, senderKey, sessionId, getKey,
+    );
+};
+
 /**
  * Export an inbound group session
  *
  * @param {string} senderKey base64-encoded curve25519 key of the sender
  * @param {string} sessionId session identifier
- * @return {module:crypto/OlmDevice.MegolmSessionData} exported session data
+ * @return {Promise<module:crypto/OlmDevice.MegolmSessionData>} exported session data
  */
-OlmDevice.prototype.exportInboundGroupSession = function(senderKey, sessionId) {
+OlmDevice.prototype.exportInboundGroupSession = async function(senderKey, sessionId) {
     const s = this._sessionStore.getEndToEndInboundGroupSession(
         senderKey, sessionId,
     );
@@ -828,6 +957,8 @@ OlmDevice.prototype.exportInboundGroupSession = function(senderKey, sessionId) {
             "room_id": r.room_id,
             "session_id": sessionId,
             "session_key": session.export_session(messageIndex),
+            "forwarding_curve25519_key_chain":
+                session.forwardingCurve25519KeyChain || [],
         };
     } finally {
         session.free();

src/crypto/OutgoingRoomKeyRequestManager.js

@@ -0,0 +1,363 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+
+import utils from '../utils';
+
+/**
+ * Internal module. Management of outgoing room key requests.
+ *
+ * See https://docs.google.com/document/d/1m4gQkcnJkxNuBmb5NoFCIadIY-DyqqNAS3lloE73BlQ
+ * for draft documentation on what we're supposed to be implementing here.
+ *
+ * @module
+ */
+
+// delay between deciding we want some keys, and sending out the request, to
+// allow for (a) it turning up anyway, (b) grouping requests together
+const SEND_KEY_REQUESTS_DELAY_MS = 500;
+
+/** possible states for a room key request
+ *
+ * The state machine looks like:
+ *
+ *     |
+ *     V         (cancellation requested)
+ *   UNSENT  -----------------------------+
+ *     |                                  |
+ *     | (send successful)                |
+ *     V                                  |
+ *    SENT                                |
+ *     |                                  |
+ *     | (cancellation requested)         |
+ *     V                                  |
+ * CANCELLATION_PENDING                   |
+ *     |                                  |
+ *     | (cancellation sent)              |
+ *     V                                  |
+ * (deleted)  <---------------------------+
+ *
+ * @enum {number}
+ */
+const ROOM_KEY_REQUEST_STATES = {
+    /** request not yet sent */
+    UNSENT: 0,
+
+    /** request sent, awaiting reply */
+    SENT: 1,
+
+    /** reply received, cancellation not yet sent */
+    CANCELLATION_PENDING: 2,
+};
+
+export default class OutgoingRoomKeyRequestManager {
+    constructor(baseApis, deviceId, cryptoStore) {
+        this._baseApis = baseApis;
+        this._deviceId = deviceId;
+        this._cryptoStore = cryptoStore;
+
+        // handle for the delayed call to _sendOutgoingRoomKeyRequests. Non-null
+        // if the callback has been set, or if it is still running.
+        this._sendOutgoingRoomKeyRequestsTimer = null;
+
+        // sanity check to ensure that we don't end up with two concurrent runs
+        // of _sendOutgoingRoomKeyRequests
+        this._sendOutgoingRoomKeyRequestsRunning = false;
+
+        this._clientRunning = false;
+    }
+
+    /**
+     * Called when the client is started. Sets background processes running.
+     */
+    start() {
+        this._clientRunning = true;
+
+        // set the timer going, to handle any requests which didn't get sent
+        // on the previous run of the client.
+        this._startTimer();
+    }
+
+    /**
+     * Called when the client is stopped. Stops any running background processes.
+     */
+    stop() {
+        console.log('stopping OutgoingRoomKeyRequestManager');
+        // stop the timer on the next run
+        this._clientRunning = false;
+    }
+
+    /**
+     * Send off a room key request, if we haven't already done so.
+     *
+     * The `requestBody` is compared (with a deep-equality check) against
+     * previous queued or sent requests and if it matches, no change is made.
+     * Otherwise, a request is added to the pending list, and a job is started
+     * in the background to send it.
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     * @param {Array<{userId: string, deviceId: string}>} recipients
+     *
+     * @returns {Promise} resolves when the request has been added to the
+     *    pending list (or we have established that a similar request already
+     *    exists)
+     */
+    sendRoomKeyRequest(requestBody, recipients) {
+        return this._cryptoStore.getOrAddOutgoingRoomKeyRequest({
+            requestBody: requestBody,
+            recipients: recipients,
+            requestId: this._baseApis.makeTxnId(),
+            state: ROOM_KEY_REQUEST_STATES.UNSENT,
+        }).then((req) => {
+            if (req.state === ROOM_KEY_REQUEST_STATES.UNSENT) {
+                this._startTimer();
+            }
+        });
+    }
+
+    /**
+     * Cancel room key requests, if any match the given details
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *
+     * @returns {Promise} resolves when the request has been updated in our
+     *    pending list.
+     */
+    cancelRoomKeyRequest(requestBody) {
+        return this._cryptoStore.getOutgoingRoomKeyRequest(
+            requestBody,
+        ).then((req) => {
+            if (!req) {
+                // no request was made for this key
+                return;
+            }
+            switch (req.state) {
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING:
+                    // nothing to do here
+                    return;
+
+                case ROOM_KEY_REQUEST_STATES.UNSENT:
+                    // just delete it
+
+                    // FIXME: ghahah we may have attempted to send it, and
+                    // not yet got a successful response. So the server
+                    // may have seen it, so we still need to send a cancellation
+                    // in that case :/
+
+                    console.log(
+                        'deleting unnecessary room key request for ' +
+                        stringifyRequestBody(requestBody),
+                    );
+                    return this._cryptoStore.deleteOutgoingRoomKeyRequest(
+                        req.requestId, ROOM_KEY_REQUEST_STATES.UNSENT,
+                    );
+
+                case ROOM_KEY_REQUEST_STATES.SENT:
+                    // send a cancellation.
+                    return this._cryptoStore.updateOutgoingRoomKeyRequest(
+                        req.requestId, ROOM_KEY_REQUEST_STATES.SENT, {
+                            state: ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
+                            cancellationTxnId: this._baseApis.makeTxnId(),
+                        },
+                    ).then((updatedReq) => {
+                        if (!updatedReq) {
+                            // updateOutgoingRoomKeyRequest couldn't find the
+                            // request in state ROOM_KEY_REQUEST_STATES.SENT,
+                            // so we must have raced with another tab to mark
+                            // the request cancelled. There is no point in
+                            // sending another cancellation since the other tab
+                            // will do it.
+                            console.log(
+                                'Tried to cancel room key request for ' +
+                                stringifyRequestBody(requestBody) +
+                                ' but it was already cancelled in another tab',
+                            );
+                            return;
+                        }
+
+                        // We don't want to wait for the timer, so we send it
+                        // immediately. (We might actually end up racing with the timer,
+                        // but that's ok: even if we make the request twice, we'll do it
+                        // with the same transaction_id, so only one message will get
+                        // sent).
+                        //
+                        // (We also don't want to wait for the response from the server
+                        // here, as it will slow down processing of received keys if we
+                        // do.)
+                        this._sendOutgoingRoomKeyRequestCancellation(
+                            updatedReq,
+                        ).catch((e) => {
+                            console.error(
+                                "Error sending room key request cancellation;"
+                                + " will retry later.", e,
+                            );
+                            this._startTimer();
+                        }).done();
+                    });
+
+                default:
+                    throw new Error('unhandled state: ' + req.state);
+            }
+        });
+    }
+
+    // start the background timer to send queued requests, if the timer isn't
+    // already running
+    _startTimer() {
+        if (this._sendOutgoingRoomKeyRequestsTimer) {
+            return;
+        }
+
+        const startSendingOutgoingRoomKeyRequests = () => {
+            if (this._sendOutgoingRoomKeyRequestsRunning) {
+                throw new Error("RoomKeyRequestSend already in progress!");
+            }
+            this._sendOutgoingRoomKeyRequestsRunning = true;
+
+            this._sendOutgoingRoomKeyRequests().finally(() => {
+                this._sendOutgoingRoomKeyRequestsRunning = false;
+            }).catch((e) => {
+                // this should only happen if there is an indexeddb error,
+                // in which case we're a bit stuffed anyway.
+                console.warn(
+                    `error in OutgoingRoomKeyRequestManager: ${e}`,
+                );
+            }).done();
+        };
+
+        this._sendOutgoingRoomKeyRequestsTimer = global.setTimeout(
+            startSendingOutgoingRoomKeyRequests,
+            SEND_KEY_REQUESTS_DELAY_MS,
+        );
+    }
+
+    // look for and send any queued requests. Runs itself recursively until
+    // there are no more requests, or there is an error (in which case, the
+    // timer will be restarted before the promise resolves).
+    _sendOutgoingRoomKeyRequests() {
+        if (!this._clientRunning) {
+            this._sendOutgoingRoomKeyRequestsTimer = null;
+            return Promise.resolve();
+        }
+
+        console.log("Looking for queued outgoing room key requests");
+
+        return this._cryptoStore.getOutgoingRoomKeyRequestByState([
+            ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
+            ROOM_KEY_REQUEST_STATES.UNSENT,
+        ]).then((req) => {
+            if (!req) {
+                console.log("No more outgoing room key requests");
+                this._sendOutgoingRoomKeyRequestsTimer = null;
+                return;
+            }
+
+            let prom;
+            if (req.state === ROOM_KEY_REQUEST_STATES.UNSENT) {
+                prom = this._sendOutgoingRoomKeyRequest(req);
+            } else { // must be a cancellation
+                prom = this._sendOutgoingRoomKeyRequestCancellation(req);
+            }
+
+            return prom.then(() => {
+                // go around the loop again
+                return this._sendOutgoingRoomKeyRequests();
+            }).catch((e) => {
+                console.error("Error sending room key request; will retry later.", e);
+                this._sendOutgoingRoomKeyRequestsTimer = null;
+                this._startTimer();
+            }).done();
+        });
+    }
+
+    // given a RoomKeyRequest, send it and update the request record
+    _sendOutgoingRoomKeyRequest(req) {
+        console.log(
+            `Requesting keys for ${stringifyRequestBody(req.requestBody)}` +
+            ` from ${stringifyRecipientList(req.recipients)}` +
+            `(id ${req.requestId})`,
+        );
+
+        const requestMessage = {
+            action: "request",
+            requesting_device_id: this._deviceId,
+            request_id: req.requestId,
+            body: req.requestBody,
+        };
+
+        return this._sendMessageToDevices(
+            requestMessage, req.recipients, req.requestId,
+        ).then(() => {
+            return this._cryptoStore.updateOutgoingRoomKeyRequest(
+                req.requestId, ROOM_KEY_REQUEST_STATES.UNSENT,
+                { state: ROOM_KEY_REQUEST_STATES.SENT },
+            );
+        });
+    }
+
+    // given a RoomKeyRequest, cancel it and delete the request record
+    _sendOutgoingRoomKeyRequestCancellation(req) {
+        console.log(
+            `Sending cancellation for key request for ` +
+            `${stringifyRequestBody(req.requestBody)} to ` +
+            `${stringifyRecipientList(req.recipients)} ` +
+            `(cancellation id ${req.cancellationTxnId})`,
+        );
+
+        const requestMessage = {
+            action: "request_cancellation",
+            requesting_device_id: this._deviceId,
+            request_id: req.requestId,
+        };
+
+        return this._sendMessageToDevices(
+            requestMessage, req.recipients, req.cancellationTxnId,
+        ).then(() => {
+            return this._cryptoStore.deleteOutgoingRoomKeyRequest(
+                req.requestId, ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
+            );
+        });
+    }
+
+    // send a RoomKeyRequest to a list of recipients
+    _sendMessageToDevices(message, recipients, txnId) {
+        const contentMap = {};
+        for (const recip of recipients) {
+            if (!contentMap[recip.userId]) {
+                contentMap[recip.userId] = {};
+            }
+            contentMap[recip.userId][recip.deviceId] = message;
+        }
+
+        return this._baseApis.sendToDevice(
+            'm.room_key_request', contentMap, txnId,
+        );
+    }
+}
+
+function stringifyRequestBody(requestBody) {
+    // we assume that the request is for megolm keys, which are identified by
+    // room id and session id
+    return requestBody.room_id + " / " + requestBody.session_id;
+}
+
+function stringifyRecipientList(recipients) {
+    return '['
+        + utils.map(recipients, (r) => `${r.userId}:${r.deviceId}`).join(",")
+        + ']';
+}
+

src/crypto/algorithms/base.js

@@ -13,14 +13,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
 
 /**
  * Internal module. Defines the base classes of the encryption implementations
  *
- * @module crypto/algorithms/base
+ * @module
  */
-const utils = require("../../utils");
+
+import Promise from 'bluebird';
 
 /**
  * map of registered encryption algorithm classes. A map from string to {@link
@@ -28,7 +28,7 @@ const utils = require("../../utils");
  *
  * @type {Object.<string, function(new: module:crypto/algorithms/base.EncryptionAlgorithm)>}
  */
-module.exports.ENCRYPTION_CLASSES = {};
+export const ENCRYPTION_CLASSES = {};
 
 /**
  * map of registered encryption algorithm classes. Map from string to {@link
@@ -36,12 +36,11 @@ module.exports.ENCRYPTION_CLASSES = {};
  *
  * @type {Object.<string, function(new: module:crypto/algorithms/base.DecryptionAlgorithm)>}
  */
-module.exports.DECRYPTION_CLASSES = {};
+export const DECRYPTION_CLASSES = {};
 
 /**
  * base type for encryption implementations
  *
- * @constructor
  * @alias module:crypto/algorithms/base.EncryptionAlgorithm
  *
  * @param {object} params parameters
@@ -53,103 +52,122 @@ module.exports.DECRYPTION_CLASSES = {};
  * @param {string} params.roomId  The ID of the room we will be sending to
  * @param {object} params.config  The body of the m.room.encryption event
  */
-const EncryptionAlgorithm = function(params) {
-    this._userId = params.userId;
-    this._deviceId = params.deviceId;
-    this._crypto = params.crypto;
-    this._olmDevice = params.olmDevice;
-    this._baseApis = params.baseApis;
-    this._roomId = params.roomId;
-};
-/** */
-module.exports.EncryptionAlgorithm = EncryptionAlgorithm;
+class EncryptionAlgorithm {
+    constructor(params) {
+        this._userId = params.userId;
+        this._deviceId = params.deviceId;
+        this._crypto = params.crypto;
+        this._olmDevice = params.olmDevice;
+        this._baseApis = params.baseApis;
+        this._roomId = params.roomId;
+    }
 
-/**
- * Encrypt a message event
- *
- * @method module:crypto/algorithms/base.EncryptionAlgorithm#encryptMessage
- * @abstract
- *
- * @param {module:models/room} room
- * @param {string} eventType
- * @param {object} plaintext event content
- *
- * @return {module:client.Promise} Promise which resolves to the new event body
- */
+    /**
+     * Encrypt a message event
+     *
+     * @method module:crypto/algorithms/base.EncryptionAlgorithm.encryptMessage
+     * @abstract
+     *
+     * @param {module:models/room} room
+     * @param {string} eventType
+     * @param {object} plaintext event content
+     *
+     * @return {module:client.Promise} Promise which resolves to the new event body
+     */
 
-/**
- * Called when the membership of a member of the room changes.
- *
- * @param {module:models/event.MatrixEvent} event  event causing the change
- * @param {module:models/room-member} member  user whose membership changed
- * @param {string=} oldMembership  previous membership
- */
-EncryptionAlgorithm.prototype.onRoomMembership = function(
-    event, member, oldMembership,
-) {};
+    /**
+     * Called when the membership of a member of the room changes.
+     *
+     * @param {module:models/event.MatrixEvent} event  event causing the change
+     * @param {module:models/room-member} member  user whose membership changed
+     * @param {string=} oldMembership  previous membership
+     * @public
+     */
+     onRoomMembership(event, member, oldMembership) {
+     }
+}
+export {EncryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 /**
  * base type for decryption implementations
  *
- * @constructor
  * @alias module:crypto/algorithms/base.DecryptionAlgorithm
- *
  * @param {object} params parameters
  * @param {string} params.userId  The UserID for the local user
  * @param {module:crypto} params.crypto crypto core
  * @param {module:crypto/OlmDevice} params.olmDevice olm.js wrapper
+ * @param {module:base-apis~MatrixBaseApis} baseApis base matrix api interface
  * @param {string=} params.roomId The ID of the room we will be receiving
  *     from. Null for to-device events.
  */
-const DecryptionAlgorithm = function(params) {
-    this._userId = params.userId;
-    this._crypto = params.crypto;
-    this._olmDevice = params.olmDevice;
-    this._roomId = params.roomId;
-};
-/** */
-module.exports.DecryptionAlgorithm = DecryptionAlgorithm;
+class DecryptionAlgorithm {
+    constructor(params) {
+        this._userId = params.userId;
+        this._crypto = params.crypto;
+        this._olmDevice = params.olmDevice;
+        this._baseApis = params.baseApis;
+        this._roomId = params.roomId;
+    }
 
-/**
- * Decrypt an event
- *
- * @method module:crypto/algorithms/base.DecryptionAlgorithm#decryptEvent
- * @abstract
- *
- * @param {object} event raw event
- *
- * @return {null} if the event referred to an unknown megolm session
- * @return {module:crypto.DecryptionResult} decryption result
- *
- * @throws {module:crypto/algorithms/base.DecryptionError} if there is a
- *   problem decrypting the event
- */
+    /**
+     * Decrypt an event
+     *
+     * @method module:crypto/algorithms/base.DecryptionAlgorithm#decryptEvent
+     * @abstract
+     *
+     * @param {MatrixEvent} event undecrypted event
+     *
+     * @return {Promise<module:crypto~EventDecryptionResult>} promise which
+     * resolves once we have finished decrypting. Rejects with an
+     * `algorithms.DecryptionError` if there is a problem decrypting the event.
+     */
 
-/**
- * Handle a key event
- *
- * @method module:crypto/algorithms/base.DecryptionAlgorithm#onRoomKeyEvent
- *
- * @param {module:models/event.MatrixEvent} params event key event
- */
-DecryptionAlgorithm.prototype.onRoomKeyEvent = function(params) {
-    // ignore by default
-};
+    /**
+     * Handle a key event
+     *
+     * @method module:crypto/algorithms/base.DecryptionAlgorithm#onRoomKeyEvent
+     *
+     * @param {module:models/event.MatrixEvent} params event key event
+     */
+    onRoomKeyEvent(params) {
+        // ignore by default
+    }
 
-/**
- * Import a room key
- *
- * @param {module:crypto/OlmDevice.MegolmSessionData} session
- */
-DecryptionAlgorithm.prototype.importRoomKey = function(session) {
-    // ignore by default
-};
+    /**
+     * Import a room key
+     *
+     * @param {module:crypto/OlmDevice.MegolmSessionData} session
+     */
+    importRoomKey(session) {
+        // ignore by default
+    }
+
+    /**
+     * Determine if we have the keys necessary to respond to a room key request
+     *
+     * @param {module:crypto~IncomingRoomKeyRequest} keyRequest
+     * @return {Promise<boolean>} true if we have the keys and could (theoretically) share
+     *  them; else false.
+     */
+    hasKeysForKeyRequest(keyRequest) {
+        return Promise.resolve(false);
+    }
+
+    /**
+     * Send the response to a room key request
+     *
+     * @param {module:crypto~IncomingRoomKeyRequest} keyRequest
+     */
+    shareKeysWithDevice(keyRequest) {
+        throw new Error("shareKeysWithDevice not supported for this DecryptionAlgorithm");
+    }
+}
+export {DecryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 /**
  * Exception thrown when decryption fails
  *
  * @alias module:crypto/algorithms/base.DecryptionError
- * @constructor
  * @param {string} msg user-visible message describing the problem
  *
  * @param {Object=} details key/value pairs reported in the logs but not shown
@@ -157,50 +175,51 @@ DecryptionAlgorithm.prototype.importRoomKey = function(session) {
  *
  * @extends Error
  */
-const DecryptionError = function(msg, details) {
-    this.name = 'DecryptionError';
-    this.message = msg;
-    this.details = details;
-};
-utils.inherits(DecryptionError, Error);
-
-/** override the string used when logging
- *
- * @returns {String}
- */
-DecryptionError.prototype.toString = function() {
-    let result = this.name + '[msg: ' + this.message;
-
-    if (this.details) {
-        result += ', ' +
-            Object.keys(this.details).map(
-                (k) => k + ': ' + this.details[k],
-            ).join(', ');
+class DecryptionError extends Error {
+    constructor(msg, details) {
+        super(msg);
+        this.name = 'DecryptionError';
+        this.details = details;
     }
 
-    result += ']';
+    /**
+     * override the string used when logging
+     *
+     * @returns {String}
+     */
+    toString() {
+        let result = this.name + '[msg: ' + this.message;
 
-    return result;
-};
+        if (this.details) {
+            result += ', ' +
+                Object.keys(this.details).map(
+                    (k) => k + ': ' + this.details[k],
+                ).join(', ');
+        }
 
-module.exports.DecryptionError = DecryptionError;
+        result += ']';
+
+        return result;
+    }
+}
+export {DecryptionError}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 /**
  * Exception thrown specifically when we want to warn the user to consider
  * the security of their conversation before continuing
  *
- * @constructor
  * @param {string} msg message describing the problem
  * @param {Object} devices userId -> {deviceId -> object}
  *      set of unknown devices per user we're warning about
  * @extends Error
  */
-module.exports.UnknownDeviceError = function(msg, devices) {
-    this.name = "UnknownDeviceError";
-    this.message = msg;
-    this.devices = devices;
-};
-utils.inherits(module.exports.UnknownDeviceError, Error);
+export class UnknownDeviceError extends Error {
+    constructor(msg, devices) {
+        super(msg);
+        this.name = "UnknownDeviceError";
+        this.devices = devices;
+    }
+}
 
 /**
  * Registers an encryption/decryption class for a particular algorithm
@@ -215,7 +234,7 @@ utils.inherits(module.exports.UnknownDeviceError, Error);
  *     module:crypto/algorithms/base.DecryptionAlgorithm|DecryptionAlgorithm}
  *     implementation
  */
-module.exports.registerAlgorithm = function(algorithm, encryptor, decryptor) {
-    module.exports.ENCRYPTION_CLASSES[algorithm] = encryptor;
-    module.exports.DECRYPTION_CLASSES[algorithm] = decryptor;
-};
+export function registerAlgorithm(algorithm, encryptor, decryptor) {
+    ENCRYPTION_CLASSES[algorithm] = encryptor;
+    DECRYPTION_CLASSES[algorithm] = decryptor;
+}

src/crypto/algorithms/megolm.js

@@ -21,7 +21,7 @@ limitations under the License.
  * @module crypto/algorithms/megolm
  */
 
-const q = require("q");
+import Promise from 'bluebird';
 
 const utils = require("../../utils");
 const olmlib = require("../olmlib");
@@ -74,6 +74,14 @@ OutboundSessionInfo.prototype.needsRotation = function(
     return false;
 };
 
+OutboundSessionInfo.prototype.markSharedWithDevice = function(
+    userId, deviceId, chainIndex,
+) {
+    if (!this.sharedWithDevices[userId]) {
+        this.sharedWithDevices[userId] = {};
+    }
+    this.sharedWithDevices[userId][deviceId] = chainIndex;
+};
 
 /**
  * Determine if this session has been shared with devices which it shouldn't
@@ -132,7 +140,7 @@ function MegolmEncryption(params) {
     // are using, and which devices we have shared the keys with. It resolves
     // with an OutboundSessionInfo (or undefined, for the first message in the
     // room).
-    this._setupPromise = q();
+    this._setupPromise = Promise.resolve();
 
     // default rotation periods
     this._sessionRotationPeriodMsgs = 100;
@@ -166,7 +174,7 @@ MegolmEncryption.prototype._ensureOutboundSession = function(devicesInRoom) {
     // Updates `session` to hold the final OutboundSessionInfo.
     //
     // returns a promise which resolves once the keyshare is successful.
-    function prepareSession(oldSession) {
+    async function prepareSession(oldSession) {
         session = oldSession;
 
         // need to make a brand new session?
@@ -183,7 +191,8 @@ MegolmEncryption.prototype._ensureOutboundSession = function(devicesInRoom) {
         }
 
         if (!session) {
-            session = self._prepareNewSession();
+            console.log(`Starting new megolm session for room ${self._roomId}`);
+            session = await self._prepareNewSession();
         }
 
         // now check if we need to share with any devices
@@ -244,16 +253,157 @@ MegolmEncryption.prototype._ensureOutboundSession = function(devicesInRoom) {
  *
  * @return {module:crypto/algorithms/megolm.OutboundSessionInfo} session
  */
-MegolmEncryption.prototype._prepareNewSession = function() {
-    const session_id = this._olmDevice.createOutboundGroupSession();
-    const key = this._olmDevice.getOutboundGroupSessionKey(session_id);
+MegolmEncryption.prototype._prepareNewSession = async function() {
+    const sessionId = this._olmDevice.createOutboundGroupSession();
+    const key = this._olmDevice.getOutboundGroupSessionKey(sessionId);
 
-    this._olmDevice.addInboundGroupSession(
-        this._roomId, this._olmDevice.deviceCurve25519Key, session_id,
+    await this._olmDevice.addInboundGroupSession(
+        this._roomId, this._olmDevice.deviceCurve25519Key, [], sessionId,
         key.key, {ed25519: this._olmDevice.deviceEd25519Key},
     );
 
-    return new OutboundSessionInfo(session_id);
+    return new OutboundSessionInfo(sessionId);
+};
+
+/**
+ * @private
+ *
+ * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ *
+ * @param {number} chainIndex current chain index
+ *
+ * @param {object<userId, deviceId>} devicemap
+ *   mapping from userId to deviceId to {@link module:crypto~OlmSessionResult}
+ *
+ * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
+ *    map from userid to list of devices
+ *
+ * @return {array<object<userid, deviceInfo>>}
+ */
+MegolmEncryption.prototype._splitUserDeviceMap = function(
+    session, chainIndex, devicemap, devicesByUser,
+) {
+    const maxToDeviceMessagesPerRequest = 20;
+
+    // use an array where the slices of a content map gets stored
+    const mapSlices = [];
+    let currentSliceId = 0; // start inserting in the first slice
+    let entriesInCurrentSlice = 0;
+
+    for (const userId of Object.keys(devicesByUser)) {
+        const devicesToShareWith = devicesByUser[userId];
+        const sessionResults = devicemap[userId];
+
+        for (let i = 0; i < devicesToShareWith.length; i++) {
+            const deviceInfo = devicesToShareWith[i];
+            const deviceId = deviceInfo.deviceId;
+
+            const sessionResult = sessionResults[deviceId];
+            if (!sessionResult.sessionId) {
+                // no session with this device, probably because there
+                // were no one-time keys.
+                //
+                // we could send them a to_device message anyway, as a
+                // signal that they have missed out on the key sharing
+                // message because of the lack of keys, but there's not
+                // much point in that really; it will mostly serve to clog
+                // up to_device inboxes.
+
+                // mark this device as "handled" because we don't want to try
+                // to claim a one-time-key for dead devices on every message.
+                session.markSharedWithDevice(userId, deviceId, chainIndex);
+
+                // ensureOlmSessionsForUsers has already done the logging,
+                // so just skip it.
+                continue;
+            }
+
+            console.log(
+                "share keys with device " + userId + ":" + deviceId,
+            );
+
+            if (entriesInCurrentSlice > maxToDeviceMessagesPerRequest) {
+                // the current slice is filled up. Start inserting into the next slice
+                entriesInCurrentSlice = 0;
+                currentSliceId++;
+            }
+            if (!mapSlices[currentSliceId]) {
+                mapSlices[currentSliceId] = [];
+            }
+
+            mapSlices[currentSliceId].push({
+                userId: userId,
+                deviceInfo: deviceInfo,
+            });
+
+            entriesInCurrentSlice++;
+        }
+    }
+    return mapSlices;
+};
+
+/**
+ * @private
+ *
+ * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ *
+ * @param {number} chainIndex current chain index
+ *
+ * @param {object<userId, deviceInfo>} userDeviceMap
+ *   mapping from userId to deviceInfo
+ *
+ * @param {object} payload fields to include in the encrypted payload
+ *
+ * @return {module:client.Promise} Promise which resolves once the key sharing
+ *     for the given userDeviceMap is generated and has been sent.
+ */
+MegolmEncryption.prototype._encryptAndSendKeysToDevices = function(
+    session, chainIndex, userDeviceMap, payload,
+) {
+    const encryptedContent = {
+        algorithm: olmlib.OLM_ALGORITHM,
+        sender_key: this._olmDevice.deviceCurve25519Key,
+        ciphertext: {},
+    };
+    const contentMap = {};
+
+    const promises = [];
+    for (let i = 0; i < userDeviceMap.length; i++) {
+        const val = userDeviceMap[i];
+        const userId = val.userId;
+        const deviceInfo = val.deviceInfo;
+        const deviceId = deviceInfo.deviceId;
+
+        if (!contentMap[userId]) {
+            contentMap[userId] = {};
+        }
+        contentMap[userId][deviceId] = encryptedContent;
+
+        promises.push(
+            olmlib.encryptMessageForDevice(
+                encryptedContent.ciphertext,
+                this._userId,
+                this._deviceId,
+                this._olmDevice,
+                userId,
+                deviceInfo,
+                payload,
+            ),
+        );
+    }
+
+    return Promise.all(promises).then(() => {
+        return this._baseApis.sendToDevice("m.room.encrypted", contentMap).then(() => {
+            // store that we successfully uploaded the keys of the current slice
+            for (const userId of Object.keys(contentMap)) {
+                for (const deviceId of Object.keys(contentMap[userId])) {
+                    session.markSharedWithDevice(
+                        userId, deviceId, chainIndex,
+                    );
+                }
+            }
+        });
+    });
 };
 
 /**
@@ -263,13 +413,8 @@ MegolmEncryption.prototype._prepareNewSession = function() {
  *
  * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
  *    map from userid to list of devices
- *
- * @return {module:client.Promise} Promise which resolves once the key sharing
- *     message has been sent.
  */
-MegolmEncryption.prototype._shareKeyWithDevices = function(session, devicesByUser) {
-    const self = this;
-
+MegolmEncryption.prototype._shareKeyWithDevices = async function(session, devicesByUser) {
     const key = this._olmDevice.getOutboundGroupSessionKey(session.sessionId);
     const payload = {
         type: "m.room_key",
@@ -282,98 +427,28 @@ MegolmEncryption.prototype._shareKeyWithDevices = function(session, devicesByUse
         },
     };
 
-    const contentMap = {};
-
-    return olmlib.ensureOlmSessionsForDevices(
+    const devicemap = await olmlib.ensureOlmSessionsForDevices(
         this._olmDevice, this._baseApis, devicesByUser,
-    ).then(function(devicemap) {
-        let haveTargets = false;
+    );
 
-        for (const userId in devicesByUser) {
-            if (!devicesByUser.hasOwnProperty(userId)) {
-                continue;
-            }
+    const userDeviceMaps = this._splitUserDeviceMap(
+        session, key.chain_index, devicemap, devicesByUser,
+    );
 
-            const devicesToShareWith = devicesByUser[userId];
-            const sessionResults = devicemap[userId];
+    for (let i = 0; i < userDeviceMaps.length; i++) {
+        try {
+            await this._encryptAndSendKeysToDevices(
+                session, key.chain_index, userDeviceMaps[i], payload,
+            );
+            console.log(`Completed megolm keyshare in ${this._roomId} `
+                + `(slice ${i + 1}/${userDeviceMaps.length})`);
+        } catch (e) {
+            console.log(`megolm keyshare in ${this._roomId} `
+                + `(slice ${i + 1}/${userDeviceMaps.length}) failed`);
 
-            for (let i = 0; i < devicesToShareWith.length; i++) {
-                const deviceInfo = devicesToShareWith[i];
-                const deviceId = deviceInfo.deviceId;
-
-                const sessionResult = sessionResults[deviceId];
-                if (!sessionResult.sessionId) {
-                    // no session with this device, probably because there
-                    // were no one-time keys.
-                    //
-                    // we could send them a to_device message anyway, as a
-                    // signal that they have missed out on the key sharing
-                    // message because of the lack of keys, but there's not
-                    // much point in that really; it will mostly serve to clog
-                    // up to_device inboxes.
-                    //
-                    // ensureOlmSessionsForUsers has already done the logging,
-                    // so just skip it.
-                    continue;
-                }
-
-                console.log(
-                    "sharing keys with device " + userId + ":" + deviceId,
-                );
-
-                const encryptedContent = {
-                    algorithm: olmlib.OLM_ALGORITHM,
-                    sender_key: self._olmDevice.deviceCurve25519Key,
-                    ciphertext: {},
-                };
-
-                olmlib.encryptMessageForDevice(
-                    encryptedContent.ciphertext,
-                    self._userId,
-                    self._deviceId,
-                    self._olmDevice,
-                    userId,
-                    deviceInfo,
-                    payload,
-                );
-
-                if (!contentMap[userId]) {
-                    contentMap[userId] = {};
-                }
-
-                contentMap[userId][deviceId] = encryptedContent;
-                haveTargets = true;
-            }
+            throw e;
         }
-
-        if (!haveTargets) {
-            return q();
-        }
-
-        // TODO: retries
-        return self._baseApis.sendToDevice("m.room.encrypted", contentMap);
-    }).then(function() {
-        // Add the devices we have shared with to session.sharedWithDevices.
-        //
-        // we deliberately iterate over devicesByUser (ie, the devices we
-        // attempted to share with) rather than the contentMap (those we did
-        // share with), because we don't want to try to claim a one-time-key
-        // for dead devices on every message.
-        for (const userId in devicesByUser) {
-            if (!devicesByUser.hasOwnProperty(userId)) {
-                continue;
-            }
-            if (!session.sharedWithDevices[userId]) {
-                session.sharedWithDevices[userId] = {};
-            }
-            const devicesToShareWith = devicesByUser[userId];
-            for (let i = 0; i < devicesToShareWith.length; i++) {
-                const deviceInfo = devicesToShareWith[i];
-                session.sharedWithDevices[userId][deviceInfo.deviceId] =
-                    key.chain_index;
-            }
-        }
-    });
+    }
 };
 
 /**
@@ -387,6 +462,8 @@ MegolmEncryption.prototype._shareKeyWithDevices = function(session, devicesByUse
  */
 MegolmEncryption.prototype.encryptMessage = function(room, eventType, content) {
     const self = this;
+    console.log(`Starting to encrypt event for ${this._roomId}`);
+
     return this._getDevicesInRoom(room).then(function(devicesInRoom) {
         // check if any of these devices are not yet known to the user.
         // if so, warn the user so they can verify or ignore.
@@ -513,8 +590,11 @@ function MegolmDecryption(params) {
     base.DecryptionAlgorithm.call(this, params);
 
     // events which we couldn't decrypt due to unknown sessions / indexes: map from
-    // senderKey|sessionId to list of MatrixEvents
+    // senderKey|sessionId to Set of MatrixEvents
     this._pendingEvents = {};
+
+    // this gets stubbed out by the unit tests.
+    this.olmlib = olmlib;
 }
 utils.inherits(MegolmDecryption, base.DecryptionAlgorithm);
 
@@ -523,10 +603,12 @@ utils.inherits(MegolmDecryption, base.DecryptionAlgorithm);
  *
  * @param {MatrixEvent} event
  *
- * @throws {module:crypto/algorithms/base.DecryptionError} if there is a
- *   problem decrypting the event
+ * returns a promise which resolves to a
+ * {@link module:crypto~EventDecryptionResult} once we have finished
+ * decrypting, or rejects with an `algorithms.DecryptionError` if there is a
+ * problem decrypting the event.
  */
-MegolmDecryption.prototype.decryptEvent = function(event) {
+MegolmDecryption.prototype.decryptEvent = async function(event) {
     const content = event.getWireContent();
 
     if (!content.sender_key || !content.session_id ||
@@ -535,14 +617,22 @@ MegolmDecryption.prototype.decryptEvent = function(event) {
         throw new base.DecryptionError("Missing fields in input");
     }
 
+    // we add the event to the pending list *before* we start decryption.
+    //
+    // then, if the key turns up while decryption is in progress (and
+    // decryption fails), we will schedule a retry.
+    // (fixes https://github.com/vector-im/riot-web/issues/5001)
+    this._addEventToPendingList(event);
+
     let res;
     try {
-        res = this._olmDevice.decryptGroupMessage(
+        res = await this._olmDevice.decryptGroupMessage(
             event.getRoomId(), content.sender_key, content.session_id, content.ciphertext,
+            event.getId(), event.getTs(),
         );
     } catch (e) {
         if (e.message === 'OLM.UNKNOWN_MESSAGE_INDEX') {
-            this._addEventToPendingList(event);
+            this._requestKeysForEvent(event);
         }
         throw new base.DecryptionError(
             e.toString(), {
@@ -553,7 +643,13 @@ MegolmDecryption.prototype.decryptEvent = function(event) {
 
     if (res === null) {
         // We've got a message for a session we don't have.
-        this._addEventToPendingList(event);
+        //
+        // (XXX: We might actually have received this key since we started
+        // decrypting, in which case we'll have scheduled a retry, and this
+        // request will be redundant. We could probably check to see if the
+        // event is still in the pending list; if not, a retry will have been
+        // scheduled, so we needn't send out the request here.)
+        this._requestKeysForEvent(event);
         throw new base.DecryptionError(
             "The sender's device has not sent us the keys for this message.",
             {
@@ -562,6 +658,10 @@ MegolmDecryption.prototype.decryptEvent = function(event) {
         );
     }
 
+    // success. We can remove the event from the pending list, if that hasn't
+    // already happened.
+    this._removeEventFromPendingList(event);
+
     const payload = JSON.parse(res.result);
 
     // belt-and-braces check that the room id matches that indicated by the HS
@@ -573,13 +673,39 @@ MegolmDecryption.prototype.decryptEvent = function(event) {
         );
     }
 
-    event.setClearData(payload, res.keysProved, res.keysClaimed);
+    return {
+        clearEvent: payload,
+        senderCurve25519Key: res.senderKey,
+        claimedEd25519Key: res.keysClaimed.ed25519,
+        forwardingCurve25519KeyChain: res.forwardingCurve25519KeyChain,
+    };
 };
 
+MegolmDecryption.prototype._requestKeysForEvent = function(event) {
+    const sender = event.getSender();
+    const wireContent = event.getWireContent();
+
+    // send the request to all of our own devices, and the
+    // original sending device if it wasn't us.
+    const recipients = [{
+        userId: this._userId, deviceId: '*',
+    }];
+    if (sender != this._userId) {
+        recipients.push({
+            userId: sender, deviceId: wireContent.device_id,
+        });
+    }
+
+    this._crypto.requestRoomKey({
+        room_id: event.getRoomId(),
+        algorithm: wireContent.algorithm,
+        sender_key: wireContent.sender_key,
+        session_id: wireContent.session_id,
+    }, recipients);
+};
 
 /**
- * Add an event to the list of those we couldn't decrypt the first time we
- * saw them.
+ * Add an event to the list of those awaiting their session keys.
  *
  * @private
  *
@@ -589,11 +715,32 @@ MegolmDecryption.prototype._addEventToPendingList = function(event) {
     const content = event.getWireContent();
     const k = content.sender_key + "|" + content.session_id;
     if (!this._pendingEvents[k]) {
-        this._pendingEvents[k] = [];
+        this._pendingEvents[k] = new Set();
     }
-    this._pendingEvents[k].push(event);
+    this._pendingEvents[k].add(event);
 };
 
+/**
+ * Remove an event from the list of those awaiting their session keys.
+ *
+ * @private
+ *
+ * @param {module:models/event.MatrixEvent} event
+ */
+MegolmDecryption.prototype._removeEventFromPendingList = function(event) {
+    const content = event.getWireContent();
+    const k = content.sender_key + "|" + content.session_id;
+    if (!this._pendingEvents[k]) {
+        return;
+    }
+
+    this._pendingEvents[k].delete(event);
+    if (this._pendingEvents[k].size === 0) {
+        delete this._pendingEvents[k];
+    }
+};
+
+
 /**
  * @inheritdoc
  *
@@ -601,8 +748,11 @@ MegolmDecryption.prototype._addEventToPendingList = function(event) {
  */
 MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
     const content = event.getContent();
-    const senderKey = event.getSenderKey();
     const sessionId = content.session_id;
+    let senderKey = event.getSenderKey();
+    let forwardingKeyChain = [];
+    let exportFormat = false;
+    let keysClaimed;
 
     if (!content.room_id ||
         !sessionId ||
@@ -611,21 +761,161 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
         console.error("key event is missing fields");
         return;
     }
+
     if (!senderKey) {
         console.error("key event has no sender key (not encrypted?)");
         return;
     }
 
+    if (event.getType() == "m.forwarded_room_key") {
+        exportFormat = true;
+        forwardingKeyChain = content.forwarding_curve25519_key_chain;
+        if (!utils.isArray(forwardingKeyChain)) {
+            forwardingKeyChain = [];
+        }
+
+        // copy content before we modify it
+        forwardingKeyChain = forwardingKeyChain.slice();
+        forwardingKeyChain.push(senderKey);
+
+        senderKey = content.sender_key;
+        if (!senderKey) {
+            console.error("forwarded_room_key event is missing sender_key field");
+            return;
+        }
+
+        const ed25519Key = content.sender_claimed_ed25519_key;
+        if (!ed25519Key) {
+            console.error(
+                `forwarded_room_key_event is missing sender_claimed_ed25519_key field`,
+            );
+            return;
+        }
+
+        keysClaimed = {
+            ed25519: ed25519Key,
+        };
+    } else {
+        keysClaimed = event.getKeysClaimed();
+    }
+
     console.log(`Adding key for megolm session ${senderKey}|${sessionId}`);
     this._olmDevice.addInboundGroupSession(
-        content.room_id, senderKey, sessionId,
-        content.session_key, event.getKeysClaimed(),
-    );
+        content.room_id, senderKey, forwardingKeyChain, sessionId,
+        content.session_key, keysClaimed,
+        exportFormat,
+    ).then(() => {
+        // cancel any outstanding room key requests for this session
+        this._crypto.cancelRoomKeyRequest({
+            algorithm: content.algorithm,
+            room_id: content.room_id,
+            session_id: content.session_id,
+            sender_key: senderKey,
+        });
 
-    // have another go at decrypting events sent with this session.
-    this._retryDecryption(senderKey, sessionId);
+        // have another go at decrypting events sent with this session.
+        this._retryDecryption(senderKey, sessionId);
+    }).catch((e) => {
+        console.error(`Error handling m.room_key_event: ${e}`);
+    });
 };
 
+/**
+ * @inheritdoc
+ */
+MegolmDecryption.prototype.hasKeysForKeyRequest = function(keyRequest) {
+    const body = keyRequest.requestBody;
+
+    return this._olmDevice.hasInboundSessionKeys(
+        body.room_id,
+        body.sender_key,
+        body.session_id,
+        // TODO: ratchet index
+    );
+};
+
+/**
+ * @inheritdoc
+ */
+MegolmDecryption.prototype.shareKeysWithDevice = function(keyRequest) {
+    const userId = keyRequest.userId;
+    const deviceId = keyRequest.deviceId;
+    const deviceInfo = this._crypto.getStoredDevice(userId, deviceId);
+    const body = keyRequest.requestBody;
+
+    this.olmlib.ensureOlmSessionsForDevices(
+        this._olmDevice, this._baseApis, {
+            [userId]: [deviceInfo],
+        },
+    ).then((devicemap) => {
+        const olmSessionResult = devicemap[userId][deviceId];
+        if (!olmSessionResult.sessionId) {
+            // no session with this device, probably because there
+            // were no one-time keys.
+            //
+            // ensureOlmSessionsForUsers has already done the logging,
+            // so just skip it.
+            return null;
+        }
+
+        console.log(
+            "sharing keys for session " + body.sender_key + "|"
+            + body.session_id + " with device "
+            + userId + ":" + deviceId,
+        );
+
+        return this._buildKeyForwardingMessage(
+            body.room_id, body.sender_key, body.session_id,
+        );
+    }).then((payload) => {
+        const encryptedContent = {
+            algorithm: olmlib.OLM_ALGORITHM,
+            sender_key: this._olmDevice.deviceCurve25519Key,
+            ciphertext: {},
+        };
+
+        return this.olmlib.encryptMessageForDevice(
+            encryptedContent.ciphertext,
+            this._userId,
+            this._deviceId,
+            this._olmDevice,
+            userId,
+            deviceInfo,
+            payload,
+        ).then(() => {
+            const contentMap = {
+                [userId]: {
+                    [deviceId]: encryptedContent,
+                },
+            };
+
+            // TODO: retries
+            return this._baseApis.sendToDevice("m.room.encrypted", contentMap);
+        });
+    }).done();
+};
+
+MegolmDecryption.prototype._buildKeyForwardingMessage = async function(
+    roomId, senderKey, sessionId,
+) {
+    const key = await this._olmDevice.getInboundGroupSessionKey(
+        roomId, senderKey, sessionId,
+    );
+
+    return {
+        type: "m.forwarded_room_key",
+        content: {
+            algorithm: olmlib.MEGOLM_ALGORITHM,
+            room_id: roomId,
+            sender_key: senderKey,
+            sender_claimed_ed25519_key: key.sender_claimed_ed25519_key,
+            session_id: sessionId,
+            session_key: key.key,
+            chain_index: key.chain_index,
+            forwarding_curve25519_key_chain: key.forwarding_curve25519_key_chain,
+        },
+    };
+};
 
 /**
  * @inheritdoc
@@ -655,13 +945,8 @@ MegolmDecryption.prototype._retryDecryption = function(senderKey, sessionId) {
 
     delete this._pendingEvents[k];
 
-    for (let i = 0; i < pending.length; i++) {
-        try {
-            this.decryptEvent(pending[i]);
-            console.log("successful re-decryption of", pending[i]);
-        } catch (e) {
-            console.log("Still can't decrypt", pending[i], e.stack || e);
-        }
+    for (const ev of pending) {
+        ev.attemptDecryption(this._crypto);
     }
 };
 

src/crypto/algorithms/olm.js

@@ -20,7 +20,7 @@ limitations under the License.
  *
  * @module crypto/algorithms/olm
  */
-const q = require('q');
+import Promise from 'bluebird';
 
 const utils = require("../../utils");
 const olmlib = require("../olmlib");
@@ -60,7 +60,7 @@ OlmEncryption.prototype._ensureSession = function(roomMembers) {
 
     if (this._sessionPrepared) {
         // prep already done
-        return q();
+        return Promise.resolve();
     }
 
     const self = this;
@@ -107,6 +107,8 @@ OlmEncryption.prototype.encryptMessage = function(room, eventType, content) {
             ciphertext: {},
         };
 
+        const promises = [];
+
         for (let i = 0; i < users.length; ++i) {
             const userId = users[i];
             const devices = self._crypto.getStoredDevicesForUser(userId);
@@ -123,15 +125,17 @@ OlmEncryption.prototype.encryptMessage = function(room, eventType, content) {
                     continue;
                 }
 
-                olmlib.encryptMessageForDevice(
-                    encryptedContent.ciphertext,
-                    self._userId, self._deviceId, self._olmDevice,
-                    userId, deviceInfo, payloadFields,
+                promises.push(
+                    olmlib.encryptMessageForDevice(
+                        encryptedContent.ciphertext,
+                        self._userId, self._deviceId, self._olmDevice,
+                        userId, deviceInfo, payloadFields,
+                    ),
                 );
             }
         }
 
-        return encryptedContent;
+        return Promise.all(promises).return(encryptedContent);
     });
 };
 
@@ -153,10 +157,12 @@ utils.inherits(OlmDecryption, base.DecryptionAlgorithm);
  *
  * @param {MatrixEvent} event
  *
- * @throws {module:crypto/algorithms/base.DecryptionError} if there is a
- *   problem decrypting the event
+ * returns a promise which resolves to a
+ * {@link module:crypto~EventDecryptionResult} once we have finished
+ * decrypting. Rejects with an `algorithms.DecryptionError` if there is a
+ * problem decrypting the event.
  */
-OlmDecryption.prototype.decryptEvent = function(event) {
+OlmDecryption.prototype.decryptEvent = async function(event) {
     const content = event.getWireContent();
     const deviceKey = content.sender_key;
     const ciphertext = content.ciphertext;
@@ -172,7 +178,7 @@ OlmDecryption.prototype.decryptEvent = function(event) {
     let payloadString;
 
     try {
-        payloadString = this._decryptMessage(deviceKey, message);
+        payloadString = await this._decryptMessage(deviceKey, message);
     } catch (e) {
         throw new base.DecryptionError(
             "Bad Encrypted Message", {
@@ -222,9 +228,14 @@ OlmDecryption.prototype.decryptEvent = function(event) {
         );
     }
 
-    event.setClearData(payload, {curve25519: deviceKey}, payload.keys || {});
-};
+    const claimedKeys = payload.keys || {};
 
+    return {
+        clearEvent: payload,
+        senderCurve25519Key: deviceKey,
+        claimedEd25519Key: claimedKeys.ed25519 || null,
+    };
+};
 
 /**
  * Attempt to decrypt an Olm message
@@ -234,15 +245,19 @@ OlmDecryption.prototype.decryptEvent = function(event) {
  *
  * @return {string} payload, if decrypted successfully.
  */
-OlmDecryption.prototype._decryptMessage = function(theirDeviceIdentityKey, message) {
-    const sessionIds = this._olmDevice.getSessionIdsForDevice(theirDeviceIdentityKey);
+OlmDecryption.prototype._decryptMessage = async function(
+    theirDeviceIdentityKey, message,
+) {
+    const sessionIds = await this._olmDevice.getSessionIdsForDevice(
+        theirDeviceIdentityKey,
+    );
 
     // try each session in turn.
     const decryptionErrors = {};
     for (let i = 0; i < sessionIds.length; i++) {
         const sessionId = sessionIds[i];
         try {
-            const payload = this._olmDevice.decryptMessage(
+            const payload = await this._olmDevice.decryptMessage(
                 theirDeviceIdentityKey, sessionId, message.type, message.body,
             );
             console.log(
@@ -251,7 +266,7 @@ OlmDecryption.prototype._decryptMessage = function(theirDeviceIdentityKey, messa
             );
             return payload;
         } catch (e) {
-            const foundSession = this._olmDevice.matchesSession(
+            const foundSession = await this._olmDevice.matchesSession(
                 theirDeviceIdentityKey, sessionId, message.type, message.body,
             );
 
@@ -289,7 +304,7 @@ OlmDecryption.prototype._decryptMessage = function(theirDeviceIdentityKey, messa
 
     let res;
     try {
-        res = this._olmDevice.createInboundSession(
+        res = await this._olmDevice.createInboundSession(
             theirDeviceIdentityKey, message.type, message.body,
         );
     } catch (e) {

src/crypto/olmlib.js

@@ -20,7 +20,7 @@ limitations under the License.
  * Utilities common to olm encryption algorithms
  */
 
-const q = require('q');
+import Promise from 'bluebird';
 const anotherjson = require('another-json');
 
 const utils = require("../utils");
@@ -48,14 +48,17 @@ module.exports.MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2";
  * @param {string} recipientUserId
  * @param {module:crypto/deviceinfo} recipientDevice
  * @param {object} payloadFields fields to include in the encrypted payload
+ *
+ * Returns a promise which resolves (to undefined) when the payload
+ *    has been encrypted into `resultsObject`
  */
-module.exports.encryptMessageForDevice = function(
+module.exports.encryptMessageForDevice = async function(
     resultsObject,
     ourUserId, ourDeviceId, olmDevice, recipientUserId, recipientDevice,
     payloadFields,
 ) {
     const deviceKey = recipientDevice.getIdentityKey();
-    const sessionId = olmDevice.getSessionIdForDevice(deviceKey);
+    const sessionId = await olmDevice.getSessionIdForDevice(deviceKey);
     if (sessionId === null) {
         // If we don't have a session for a device then
         // we can't encrypt a message for it.
@@ -99,7 +102,7 @@ module.exports.encryptMessageForDevice = function(
 
     utils.extend(payload, payloadFields);
 
-    resultsObject[deviceKey] = olmDevice.encryptMessage(
+    resultsObject[deviceKey] = await olmDevice.encryptMessage(
         deviceKey, sessionId, JSON.stringify(payload),
     );
 };
@@ -118,7 +121,7 @@ module.exports.encryptMessageForDevice = function(
  *    an Object mapping from userId to deviceId to
  *    {@link module:crypto~OlmSessionResult}
  */
-module.exports.ensureOlmSessionsForDevices = function(
+module.exports.ensureOlmSessionsForDevices = async function(
     olmDevice, baseApis, devicesByUser,
 ) {
     const devicesWithoutSession = [
@@ -136,7 +139,7 @@ module.exports.ensureOlmSessionsForDevices = function(
             const deviceInfo = devices[j];
             const deviceId = deviceInfo.deviceId;
             const key = deviceInfo.getIdentityKey();
-            const sessionId = olmDevice.getSessionIdForDevice(key);
+            const sessionId = await olmDevice.getSessionIdForDevice(key);
             if (sessionId === null) {
                 devicesWithoutSession.push([userId, deviceId]);
             }
@@ -148,7 +151,7 @@ module.exports.ensureOlmSessionsForDevices = function(
     }
 
     if (devicesWithoutSession.length === 0) {
-        return q(result);
+        return result;
     }
 
     // TODO: this has a race condition - if we try to send another message
@@ -158,55 +161,60 @@ module.exports.ensureOlmSessionsForDevices = function(
     // That should eventually resolve itself, but it's poor form.
 
     const oneTimeKeyAlgorithm = "signed_curve25519";
-    return baseApis.claimOneTimeKeys(
+    const res = await baseApis.claimOneTimeKeys(
         devicesWithoutSession, oneTimeKeyAlgorithm,
-    ).then(function(res) {
-        const otk_res = res.one_time_keys || {};
-        for (const userId in devicesByUser) {
-            if (!devicesByUser.hasOwnProperty(userId)) {
+    );
+
+    const otk_res = res.one_time_keys || {};
+    const promises = [];
+    for (const userId in devicesByUser) {
+        if (!devicesByUser.hasOwnProperty(userId)) {
+            continue;
+        }
+        const userRes = otk_res[userId] || {};
+        const devices = devicesByUser[userId];
+        for (let j = 0; j < devices.length; j++) {
+            const deviceInfo = devices[j];
+            const deviceId = deviceInfo.deviceId;
+            if (result[userId][deviceId].sessionId) {
+                // we already have a result for this device
                 continue;
             }
-            const userRes = otk_res[userId] || {};
-            const devices = devicesByUser[userId];
-            for (let j = 0; j < devices.length; j++) {
-                const deviceInfo = devices[j];
-                const deviceId = deviceInfo.deviceId;
-                if (result[userId][deviceId].sessionId) {
-                    // we already have a result for this device
-                    continue;
-                }
 
-                const deviceRes = userRes[deviceId] || {};
-                let oneTimeKey = null;
-                for (const keyId in deviceRes) {
-                    if (keyId.indexOf(oneTimeKeyAlgorithm + ":") === 0) {
-                        oneTimeKey = deviceRes[keyId];
-                    }
+            const deviceRes = userRes[deviceId] || {};
+            let oneTimeKey = null;
+            for (const keyId in deviceRes) {
+                if (keyId.indexOf(oneTimeKeyAlgorithm + ":") === 0) {
+                    oneTimeKey = deviceRes[keyId];
                 }
-
-                if (!oneTimeKey) {
-                    console.warn(
-                        "No one-time keys (alg=" + oneTimeKeyAlgorithm +
-                            ") for device " + userId + ":" + deviceId,
-                    );
-                    continue;
-                }
-
-                const sid = _verifyKeyAndStartSession(
-                    olmDevice, oneTimeKey, userId, deviceInfo,
-                );
-                result[userId][deviceId].sessionId = sid;
             }
+
+            if (!oneTimeKey) {
+                console.warn(
+                    "No one-time keys (alg=" + oneTimeKeyAlgorithm +
+                        ") for device " + userId + ":" + deviceId,
+                );
+                continue;
+            }
+
+            promises.push(
+                _verifyKeyAndStartSession(
+                    olmDevice, oneTimeKey, userId, deviceInfo,
+                ).then((sid) => {
+                    result[userId][deviceId].sessionId = sid;
+                }),
+            );
         }
-        return result;
-    });
+    }
+
+    await Promise.all(promises);
+    return result;
 };
 
-
-function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceInfo) {
+async function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceInfo) {
     const deviceId = deviceInfo.deviceId;
     try {
-        _verifySignature(
+        await _verifySignature(
             olmDevice, oneTimeKey, userId, deviceId,
             deviceInfo.getFingerprint(),
         );
@@ -220,7 +228,7 @@ function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceInfo) {
 
     let sid;
     try {
-        sid = olmDevice.createOutboundSession(
+        sid = await olmDevice.createOutboundSession(
             deviceInfo.getIdentityKey(), oneTimeKey.key,
         );
     } catch (e) {
@@ -249,8 +257,11 @@ function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceInfo) {
  * @param {string} signingDeviceId  ID of the device whose signature should be checked
  *
  * @param {string} signingKey   base64-ed ed25519 public key
+ *
+ * Returns a promise which resolves (to undefined) if the the signature is good,
+ * or rejects with an Error if it is bad.
  */
-const _verifySignature = module.exports.verifySignature = function(
+const _verifySignature = module.exports.verifySignature = async function(
     olmDevice, obj, signingUserId, signingDeviceId, signingKey,
 ) {
     const signKeyId = "ed25519:" + signingDeviceId;

src/crypto/store/base.js

@@ -0,0 +1,34 @@
+/**
+ * Internal module. Defintions for storage for the crypto module
+ *
+ * @module
+ */
+
+/**
+ * Abstraction of things that can store data required for end-to-end encryption
+ *
+ * @interface CryptoStore
+ */
+
+/**
+ * Represents an outgoing room key request
+ *
+ * @typedef {Object} OutgoingRoomKeyRequest
+ *
+ * @property {string} requestId    unique id for this request. Used for both
+ *    an id within the request for later pairing with a cancellation, and for
+ *    the transaction id when sending the to_device messages to our local
+ *    server.
+ *
+ * @property {string?} cancellationTxnId
+ *    transaction id for the cancellation, if any
+ *
+ * @property {Array<{userId: string, deviceId: string}>} recipients
+ *    list of recipients for the request
+ *
+ * @property {module:crypto~RoomKeyRequestBody} requestBody
+ *    parameters for the request.
+ *
+ * @property {Number} state   current state of this request (states are defined
+ *    in {@link module:crypto/OutgoingRoomKeyRequestManager~ROOM_KEY_REQUEST_STATES})
+ */

src/crypto/store/indexeddb-crypto-store-backend.js

@@ -0,0 +1,291 @@
+import Promise from 'bluebird';
+import utils from '../../utils';
+
+export const VERSION = 1;
+
+/**
+ * Implementation of a CryptoStore which is backed by an existing
+ * IndexedDB connection. Generally you want IndexedDBCryptoStore
+ * which connects to the database and defers to one of these.
+ *
+ * @implements {module:crypto/store/base~CryptoStore}
+ */
+export class Backend {
+    /**
+     * @param {IDBDatabase} db
+     */
+    constructor(db) {
+        this._db = db;
+
+        // make sure we close the db on `onversionchange` - otherwise
+        // attempts to delete the database will block (and subsequent
+        // attempts to re-create it will also block).
+        db.onversionchange = (ev) => {
+            console.log(`versionchange for indexeddb ${this._dbName}: closing`);
+            db.close();
+        };
+    }
+
+    /**
+     * Look for an existing outgoing room key request, and if none is found,
+     * add a new one
+     *
+     * @param {module:crypto/store/base~OutgoingRoomKeyRequest} request
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}: either the
+     *    same instance as passed in, or the existing one.
+     */
+    getOrAddOutgoingRoomKeyRequest(request) {
+        const requestBody = request.requestBody;
+
+        const deferred = Promise.defer();
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readwrite");
+        txn.onerror = deferred.reject;
+
+        // first see if we already have an entry for this request.
+        this._getOutgoingRoomKeyRequest(txn, requestBody, (existing) => {
+            if (existing) {
+                // this entry matches the request - return it.
+                console.log(
+                    `already have key request outstanding for ` +
+                        `${requestBody.room_id} / ${requestBody.session_id}: ` +
+                        `not sending another`,
+                );
+                deferred.resolve(existing);
+                return;
+            }
+
+            // we got to the end of the list without finding a match
+            // - add the new request.
+            console.log(
+                `enqueueing key request for ${requestBody.room_id} / ` +
+                    requestBody.session_id,
+            );
+            const store = txn.objectStore("outgoingRoomKeyRequests");
+            store.add(request);
+            txn.onsuccess = () => { deferred.resolve(request); };
+        });
+
+        return deferred.promise;
+    }
+
+    /**
+     * Look for an existing room key request
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     *
+     * @return {Promise} resolves to the matching
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    not found
+     */
+    getOutgoingRoomKeyRequest(requestBody) {
+        const deferred = Promise.defer();
+
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readonly");
+        txn.onerror = deferred.reject;
+
+        this._getOutgoingRoomKeyRequest(txn, requestBody, (existing) => {
+            deferred.resolve(existing);
+        });
+        return deferred.promise;
+    }
+
+    /**
+     * look for an existing room key request in the db
+     *
+     * @private
+     * @param {IDBTransaction} txn  database transaction
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     * @param {Function} callback  function to call with the results of the
+     *    search. Either passed a matching
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    not found.
+     */
+    _getOutgoingRoomKeyRequest(txn, requestBody, callback) {
+        const store = txn.objectStore("outgoingRoomKeyRequests");
+
+        const idx = store.index("session");
+        const cursorReq = idx.openCursor([
+            requestBody.room_id,
+            requestBody.session_id,
+        ]);
+
+        cursorReq.onsuccess = (ev) => {
+            const cursor = ev.target.result;
+            if(!cursor) {
+                // no match found
+                callback(null);
+                return;
+            }
+
+            const existing = cursor.value;
+
+            if (utils.deepCompare(existing.requestBody, requestBody)) {
+                // got a match
+                callback(existing);
+                return;
+            }
+
+            // look at the next entry in the index
+            cursor.continue();
+        };
+    }
+
+    /**
+     * Look for room key requests by state
+     *
+     * @param {Array<Number>} wantedStates list of acceptable states
+     *
+     * @return {Promise} resolves to the a
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    there are no pending requests in those states. If there are multiple
+     *    requests in those states, an arbitrary one is chosen.
+     */
+    getOutgoingRoomKeyRequestByState(wantedStates) {
+        if (wantedStates.length === 0) {
+            return Promise.resolve(null);
+        }
+
+        // this is a bit tortuous because we need to make sure we do the lookup
+        // in a single transaction, to avoid having a race with the insertion
+        // code.
+
+        // index into the wantedStates array
+        let stateIndex = 0;
+        let result;
+
+        function onsuccess(ev) {
+            const cursor = ev.target.result;
+            if (cursor) {
+                // got a match
+                result = cursor.value;
+                return;
+            }
+
+            // try the next state in the list
+            stateIndex++;
+            if (stateIndex >= wantedStates.length) {
+                // no matches
+                return;
+            }
+
+            const wantedState = wantedStates[stateIndex];
+            const cursorReq = ev.target.source.openCursor(wantedState);
+            cursorReq.onsuccess = onsuccess;
+        }
+
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readonly");
+        const store = txn.objectStore("outgoingRoomKeyRequests");
+
+        const wantedState = wantedStates[stateIndex];
+        const cursorReq = store.index("state").openCursor(wantedState);
+        cursorReq.onsuccess = onsuccess;
+
+        return promiseifyTxn(txn).then(() => result);
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and update it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     * @param {Object} updates        name/value map of updates to apply
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}
+     *    updated request, or null if no matching row was found
+     */
+    updateOutgoingRoomKeyRequest(requestId, expectedState, updates) {
+        let result = null;
+
+        function onsuccess(ev) {
+            const cursor = ev.target.result;
+            if (!cursor) {
+                return;
+            }
+            const data = cursor.value;
+            if (data.state != expectedState) {
+                console.warn(
+                    `Cannot update room key request from ${expectedState} ` +
+                    `as it was already updated to ${data.state}`,
+                );
+                return;
+            }
+            Object.assign(data, updates);
+            cursor.update(data);
+            result = data;
+        }
+
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readwrite");
+        const cursorReq = txn.objectStore("outgoingRoomKeyRequests")
+                  .openCursor(requestId);
+        cursorReq.onsuccess = onsuccess;
+        return promiseifyTxn(txn).then(() => result);
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and delete it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     *
+     * @returns {Promise} resolves once the operation is completed
+     */
+    deleteOutgoingRoomKeyRequest(requestId, expectedState) {
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readwrite");
+        const cursorReq = txn.objectStore("outgoingRoomKeyRequests")
+                  .openCursor(requestId);
+        cursorReq.onsuccess = (ev) => {
+            const cursor = ev.target.result;
+            if (!cursor) {
+                return;
+            }
+            const data = cursor.value;
+            if (data.state != expectedState) {
+                console.warn(
+                    `Cannot delete room key request in state ${data.state} `
+                        + `(expected ${expectedState})`,
+                );
+                return;
+            }
+            cursor.delete();
+        };
+        return promiseifyTxn(txn);
+    }
+}
+
+export function upgradeDatabase(db, oldVersion) {
+    console.log(
+        `Upgrading IndexedDBCryptoStore from version ${oldVersion}`
+            + ` to ${VERSION}`,
+    );
+    if (oldVersion < 1) { // The database did not previously exist.
+        createDatabase(db);
+    }
+    // Expand as needed.
+}
+
+function createDatabase(db) {
+    const outgoingRoomKeyRequestsStore =
+        db.createObjectStore("outgoingRoomKeyRequests", { keyPath: "requestId" });
+
+    // we assume that the RoomKeyRequestBody will have room_id and session_id
+    // properties, to make the index efficient.
+    outgoingRoomKeyRequestsStore.createIndex("session",
+        ["requestBody.room_id", "requestBody.session_id"],
+    );
+
+    outgoingRoomKeyRequestsStore.createIndex("state", "state");
+}
+
+function promiseifyTxn(txn) {
+    return new Promise((resolve, reject) => {
+        txn.oncomplete = resolve;
+        txn.onerror = reject;
+    });
+}

src/crypto/store/indexeddb-crypto-store.js

@@ -0,0 +1,229 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+
+import MemoryCryptoStore from './memory-crypto-store';
+import * as IndexedDBCryptoStoreBackend from './indexeddb-crypto-store-backend';
+
+/**
+ * Internal module. indexeddb storage for e2e.
+ *
+ * @module
+ */
+
+/**
+ * An implementation of CryptoStore, which is normally backed by an indexeddb,
+ * but with fallback to MemoryCryptoStore.
+ *
+ * @implements {module:crypto/store/base~CryptoStore}
+ */
+export default class IndexedDBCryptoStore {
+    /**
+     * Create a new IndexedDBCryptoStore
+     *
+     * @param {IDBFactory} indexedDB  global indexedDB instance
+     * @param {string} dbName   name of db to connect to
+     */
+    constructor(indexedDB, dbName) {
+        this._indexedDB = indexedDB;
+        this._dbName = dbName;
+        this._backendPromise = null;
+    }
+
+    /**
+     * Ensure the database exists and is up-to-date, or fall back to
+     * an in-memory store.
+     *
+     * @return {Promise} resolves to either an IndexedDBCryptoStoreBackend.Backend,
+     * or a MemoryCryptoStore
+     */
+    _connect() {
+        if (this._backendPromise) {
+            return this._backendPromise;
+        }
+
+        this._backendPromise = new Promise((resolve, reject) => {
+            if (!this._indexedDB) {
+                reject(new Error('no indexeddb support available'));
+                return;
+            }
+
+            console.log(`connecting to indexeddb ${this._dbName}`);
+
+            const req = this._indexedDB.open(
+                this._dbName, IndexedDBCryptoStoreBackend.VERSION,
+            );
+
+            req.onupgradeneeded = (ev) => {
+                const db = ev.target.result;
+                const oldVersion = ev.oldVersion;
+                IndexedDBCryptoStoreBackend.upgradeDatabase(db, oldVersion);
+            };
+
+            req.onblocked = () => {
+                console.log(
+                    `can't yet open IndexedDBCryptoStore because it is open elsewhere`,
+                );
+            };
+
+            req.onerror = (ev) => {
+                reject(ev.target.error);
+            };
+
+            req.onsuccess = (r) => {
+                const db = r.target.result;
+
+                console.log(`connected to indexeddb ${this._dbName}`);
+                resolve(new IndexedDBCryptoStoreBackend.Backend(db));
+            };
+        }).catch((e) => {
+            // Don't fall back to memory in this case: we'd end up recreating
+            // a new Olm account in memory and advertising new keys for the
+            // same device.
+            if (e.name == 'VersionError') {
+                throw e;
+            }
+            console.warn(
+                `unable to connect to indexeddb ${this._dbName}` +
+                    `: falling back to in-memory store: ${e}`,
+            );
+            return new MemoryCryptoStore();
+        });
+
+        return this._backendPromise;
+    }
+
+    /**
+     * Delete all data from this store.
+     *
+     * @returns {Promise} resolves when the store has been cleared.
+     */
+    deleteAllData() {
+        return new Promise((resolve, reject) => {
+            if (!this._indexedDB) {
+                reject(new Error('no indexeddb support available'));
+                return;
+            }
+
+            console.log(`Removing indexeddb instance: ${this._dbName}`);
+            const req = this._indexedDB.deleteDatabase(this._dbName);
+
+            req.onblocked = () => {
+                console.log(
+                    `can't yet delete IndexedDBCryptoStore because it is open elsewhere`,
+                );
+            };
+
+            req.onerror = (ev) => {
+                reject(ev.target.error);
+            };
+
+            req.onsuccess = () => {
+                console.log(`Removed indexeddb instance: ${this._dbName}`);
+                resolve();
+            };
+        }).catch((e) => {
+            // in firefox, with indexedDB disabled, this fails with a
+            // DOMError. We treat this as non-fatal, so that people can
+            // still use the app.
+            console.warn(`unable to delete IndexedDBCryptoStore: ${e}`);
+        });
+    }
+
+    /**
+     * Look for an existing outgoing room key request, and if none is found,
+     * add a new one
+     *
+     * @param {module:crypto/store/base~OutgoingRoomKeyRequest} request
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}: either the
+     *    same instance as passed in, or the existing one.
+     */
+    getOrAddOutgoingRoomKeyRequest(request) {
+        return this._connect().then((backend) => {
+            return backend.getOrAddOutgoingRoomKeyRequest(request);
+        });
+    }
+
+    /**
+     * Look for an existing room key request
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     *
+     * @return {Promise} resolves to the matching
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    not found
+     */
+    getOutgoingRoomKeyRequest(requestBody) {
+        return this._connect().then((backend) => {
+            return backend.getOutgoingRoomKeyRequest(requestBody);
+        });
+    }
+
+    /**
+     * Look for room key requests by state
+     *
+     * @param {Array<Number>} wantedStates list of acceptable states
+     *
+     * @return {Promise} resolves to the a
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    there are no pending requests in those states. If there are multiple
+     *    requests in those states, an arbitrary one is chosen.
+     */
+    getOutgoingRoomKeyRequestByState(wantedStates) {
+        return this._connect().then((backend) => {
+            return backend.getOutgoingRoomKeyRequestByState(wantedStates);
+        });
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and update it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     * @param {Object} updates        name/value map of updates to apply
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}
+     *    updated request, or null if no matching row was found
+     */
+    updateOutgoingRoomKeyRequest(requestId, expectedState, updates) {
+        return this._connect().then((backend) => {
+            return backend.updateOutgoingRoomKeyRequest(
+                requestId, expectedState, updates,
+            );
+        });
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and delete it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     *
+     * @returns {Promise} resolves once the operation is completed
+     */
+    deleteOutgoingRoomKeyRequest(requestId, expectedState) {
+        return this._connect().then((backend) => {
+            return backend.deleteOutgoingRoomKeyRequest(requestId, expectedState);
+        });
+    }
+}

src/crypto/store/memory-crypto-store.js

@@ -0,0 +1,199 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+
+import utils from '../../utils';
+
+/**
+ * Internal module. in-memory storage for e2e.
+ *
+ * @module
+ */
+
+/**
+ * @implements {module:crypto/store/base~CryptoStore}
+ */
+export default class MemoryCryptoStore {
+    constructor() {
+        this._outgoingRoomKeyRequests = [];
+    }
+
+    /**
+     * Delete all data from this store.
+     *
+     * @returns {Promise} Promise which resolves when the store has been cleared.
+     */
+    deleteAllData() {
+        return Promise.resolve();
+    }
+
+    /**
+     * Look for an existing outgoing room key request, and if none is found,
+     * add a new one
+     *
+     * @param {module:crypto/store/base~OutgoingRoomKeyRequest} request
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}: either the
+     *    same instance as passed in, or the existing one.
+     */
+    getOrAddOutgoingRoomKeyRequest(request) {
+        const requestBody = request.requestBody;
+
+        return Promise.try(() => {
+            // first see if we already have an entry for this request.
+            const existing = this._getOutgoingRoomKeyRequest(requestBody);
+
+            if (existing) {
+                // this entry matches the request - return it.
+                console.log(
+                    `already have key request outstanding for ` +
+                    `${requestBody.room_id} / ${requestBody.session_id}: ` +
+                    `not sending another`,
+                );
+                return existing;
+            }
+
+            // we got to the end of the list without finding a match
+            // - add the new request.
+            console.log(
+                `enqueueing key request for ${requestBody.room_id} / ` +
+                requestBody.session_id,
+            );
+            this._outgoingRoomKeyRequests.push(request);
+            return request;
+        });
+    }
+
+    /**
+     * Look for an existing room key request
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     *
+     * @return {Promise} resolves to the matching
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    not found
+     */
+    getOutgoingRoomKeyRequest(requestBody) {
+        return Promise.resolve(this._getOutgoingRoomKeyRequest(requestBody));
+    }
+
+    /**
+     * Looks for existing room key request, and returns the result synchronously.
+     *
+     * @internal
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     *
+     * @return {module:crypto/store/base~OutgoingRoomKeyRequest?}
+     *    the matching request, or null if not found
+     */
+    _getOutgoingRoomKeyRequest(requestBody) {
+        for (const existing of this._outgoingRoomKeyRequests) {
+            if (utils.deepCompare(existing.requestBody, requestBody)) {
+                return existing;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Look for room key requests by state
+     *
+     * @param {Array<Number>} wantedStates list of acceptable states
+     *
+     * @return {Promise} resolves to the a
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    there are no pending requests in those states
+     */
+    getOutgoingRoomKeyRequestByState(wantedStates) {
+        for (const req of this._outgoingRoomKeyRequests) {
+            for (const state of wantedStates) {
+                if (req.state === state) {
+                    return Promise.resolve(req);
+                }
+            }
+        }
+        return Promise.resolve(null);
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and update it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     * @param {Object} updates        name/value map of updates to apply
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}
+     *    updated request, or null if no matching row was found
+     */
+    updateOutgoingRoomKeyRequest(requestId, expectedState, updates) {
+        for (const req of this._outgoingRoomKeyRequests) {
+            if (req.requestId !== requestId) {
+                continue;
+            }
+
+            if (req.state != expectedState) {
+                console.warn(
+                    `Cannot update room key request from ${expectedState} ` +
+                    `as it was already updated to ${req.state}`,
+                );
+                return Promise.resolve(null);
+            }
+            Object.assign(req, updates);
+            return Promise.resolve(req);
+        }
+
+        return Promise.resolve(null);
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and delete it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     *
+     * @returns {Promise} resolves once the operation is completed
+     */
+    deleteOutgoingRoomKeyRequest(requestId, expectedState) {
+        for (let i = 0; i < this._outgoingRoomKeyRequests.length; i++) {
+            const req = this._outgoingRoomKeyRequests[i];
+
+            if (req.requestId !== requestId) {
+                continue;
+            }
+
+            if (req.state != expectedState) {
+                console.warn(
+                    `Cannot delete room key request in state ${req.state} `
+                    + `(expected ${expectedState})`,
+                );
+                return Promise.resolve(null);
+            }
+
+            this._outgoingRoomKeyRequests.splice(i, 1);
+            return Promise.resolve(req);
+        }
+
+        return Promise.resolve(null);
+    }
+}

src/http-api.js

@@ -18,7 +18,9 @@ limitations under the License.
  * This is an internal module. See {@link MatrixHttpApi} for the public class.
  * @module http-api
  */
-const q = require("q");
+import Promise from 'bluebird';
+const parseContentType = require('content-type').parse;
+
 const utils = require("./utils");
 
 // we use our own implementation of setTimeout, so that if we get suspended in
@@ -64,7 +66,7 @@ module.exports.PREFIX_MEDIA_R0 = "/_matrix/media/r0";
  * @param {string} opts.prefix Required. The matrix client prefix to use, e.g.
  * '/_matrix/client/r0'. See PREFIX_R0 and PREFIX_UNSTABLE for constants.
  *
- * @param {bool=} opts.onlyData True to return only the 'data' component of the
+ * @param {boolean} opts.onlyData True to return only the 'data' component of the
  * response (e.g. the parsed HTTP body). If false, requests will return an
  * object with the properties <tt>code</tt>, <tt>headers</tt> and <tt>data</tt>.
  *
@@ -74,12 +76,15 @@ module.exports.PREFIX_MEDIA_R0 = "/_matrix/media/r0";
  * requests.
  * @param {Number=} opts.localTimeoutMs The default maximum amount of time to wait
  * before timing out the request. If not specified, there is no timeout.
+ * @param {boolean} [opts.useAuthorizationHeader = false] Set to true to use
+ * Authorization header instead of query param to send the access token to the server.
  */
 module.exports.MatrixHttpApi = function MatrixHttpApi(event_emitter, opts) {
     utils.checkObjectHasKeys(opts, ["baseUrl", "request", "prefix"]);
     opts.onlyData = opts.onlyData || false;
     this.event_emitter = event_emitter;
     this.opts = opts;
+    this.useAuthorizationHeader = Boolean(opts.useAuthorizationHeader);
     this.uploads = [];
 };
 
@@ -129,6 +134,10 @@ module.exports.MatrixHttpApi.prototype = {
      *    invoke on success/failure. See the promise return values for more
      *    information.
      *
+     * @param {Function=} opts.progressHandler Optional. Called when a chunk of
+     *    data has been uploaded, with an object containing the fields `loaded`
+     *    (number of bytes transferred) and `total` (total size, if known).
+     *
      * @return {module:client.Promise} Resolves to response object, as
      *    determined by this.opts.onlyData, opts.rawResponse, and
      *    opts.onlyContentUri.  Rejects with an error (usually a MatrixError).
@@ -214,7 +223,7 @@ module.exports.MatrixHttpApi.prototype = {
         }
 
         if (global.XMLHttpRequest) {
-            const defer = q.defer();
+            const defer = Promise.defer();
             const xhr = new global.XMLHttpRequest();
             upload.xhr = xhr;
             const cb = requestCallback(defer, opts.callback, this.opts.onlyData);
@@ -255,7 +264,12 @@ module.exports.MatrixHttpApi.prototype = {
                 upload.loaded = ev.loaded;
                 upload.total = ev.total;
                 xhr.timeout_timer = callbacks.setTimeout(timeout_fn, 30000);
-                defer.notify(ev);
+                if (opts.progressHandler) {
+                    opts.progressHandler({
+                        loaded: ev.loaded,
+                        total: ev.total,
+                    });
+                }
             });
             let url = this.opts.baseUrl + "/_matrix/media/v1/upload";
             url += "?access_token=" + encodeURIComponent(this.opts.accessToken);
@@ -338,7 +352,7 @@ module.exports.MatrixHttpApi.prototype = {
             opts.form = params;
         }
 
-        const defer = q.defer();
+        const defer = Promise.defer();
         this.opts.request(
             opts,
             requestCallback(defer, callback, this.opts.onlyData),
@@ -364,7 +378,8 @@ module.exports.MatrixHttpApi.prototype = {
      *
      * @param {Object} data The HTTP JSON body.
      *
-     * @param {Object=} opts additional options
+     * @param {Object|Number=} opts additional options. If a number is specified,
+     * this is treated as `opts.localTimeoutMs`.
      *
      * @param {Number=} opts.localTimeoutMs The maximum amount of time to wait before
      * timing out the request. If not specified, there is no timeout.
@@ -385,16 +400,37 @@ module.exports.MatrixHttpApi.prototype = {
         if (!queryParams) {
             queryParams = {};
         }
-        if (!queryParams.access_token) {
-            queryParams.access_token = this.opts.accessToken;
+        if (this.useAuthorizationHeader) {
+            if (isFinite(opts)) {
+                // opts used to be localTimeoutMs
+                opts = {
+                    localTimeoutMs: opts,
+                };
+            }
+            if (!opts) {
+                opts = {};
+            }
+            if (!opts.headers) {
+                opts.headers = {};
+            }
+            if (!opts.headers.Authorization) {
+                opts.headers.Authorization = "Bearer " + this.opts.accessToken;
+            }
+            if (queryParams.access_token) {
+                delete queryParams.access_token;
+            }
+        } else {
+            if (!queryParams.access_token) {
+                queryParams.access_token = this.opts.accessToken;
+            }
         }
 
-        const request_promise = this.request(
+        const requestPromise = this.request(
             callback, method, path, queryParams, data, opts,
         );
 
         const self = this;
-        request_promise.catch(function(err) {
+        requestPromise.catch(function(err) {
             if (err.errcode == 'M_UNKNOWN_TOKEN') {
                 self.event_emitter.emit("Session.logged_out");
             }
@@ -402,7 +438,7 @@ module.exports.MatrixHttpApi.prototype = {
 
         // return the original promise, otherwise tests break due to it having to
         // go around the event loop one more time to process the result of the request
-        return request_promise;
+        return requestPromise;
     },
 
     /**
@@ -623,9 +659,33 @@ module.exports.MatrixHttpApi.prototype = {
             }
         }
 
+        const headers = utils.extend({}, opts.headers || {});
         const json = opts.json === undefined ? true : opts.json;
+        let bodyParser = opts.bodyParser;
 
-        const defer = q.defer();
+        // we handle the json encoding/decoding here, because request and
+        // browser-request make a mess of it. Specifically, they attempt to
+        // json-decode plain-text error responses, which in turn means that the
+        // actual error gets swallowed by a SyntaxError.
+
+        if (json) {
+            if (data) {
+                data = JSON.stringify(data);
+                headers['content-type'] = 'application/json';
+            }
+
+            if (!headers['accept']) {
+                headers['accept'] = 'application/json';
+            }
+
+            if (bodyParser === undefined) {
+                bodyParser = function(rawBody) {
+                    return JSON.parse(rawBody);
+                };
+            }
+        }
+
+        const defer = Promise.defer();
 
         let timeoutId;
         let timedOut = false;
@@ -662,7 +722,7 @@ module.exports.MatrixHttpApi.prototype = {
                     withCredentials: false,
                     qs: queryParams,
                     body: data,
-                    json: json,
+                    json: false,
                     timeout: localTimeoutMs,
                     headers: opts.headers || {},
                     _matrix_opts: this.opts,
@@ -675,13 +735,9 @@ module.exports.MatrixHttpApi.prototype = {
                         }
                     }
 
-                    // if json is falsy, we won't parse any error response, so need
-                    // to do so before turning it into a MatrixError
-                    const parseErrorJson = !json;
                     const handlerFn = requestCallback(
                         defer, callback, self.opts.onlyData,
-                        parseErrorJson,
-                        opts.bodyParser,
+                        bodyParser,
                     );
                     handlerFn(err, response, body);
                 },
@@ -718,15 +774,18 @@ module.exports.MatrixHttpApi.prototype = {
  * that will either resolve or reject the given defer as well as invoke the
  * given userDefinedCallback (if any).
  *
- * If onlyData is true, the defer/callback is invoked with the body of the
- * response, otherwise the result code.
+ * HTTP errors are transformed into javascript errors and the deferred is rejected.
  *
- * If parseErrorJson is true, we will JSON.parse the body if we get a 4xx error.
+ * If bodyParser is given, it is used to transform the body of the successful
+ * responses before passing to the defer/callback.
+ *
+ * If onlyData is true, the defer/callback is invoked with the body of the
+ * response, otherwise the result object (with `code` and `data` fields)
  *
  */
 const requestCallback = function(
     defer, userDefinedCallback, onlyData,
-    parseErrorJson, bodyParser,
+    bodyParser,
 ) {
     userDefinedCallback = userDefinedCallback || function() {};
 
@@ -734,19 +793,12 @@ const requestCallback = function(
         if (!err) {
             try {
                 if (response.statusCode >= 400) {
-                    if (parseErrorJson) {
-                        // we won't have json-decoded the response.
-                        body = JSON.parse(body);
-                    }
-                    err = new module.exports.MatrixError(body);
+                    err = parseErrorResponse(response, body);
                 } else if (bodyParser) {
                     body = bodyParser(body);
                 }
             } catch (e) {
-                err = e;
-            }
-            if (err) {
-                err.httpStatus = response.statusCode;
+                err = new Error(`Error parsing server response: ${e}`);
             }
         }
 
@@ -756,6 +808,9 @@ const requestCallback = function(
         } else {
             const res = {
                 code: response.statusCode,
+
+                // XXX: why do we bother with this? it doesn't work for
+                // XMLHttpRequest, so clearly we don't use it.
                 headers: response.headers,
                 data: body,
             };
@@ -765,6 +820,67 @@ const requestCallback = function(
     };
 };
 
+/**
+ * Attempt to turn an HTTP error response into a Javascript Error.
+ *
+ * If it is a JSON response, we will parse it into a MatrixError. Otherwise
+ * we return a generic Error.
+ *
+ * @param {XMLHttpRequest|http.IncomingMessage} response response object
+ * @param {String} body raw body of the response
+ * @returns {Error}
+ */
+function parseErrorResponse(response, body) {
+    const httpStatus = response.statusCode;
+    const contentType = getResponseContentType(response);
+
+    let err;
+    if (contentType) {
+        if (contentType.type === 'application/json') {
+            err = new module.exports.MatrixError(JSON.parse(body));
+        } else if (contentType.type === 'text/plain') {
+            err = new Error(`Server returned ${httpStatus} error: ${body}`);
+        }
+    }
+
+    if (!err) {
+        err = new Error(`Server returned ${httpStatus} error`);
+    }
+    err.httpStatus = httpStatus;
+    return err;
+}
+
+
+/**
+ * extract the Content-Type header from the response object, and
+ * parse it to a `{type, parameters}` object.
+ *
+ * returns null if no content-type header could be found.
+ *
+ * @param {XMLHttpRequest|http.IncomingMessage} response response object
+ * @returns {{type: String, parameters: Object}?} parsed content-type header, or null if not found
+ */
+function getResponseContentType(response) {
+    let contentType;
+    if (response.getResponseHeader) {
+        // XMLHttpRequest provides getResponseHeader
+        contentType = response.getResponseHeader("Content-Type");
+    } else if (response.headers) {
+        // request provides http.IncomingMessage which has a message.headers map
+        contentType = response.headers['content-type'] || null;
+    }
+
+    if (!contentType) {
+        return null;
+    }
+
+    try {
+        return parseContentType(contentType);
+    } catch(e) {
+        throw new Error(`Error parsing Content-Type '${contentType}': ${e}`);
+    }
+}
+
 /**
  * Construct a Matrix error. This is a JavaScript Error with additional
  * information specific to the standard Matrix error response.

src/interactive-auth.js

@@ -17,7 +17,7 @@ limitations under the License.
 "use strict";
 
 /** @module interactive-auth */
-const q = require("q");
+import Promise from 'bluebird';
 const url = require("url");
 
 const utils = require("./utils");
@@ -115,11 +115,11 @@ InteractiveAuth.prototype = {
      *     no suitable authentication flow can be found
      */
     attemptAuth: function() {
-        this._completionDeferred = q.defer();
+        this._completionDeferred = Promise.defer();
 
         // wrap in a promise so that if _startNextAuthStage
         // throws, it rejects the promise in a consistent way
-        return q().then(() => {
+        return Promise.resolve().then(() => {
             // if we have no flows, try a request (we'll have
             // just a session ID in _data if resuming)
             if (!this._data.flows) {
@@ -258,12 +258,12 @@ InteractiveAuth.prototype = {
 
         // hackery to make sure that synchronous exceptions end up in the catch
         // handler (without the additional event loop entailed by q.fcall or an
-        // extra q().then)
+        // extra Promise.resolve().then)
         let prom;
         try {
             prom = this._requestCallback(auth, background);
         } catch (e) {
-            prom = q.reject(e);
+            prom = Promise.reject(e);
         }
 
         prom = prom.then(
@@ -293,7 +293,9 @@ InteractiveAuth.prototype = {
             },
         );
         if (!background) {
-            prom = prom.catch(this._completionDeferred.reject);
+            prom = prom.catch((e) => {
+                this._completionDeferred.reject(e);
+            });
         } else {
             // We ignore all failures here (even non-UI auth related ones)
             // since we don't want to suddenly fail if the internet connection

src/matrix.js

@@ -63,6 +63,11 @@ module.exports.TimelineWindow = require("./timeline-window").TimelineWindow;
 module.exports.InteractiveAuth = require("./interactive-auth");
 
 
+module.exports.MemoryCryptoStore =
+    require("./crypto/store/memory-crypto-store").default;
+module.exports.IndexedDBCryptoStore =
+    require("./crypto/store/indexeddb-crypto-store").default;
+
 /**
  * Create a new Matrix Call.
  * @function
@@ -124,6 +129,19 @@ module.exports.wrapRequest = function(wrapper) {
     };
 };
 
+
+let cryptoStoreFactory = () => new module.exports.MemoryCryptoStore;
+
+/**
+ * Configure a different factory to be used for creating crypto stores
+ *
+ * @param {Function} fac  a function which will return a new
+ *    {@link module:crypto.store.base~CryptoStore}.
+ */
+module.exports.setCryptoStoreFactory = function(fac) {
+    cryptoStoreFactory = fac;
+};
+
 /**
  * Construct a Matrix Client. Similar to {@link module:client~MatrixClient}
  * except that the 'request', 'store' and 'scheduler' dependencies are satisfied.
@@ -136,6 +154,13 @@ module.exports.wrapRequest = function(wrapper) {
  * {@link module:scheduler~MatrixScheduler}.
  * @param {requestFunction} opts.request If not set, defaults to the function
  * supplied to {@link request} which defaults to the request module from NPM.
+ *
+ * @param {module:crypto.store.base~CryptoStore=} opts.cryptoStore
+ *    crypto store implementation. Calls the factory supplied to
+ *    {@link setCryptoStoreFactory} if unspecified; or if no factory has been
+ *    specified, uses a default implementation (indexeddb in the browser,
+ *    in-memory otherwise).
+ *
  * @return {MatrixClient} A new matrix client.
  * @see {@link module:client~MatrixClient} for the full list of options for
  * <code>opts</code>.
@@ -151,6 +176,7 @@ module.exports.createClient = function(opts) {
       localStorage: global.localStorage,
     });
     opts.scheduler = opts.scheduler || new module.exports.MatrixScheduler();
+    opts.cryptoStore = opts.cryptoStore || cryptoStoreFactory();
     return new module.exports.MatrixClient(opts);
 };
 

src/models/event-timeline-set.js

@@ -149,13 +149,24 @@ EventTimelineSet.prototype.replaceEventId = function(oldEventId, newEventId) {
  * <p>This is used when /sync returns a 'limited' timeline.
  *
  * @param {string=} backPaginationToken   token for back-paginating the new timeline
- * @param {?bool} flush  Whether to flush the non-live timelines too.
+ * @param {string=} forwardPaginationToken token for forward-paginating the old live timeline,
+ * if absent or null, all timelines are reset.
  *
  * @fires module:client~MatrixClient#event:"Room.timelineReset"
  */
-EventTimelineSet.prototype.resetLiveTimeline = function(backPaginationToken, flush) {
+EventTimelineSet.prototype.resetLiveTimeline = function(
+    backPaginationToken, forwardPaginationToken,
+) {
+    // Each EventTimeline has RoomState objects tracking the state at the start
+    // and end of that timeline. The copies at the end of the live timeline are
+    // special because they will have listeners attached to monitor changes to
+    // the current room state, so we move this RoomState from the end of the
+    // current live timeline to the end of the new one and, if necessary,
+    // replace it with a newly created one. We also make a copy for the start
+    // of the new timeline.
+
     // if timeline support is disabled, forget about the old timelines
-    const resetAllTimelines = !this._timelineSupport || flush;
+    const resetAllTimelines = !this._timelineSupport || !forwardPaginationToken;
 
     let newTimeline;
     if (resetAllTimelines) {
@@ -166,8 +177,10 @@ EventTimelineSet.prototype.resetLiveTimeline = function(backPaginationToken, flu
         newTimeline = this.addTimeline();
     }
 
-    // initialise the state in the new timeline from our last known state
-    const evMap = this._liveTimeline.getState(EventTimeline.FORWARDS).events;
+    const oldTimeline = this._liveTimeline;
+
+    // Collect the state events from the old timeline
+    const evMap = oldTimeline.getState(EventTimeline.FORWARDS).events;
     const events = [];
     for (const evtype in evMap) {
         if (!evMap.hasOwnProperty(evtype)) {
@@ -180,13 +193,37 @@ EventTimelineSet.prototype.resetLiveTimeline = function(backPaginationToken, flu
             events.push(evMap[evtype][stateKey]);
         }
     }
+
+    // Use those events to initialise the state of the new live timeline
     newTimeline.initialiseState(events);
 
+    const freshEndState = newTimeline._endState;
+    // Now clobber the end state of the new live timeline with that from the
+    // previous live timeline. It will be identical except that we'll keep
+    // using the same RoomMember objects for the 'live' set of members with any
+    // listeners still attached
+    newTimeline._endState = oldTimeline._endState;
+
+    // If we're not resetting all timelines, we need to fix up the old live timeline
+    if (!resetAllTimelines) {
+        // Firstly, we just stole the old timeline's end state, so it needs a new one.
+        // Just swap them around and give it the one we just generated for the
+        // new live timeline.
+        oldTimeline._endState = freshEndState;
+
+        // Now set the forward pagination token on the old live timeline
+        // so it can be forward-paginated.
+        oldTimeline.setPaginationToken(
+            forwardPaginationToken, EventTimeline.FORWARDS,
+        );
+    }
+
     // make sure we set the pagination token before firing timelineReset,
     // otherwise clients which start back-paginating will fail, and then get
     // stuck without realising that they *can* back-paginate.
     newTimeline.setPaginationToken(backPaginationToken, EventTimeline.BACKWARDS);
 
+    // Now we can swap the live timeline to the new one.
     this._liveTimeline = newTimeline;
     this.emit("Room.timelineReset", this.room, this, resetAllTimelines);
 };

src/models/event.js

@@ -21,9 +21,9 @@ limitations under the License.
  * @module models/event
  */
 
-const EventEmitter = require("events").EventEmitter;
-
-const utils = require('../utils.js');
+import Promise from 'bluebird';
+import {EventEmitter} from 'events';
+import utils from '../utils.js';
 
 /**
  * Enum for event statuses.
@@ -112,8 +112,33 @@ module.exports.MatrixEvent = function MatrixEvent(
         new Date(this.event.origin_server_ts) : null;
 
     this._clearEvent = {};
-    this._keysProved = {};
-    this._keysClaimed = {};
+
+    /* curve25519 key which we believe belongs to the sender of the event. See
+     * getSenderKey()
+     */
+    this._senderCurve25519Key = null;
+
+    /* ed25519 key which the sender of this event (for olm) or the creator of
+     * the megolm session (for megolm) claims to own. See getClaimedEd25519Key()
+     */
+    this._claimedEd25519Key = null;
+
+    /* curve25519 keys of devices involved in telling us about the
+     * _senderCurve25519Key and _claimedEd25519Key.
+     * See getForwardingCurve25519KeyChain().
+     */
+    this._forwardingCurve25519KeyChain = [];
+
+    /* if we have a process decrypting this event, a Promise which resolves
+     * when it is finished. Normally null.
+     */
+    this._decryptionPromise = null;
+
+    /* flag to indicate if we should retry decrypting this event after the
+     * first attempt (eg, we have received new data which means that a second
+     * attempt may succeed)
+     */
+    this._retryDecryption = false;
 };
 utils.inherits(module.exports.MatrixEvent, EventEmitter);
 
@@ -261,9 +286,18 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * <tt>"m.room.encrypted"</tt>
      *
      * @param {object} crypto_content raw 'content' for the encrypted event.
-     * @param {object} keys The local keys claimed and proved by this event.
+     *
+     * @param {string} senderCurve25519Key curve25519 key to record for the
+     *   sender of this event.
+     *   See {@link module:models/event.MatrixEvent#getSenderKey}.
+     *
+     * @param {string} claimedEd25519Key claimed ed25519 key to record for the
+     *   sender if this event.
+     *   See {@link module:models/event.MatrixEvent#getClaimedEd25519Key}
      */
-    makeEncrypted: function(crypto_type, crypto_content, keys) {
+    makeEncrypted: function(
+        crypto_type, crypto_content, senderCurve25519Key, claimedEd25519Key,
+    ) {
         // keep the plain-text data for 'view source'
         this._clearEvent = {
             type: this.event.type,
@@ -271,8 +305,166 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         };
         this.event.type = crypto_type;
         this.event.content = crypto_content;
-        this._keysProved = keys;
-        this._keysClaimed = keys;
+        this._senderCurve25519Key = senderCurve25519Key;
+        this._claimedEd25519Key = claimedEd25519Key;
+    },
+
+    /**
+     * Check if this event is currently being decrypted.
+     *
+     * @return {boolean} True if this event is currently being decrypted, else false.
+     */
+    isBeingDecrypted: function() {
+        return this._decryptionPromise != null;
+    },
+
+    /**
+     * Check if this event is an encrypted event which we failed to decrypt
+     *
+     * (This implies that we might retry decryption at some point in the future)
+     *
+     * @return {boolean} True if this event is an encrypted event which we
+     *     couldn't decrypt.
+     */
+    isDecryptionFailure: function() {
+        return this._clearEvent && this._clearEvent.content &&
+            this._clearEvent.content.msgtype === "m.bad.encrypted";
+    },
+
+    /**
+     * Start the process of trying to decrypt this event.
+     *
+     * (This is used within the SDK: it isn't intended for use by applications)
+     *
+     * @internal
+     *
+     * @param {module:crypto} crypto crypto module
+     *
+     * @returns {Promise} promise which resolves (to undefined) when the decryption
+     * attempt is completed.
+     */
+    attemptDecryption: async function(crypto) {
+        // start with a couple of sanity checks.
+        if (!this.isEncrypted()) {
+            throw new Error("Attempt to decrypt event which isn't encrypted");
+        }
+
+        if (
+            this._clearEvent && this._clearEvent.content &&
+                this._clearEvent.content.msgtype !== "m.bad.encrypted"
+        ) {
+            // we may want to just ignore this? let's start with rejecting it.
+            throw new Error(
+                "Attempt to decrypt event which has already been encrypted",
+            );
+        }
+
+        // if we already have a decryption attempt in progress, then it may
+        // fail because it was using outdated info. We now have reason to
+        // succeed where it failed before, but we don't want to have multiple
+        // attempts going at the same time, so just set a flag that says we have
+        // new info.
+        //
+        if (this._decryptionPromise) {
+            console.log(
+                `Event ${this.getId()} already being decrypted; queueing a retry`,
+            );
+            this._retryDecryption = true;
+            return this._decryptionPromise;
+        }
+
+        this._decryptionPromise = this._decryptionLoop(crypto);
+        return this._decryptionPromise;
+    },
+
+    _decryptionLoop: async function(crypto) {
+        // make sure that this method never runs completely synchronously.
+        // (doing so would mean that we would clear _decryptionPromise *before*
+        // it is set in attemptDecryption - and hence end up with a stuck
+        // `_decryptionPromise`).
+        await Promise.resolve();
+
+        while (true) {
+            this._retryDecryption = false;
+
+            let res;
+            try {
+                if (!crypto) {
+                    res = this._badEncryptedMessage("Encryption not enabled");
+                } else {
+                    res = await crypto.decryptEvent(this);
+                }
+            } catch (e) {
+                if (e.name !== "DecryptionError") {
+                    // not a decryption error: log the whole exception as an error
+                    // (and don't bother with a retry)
+                    console.error(
+                        `Error decrypting event (id=${this.getId()}): ${e.stack || e}`,
+                    );
+                    this._decryptionPromise = null;
+                    this._retryDecryption = false;
+                    return;
+                }
+
+                // see if we have a retry queued.
+                //
+                // NB: make sure to keep this check in the same tick of the
+                //   event loop as `_decryptionPromise = null` below - otherwise we
+                //   risk a race:
+                //
+                //   * A: we check _retryDecryption here and see that it is
+                //        false
+                //   * B: we get a second call to attemptDecryption, which sees
+                //        that _decryptionPromise is set so sets
+                //        _retryDecryption
+                //   * A: we continue below, clear _decryptionPromise, and
+                //        never do the retry.
+                //
+                if (this._retryDecryption) {
+                    // decryption error, but we have a retry queued.
+                    console.log(
+                        `Got error decrypting event (id=${this.getId()}: ` +
+                        `${e.message}), but retrying`,
+                    );
+                    continue;
+                }
+
+                // decryption error, no retries queued. Warn about the error and
+                // set it to m.bad.encrypted.
+                console.warn(
+                    `Error decrypting event (id=${this.getId()}): ${e}`,
+                );
+
+                res = this._badEncryptedMessage(e.message);
+            }
+
+            // at this point, we've either successfully decrypted the event, or have given up
+            // (and set res to a 'badEncryptedMessage'). Either way, we can now set the
+            // cleartext of the event and raise Event.decrypted.
+            //
+            // make sure we clear '_decryptionPromise' before sending the 'Event.decrypted' event,
+            // otherwise the app will be confused to see `isBeingDecrypted` still set when
+            // there isn't an `Event.decrypted` on the way.
+            //
+            // see also notes on _retryDecryption above.
+            //
+            this._decryptionPromise = null;
+            this._retryDecryption = false;
+            this._setClearData(res);
+            return;
+        }
+    },
+
+    _badEncryptedMessage: function(reason) {
+        return {
+            clearEvent: {
+                type: "m.room.message",
+                content: {
+                    msgtype: "m.bad.encrypted",
+                    body: "** Unable to decrypt: " + reason + " **",
+                },
+            },
+        };
     },
 
     /**
@@ -284,19 +476,17 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      *
      * @fires module:models/event.MatrixEvent#"Event.decrypted"
      *
-     * @param {Object} clearEvent The plaintext payload for the event
-     *     (typically containing <tt>type</tt> and <tt>content</tt> fields).
-     *
-     * @param {Object=} keysProved Keys owned by the sender of this event.
-     *    See {@link module:models/event.MatrixEvent#getKeysProved}.
-     *
-     * @param {Object=} keysClaimed Keys the sender of this event claims.
-     *    See {@link module:models/event.MatrixEvent#getKeysClaimed}.
+     * @param {module:crypto~EventDecryptionResult} decryptionResult
+     *     the decryption result, including the plaintext and some key info
      */
-    setClearData: function(clearEvent, keysProved, keysClaimed) {
-        this._clearEvent = clearEvent;
-        this._keysProved = keysProved || {};
-        this._keysClaimed = keysClaimed || {};
+    _setClearData: function(decryptionResult) {
+        this._clearEvent = decryptionResult.clearEvent;
+        this._senderCurve25519Key =
+            decryptionResult.senderCurve25519Key || null;
+        this._claimedEd25519Key =
+            decryptionResult.claimedEd25519Key || null;
+        this._forwardingCurve25519KeyChain =
+            decryptionResult.forwardingCurve25519KeyChain || [];
         this.emit("Event.decrypted", this);
     },
 
@@ -309,37 +499,72 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     },
 
     /**
-     * The curve25519 key that sent this event
+     * The curve25519 key for the device that we think sent this event
+     *
+     * For an Olm-encrypted event, this is inferred directly from the DH
+     * exchange at the start of the session: the curve25519 key is involved in
+     * the DH exchange, so only a device which holds the private part of that
+     * key can establish such a session.
+     *
+     * For a megolm-encrypted event, it is inferred from the Olm message which
+     * established the megolm session
+     *
      * @return {string}
      */
     getSenderKey: function() {
-        return this.getKeysProved().curve25519 || null;
-    },
-
-    /**
-     * The keys that must have been owned by the sender of this encrypted event.
-     * <p>
-     * These don't necessarily have to come from this event itself, but may be
-     * implied by the cryptographic session.
-     *
-     * @return {Object<string, string>}
-     */
-    getKeysProved: function() {
-        return this._keysProved;
+        return this._senderCurve25519Key;
     },
 
     /**
      * The additional keys the sender of this encrypted event claims to possess.
-     * <p>
-     * These don't necessarily have to come from this event itself, but may be
-     * implied by the cryptographic session.
-     * For example megolm messages don't claim keys directly, but instead
-     * inherit a claim from the olm message that established the session.
+     *
+     * Just a wrapper for #getClaimedEd25519Key (q.v.)
      *
      * @return {Object<string, string>}
      */
     getKeysClaimed: function() {
-        return this._keysClaimed;
+        return {
+            ed25519: this._claimedEd25519Key,
+        };
+    },
+
+    /**
+     * Get the ed25519 the sender of this event claims to own.
+     *
+     * For Olm messages, this claim is encoded directly in the plaintext of the
+     * event itself. For megolm messages, it is implied by the m.room_key event
+     * which established the megolm session.
+     *
+     * Until we download the device list of the sender, it's just a claim: the
+     * device list gives a proof that the owner of the curve25519 key used for
+     * this event (and returned by #getSenderKey) also owns the ed25519 key by
+     * signing the public curve25519 key with the ed25519 key.
+     *
+     * In general, applications should not use this method directly, but should
+     * instead use MatrixClient.getEventSenderDeviceInfo.
+     *
+     * @return {string}
+     */
+    getClaimedEd25519Key: function() {
+        return this._claimedEd25519Key;
+    },
+
+    /**
+     * Get the curve25519 keys of the devices which were involved in telling us
+     * about the claimedEd25519Key and sender curve25519 key.
+     *
+     * Normally this will be empty, but in the case of a forwarded megolm
+     * session, the sender keys are sent to us by another device (the forwarding
+     * device), which we need to trust to do this. In that case, the result will
+     * be a list consisting of one entry.
+     *
+     * If the device that sent us the key (A) got it from another device which
+     * it wasn't prepared to vouch for (B), the result will be [A, B]. And so on.
+     *
+     * @return {string[]} base64-encoded curve25519 keys, from oldest to newest.
+     */
+    getForwardingCurve25519KeyChain: function() {
+        return this._forwardingCurve25519KeyChain;
     },
 
     getUnsigned: function() {

src/models/group.js

@@ -0,0 +1,95 @@
+/*
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module models/group
+ */
+const EventEmitter = require("events").EventEmitter;
+
+const utils = require("../utils");
+
+/**
+ * Construct a new Group.
+ *
+ * @param {string} groupId The ID of this group.
+ *
+ * @prop {string} groupId The ID of this group.
+ * @prop {string} name The human-readable display name for this group.
+ * @prop {string} avatarUrl The mxc URL for this group's avatar.
+ * @prop {string} myMembership The logged in user's membership of this group
+ * @prop {Object} inviter Infomation about the user who invited the logged in user
+ *       to the group, if myMembership is 'invite'.
+ * @prop {string} inviter.userId The user ID of the inviter
+ */
+function Group(groupId) {
+    this.groupId = groupId;
+    this.name = null;
+    this.avatarUrl = null;
+    this.myMembership = null;
+    this.inviter = null;
+}
+utils.inherits(Group, EventEmitter);
+
+Group.prototype.setProfile = function(name, avatarUrl) {
+    if (this.name === name && this.avatarUrl === avatarUrl) return;
+
+    this.name = name || this.groupId;
+    this.avatarUrl = avatarUrl;
+
+    this.emit("Group.profile", this);
+};
+
+Group.prototype.setMyMembership = function(membership) {
+    if (this.myMembership === membership) return;
+
+    this.myMembership = membership;
+
+    this.emit("Group.myMembership", this);
+};
+
+/**
+ * Sets the 'inviter' property. This does not emit an event (the inviter
+ * will only change when the user is revited / reinvited to a room),
+ * so set this before setting myMembership.
+ * @param {Object} inviter Infomation about who invited us to the room
+ */
+Group.prototype.setInviter = function(inviter) {
+    this.inviter = inviter;
+};
+
+module.exports = Group;
+
+/**
+ * Fires whenever a group's profile information is updated.
+ * This means the 'name' and 'avatarUrl' properties.
+ * @event module:client~MatrixClient#"Group.profile"
+ * @param {Group} group The group whose profile was updated.
+ * @example
+ * matrixClient.on("Group.profile", function(group){
+ *   var name = group.name;
+ * });
+ */
+
+/**
+ * Fires whenever the logged in user's membership status of
+ * the group is updated.
+ * @event module:client~MatrixClient#"Group.myMembership"
+ * @param {Group} group The group in which the user's membership changed
+ * @example
+ * matrixClient.on("Group.myMembership", function(group){
+ *   var myMembership = group.myMembership;
+ * });
+ */

src/models/room-member.js

@@ -33,7 +33,10 @@ const utils = require("../utils");
  * @prop {string} roomId The room ID for this member.
  * @prop {string} userId The user ID of this member.
  * @prop {boolean} typing True if the room member is currently typing.
- * @prop {string} name The human-readable name for this room member.
+ * @prop {string} name The human-readable name for this room member. This will be
+ * disambiguated with a suffix of " (@user_id:matrix.org)" if another member shares the
+ * same displayname.
+ * @prop {string} rawDisplayName The ambiguous displayname of this room member.
  * @prop {Number} powerLevel The power level for this room member.
  * @prop {Number} powerLevelNorm The normalised power level (0-100) for this
  * room member.
@@ -47,6 +50,7 @@ function RoomMember(roomId, userId) {
     this.userId = userId;
     this.typing = false;
     this.name = userId;
+    this.rawDisplayName = userId;
     this.powerLevel = 0;
     this.powerLevelNorm = 0;
     this.user = null;
@@ -78,6 +82,7 @@ RoomMember.prototype.setMembershipEvent = function(event, roomState) {
 
     const oldName = this.name;
     this.name = calculateDisplayName(this, event, roomState);
+    this.rawDisplayName = event.getDirectionalContent().displayname || this.userId;
     if (oldMembership !== this.membership) {
         this._updateModifiedTime();
         this.emit("RoomMember.membership", event, this, oldMembership);
@@ -99,17 +104,20 @@ RoomMember.prototype.setPowerLevelEvent = function(powerLevelEvent) {
     if (powerLevelEvent.getType() !== "m.room.power_levels") {
         return;
     }
-    let maxLevel = powerLevelEvent.getContent().users_default || 0;
-    utils.forEach(utils.values(powerLevelEvent.getContent().users), function(lvl) {
+
+    const evContent = powerLevelEvent.getDirectionalContent();
+
+    let maxLevel = evContent.users_default || 0;
+    utils.forEach(utils.values(evContent.users), function(lvl) {
         maxLevel = Math.max(maxLevel, lvl);
     });
     const oldPowerLevel = this.powerLevel;
     const oldPowerLevelNorm = this.powerLevelNorm;
 
-    if (powerLevelEvent.getContent().users[this.userId] !== undefined) {
-        this.powerLevel = powerLevelEvent.getContent().users[this.userId];
-    } else if (powerLevelEvent.getContent().users_default !== undefined) {
-        this.powerLevel = powerLevelEvent.getContent().users_default;
+    if (evContent.users && evContent.users[this.userId] !== undefined) {
+        this.powerLevel = evContent.users[this.userId];
+    } else if (evContent.users_default !== undefined) {
+        this.powerLevel = evContent.users_default;
     } else {
         this.powerLevel = 0;
     }

src/models/room-state.js

@@ -197,6 +197,16 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
                 member.setPowerLevelEvent(event);
                 self.emit("RoomState.members", event, self, member);
             });
+
+            // Go through the sentinel members and see if any of them would be
+            // affected by the new power levels. If so, replace the sentinel.
+            for (const userId of Object.keys(self._sentinels)) {
+                const oldSentinel = self._sentinels[userId];
+                const newSentinel = new RoomMember(event.getRoomId(), userId);
+                newSentinel.setMembershipEvent(oldSentinel.events.member, self);
+                newSentinel.setPowerLevelEvent(event);
+                self._sentinels[userId] = newSentinel;
+            }
         }
     });
 };
@@ -246,6 +256,45 @@ RoomState.prototype.getUserIdsWithDisplayName = function(displayName) {
     return this._displayNameToUserIds[displayName] || [];
 };
 
+/**
+ * Returns true if userId is in room, event is not redacted and either sender of
+ * mxEvent or has power level sufficient to redact events other than their own.
+ * @param {MatrixEvent} mxEvent The event to test permission for
+ * @param {string} userId The user ID of the user to test permission for
+ * @return {boolean} true if the given used ID can redact given event
+ */
+RoomState.prototype.maySendRedactionForEvent = function(mxEvent, userId) {
+    const member = this.getMember(userId);
+    if (!member || member.membership === 'leave') return false;
+
+    if (mxEvent.status || mxEvent.isRedacted()) return false;
+    if (mxEvent.getSender() === userId) return true;
+
+    return this._hasSufficientPowerLevelFor('redact', member.powerLevel);
+};
+
+/**
+ * Returns true if the given power level is sufficient for action
+ * @param {string} action The type of power level to check
+ * @param {number} powerLevel The power level of the member
+ * @return {boolean} true if the given power level is sufficient
+ */
+RoomState.prototype._hasSufficientPowerLevelFor = function(action, powerLevel) {
+    const powerLevelsEvent = this.getStateEvents('m.room.power_levels', '');
+
+    let powerLevels = {};
+    if (powerLevelsEvent) {
+        powerLevels = powerLevelsEvent.getContent();
+    }
+
+    let requiredLevel = 50;
+    if (powerLevels[action] !== undefined) {
+        requiredLevel = powerLevels[action];
+    }
+
+    return powerLevel >= requiredLevel;
+};
+
 /**
  * Short-form for maySendEvent('m.room.message', userId)
  * @param {string} userId The user ID of the user to test permission for
@@ -269,7 +318,6 @@ RoomState.prototype.maySendEvent = function(eventType, userId) {
     return this._maySendEventOfType(eventType, userId, false);
 };
 
-
 /**
  * Returns true if the given MatrixClient has permission to send a state
  * event of type `stateEventType` into this room.
@@ -345,6 +393,35 @@ RoomState.prototype._maySendEventOfType = function(eventType, userId, state) {
     return member.powerLevel >= required_level;
 };
 
+/**
+ * Returns true if the given user ID has permission to trigger notification
+ * of type `notifLevelKey`
+ * @param {string} notifLevelKey The level of notification to test (eg. 'room')
+ * @param {string} userId The user ID of the user to test permission for
+ * @return {boolean} true if the given user ID has permission to trigger a
+ *                        notification of this type.
+ */
+RoomState.prototype.mayTriggerNotifOfType = function(notifLevelKey, userId) {
+    const member = this.getMember(userId);
+    if (!member) {
+        return false;
+    }
+
+    const powerLevelsEvent = this.getStateEvents('m.room.power_levels', '');
+
+    let notifLevel = 50;
+    if (
+        powerLevelsEvent &&
+        powerLevelsEvent.getContent() &&
+        powerLevelsEvent.getContent().notifications &&
+        powerLevelsEvent.getContent().notifications[notifLevelKey]
+    ) {
+        notifLevel = powerLevelsEvent.getContent().notifications[notifLevelKey];
+    }
+
+    return member.powerLevel >= notifLevel;
+};
+
 /**
  * The RoomState class.
  */

src/models/room.js

@@ -27,6 +27,7 @@ const ContentRepo = require("../content-repo");
 const EventTimeline = require("./event-timeline");
 const EventTimelineSet = require("./event-timeline-set");
 
+import ReEmitter from '../ReEmitter';
 
 function synthesizeReceipt(userId, event, receiptType) {
     // console.log("synthesizing receipt for "+event.getId());
@@ -105,6 +106,8 @@ function Room(roomId, opts) {
     opts = opts || {};
     opts.pendingEventOrdering = opts.pendingEventOrdering || "chronological";
 
+    this.reEmitter = new ReEmitter(this);
+
     if (["chronological", "detached"].indexOf(opts.pendingEventOrdering) === -1) {
         throw new Error(
             "opts.pendingEventOrdering MUST be either 'chronological' or " +
@@ -152,7 +155,7 @@ function Room(roomId, opts) {
     // all our per-room timeline sets. the first one is the unfiltered ones;
     // the subsequent ones are the filtered ones in no particular order.
     this._timelineSets = [new EventTimelineSet(this, opts)];
-    reEmit(this, this.getUnfilteredTimelineSet(),
+    this.reEmitter.reEmit(this.getUnfilteredTimelineSet(),
            ["Room.timeline", "Room.timelineReset"]);
 
     this._fixUpLegacyTimelineFields();
@@ -204,12 +207,16 @@ Room.prototype.getLiveTimeline = function() {
  * <p>This is used when /sync returns a 'limited' timeline.
  *
  * @param {string=} backPaginationToken   token for back-paginating the new timeline
- * @param {boolean=} flush True to remove all events in all timelines. If false, only
- * the live timeline is reset.
+ * @param {string=} forwardPaginationToken token for forward-paginating the old live timeline,
+ * if absent or null, all timelines are reset, removing old ones (including the previous live
+ * timeline which would otherwise be unable to paginate forwards without this token).
+ * Removing just the old live timeline whilst preserving previous ones is not supported.
  */
-Room.prototype.resetLiveTimeline = function(backPaginationToken, flush) {
+Room.prototype.resetLiveTimeline = function(backPaginationToken, forwardPaginationToken) {
     for (let i = 0; i < this._timelineSets.length; i++) {
-        this._timelineSets[i].resetLiveTimeline(backPaginationToken, flush);
+        this._timelineSets[i].resetLiveTimeline(
+            backPaginationToken, forwardPaginationToken,
+        );
     }
 
     this._fixUpLegacyTimelineFields();
@@ -485,7 +492,7 @@ Room.prototype.getOrCreateFilteredTimelineSet = function(filter) {
     }
     const opts = Object.assign({ filter: filter }, this._opts);
     const timelineSet = new EventTimelineSet(this, opts);
-    reEmit(this, timelineSet, ["Room.timeline", "Room.timelineReset"]);
+    this.reEmitter.reEmit(timelineSet, ["Room.timeline", "Room.timelineReset"]);
     this._filteredTimelineSets[filter.filterId] = timelineSet;
     this._timelineSets.push(timelineSet);
 
@@ -751,6 +758,8 @@ ALLOWED_TRANSITIONS[EventStatus.CANCELLED] =
  * @fires module:client~MatrixClient#event:"Room.localEchoUpdated"
  */
 Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
+    console.log(`setting pendingEvent status to ${newStatus} in ${event.getRoomId()}`);
+
     // if the message was sent, we expect an event id
     if (newStatus == EventStatus.SENT && !newEventId) {
         throw new Error("updatePendingEvent called with status=SENT, " +
@@ -1196,13 +1205,13 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
     if (myMemberEvent && myMemberEvent.content.membership == "invite") {
         if (room.currentState.getMember(myMemberEvent.sender)) {
             // extract who invited us to the room
-            return "Invite from " + room.currentState.getMember(
+            return room.currentState.getMember(
                 myMemberEvent.sender,
             ).name;
         } else if (allMembers[0].events.member) {
             // use the sender field from the invite event, although this only
             // gets us the mxid
-            return "Invite from " + myMemberEvent.sender;
+            return myMemberEvent.sender;
         } else {
             return "Room Invite";
         }
@@ -1252,25 +1261,6 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
     }
 }
 
-// FIXME: copypasted from sync.js
-function reEmit(reEmitEntity, emittableEntity, eventNames) {
-    utils.forEach(eventNames, function(eventName) {
-        // setup a listener on the entity (the Room, User, etc) for this event
-        emittableEntity.on(eventName, function() {
-            // take the args from the listener and reuse them, adding the
-            // event name to the arg list so it works with .emit()
-            // Transformation Example:
-            // listener on "foo" => function(a,b) { ... }
-            // Re-emit on "thing" => thing.emit("foo", a, b)
-            const newArgs = [eventName];
-            for (let i = 0; i < arguments.length; i++) {
-                newArgs.push(arguments[i]);
-            }
-            reEmitEntity.emit(...newArgs);
-        });
-    });
-}
-
 /**
  * The Room class.
  */

src/pushprocessor.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,6 +18,8 @@ limitations under the License.
  * @module pushprocessor
  */
 
+const RULEKINDS_IN_ORDER = ['override', 'content', 'room', 'sender', 'underride'];
+
 /**
  * Construct a Push Processor.
  * @constructor
@@ -27,12 +30,11 @@ function PushProcessor(client) {
         return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
     };
 
-    const matchingRuleFromKindSet = function(ev, kindset, device) {
-        const rulekinds_in_order = ['override', 'content', 'room', 'sender', 'underride'];
+    const matchingRuleFromKindSet = (ev, kindset, device) => {
         for (let ruleKindIndex = 0;
-                ruleKindIndex < rulekinds_in_order.length;
+                ruleKindIndex < RULEKINDS_IN_ORDER.length;
                 ++ruleKindIndex) {
-            const kind = rulekinds_in_order[ruleKindIndex];
+            const kind = RULEKINDS_IN_ORDER[ruleKindIndex];
             const ruleset = kindset[kind];
 
             for (let ruleIndex = 0; ruleIndex < ruleset.length; ++ruleIndex) {
@@ -46,7 +48,7 @@ function PushProcessor(client) {
                     continue;
                 }
 
-                if (ruleMatchesEvent(rawrule, ev)) {
+                if (this.ruleMatchesEvent(rawrule, ev)) {
                     rule.kind = kind;
                     return rule;
                 }
@@ -106,27 +108,38 @@ function PushProcessor(client) {
         return rawrule;
     };
 
-    const ruleMatchesEvent = function(rule, ev) {
-        let ret = true;
-        for (let i = 0; i < rule.conditions.length; ++i) {
-            const cond = rule.conditions[i];
-            ret &= eventFulfillsCondition(cond, ev);
-        }
-        //console.log("Rule "+rule.rule_id+(ret ? " matches" : " doesn't match"));
-        return ret;
-    };
-
     const eventFulfillsCondition = function(cond, ev) {
         const condition_functions = {
             "event_match": eventFulfillsEventMatchCondition,
             "device": eventFulfillsDeviceCondition,
             "contains_display_name": eventFulfillsDisplayNameCondition,
             "room_member_count": eventFulfillsRoomMemberCountCondition,
+            "sender_notification_permission": eventFulfillsSenderNotifPermCondition,
         };
         if (condition_functions[cond.kind]) {
             return condition_functions[cond.kind](cond, ev);
         }
-        return true;
+        // unknown conditions: we previously matched all unknown conditions,
+        // but given that rules can be added to the base rules on a server,
+        // it's probably better to not match unknown conditions.
+        return false;
+    };
+
+    const eventFulfillsSenderNotifPermCondition = function(cond, ev) {
+        const notifLevelKey = cond['key'];
+        if (!notifLevelKey) {
+            return false;
+        }
+
+        const room = client.getRoom(ev.getRoomId());
+        if (!room || !room.currentState) {
+            return false;
+        }
+
+        // Note that this should not be the current state of the room but the state at
+        // the point the event is in the DAG. Unfortunately the js-sdk does not store
+        // this.
+        return room.currentState.mayTriggerNotifOfType(notifLevelKey, ev.getSender());
     };
 
     const eventFulfillsRoomMemberCountCondition = function(cond, ev) {
@@ -215,9 +228,9 @@ function PushProcessor(client) {
         // Because micromatch is about 130KB with dependencies,
         // and minimatch is not much better.
         let pat = escapeRegExp(glob);
-        pat = pat.replace(/\\\*/, '.*');
-        pat = pat.replace(/\?/, '.');
-        pat = pat.replace(/\\\[(!|)(.*)\\]/, function(match, p1, p2, offset, string) {
+        pat = pat.replace(/\\\*/g, '.*');
+        pat = pat.replace(/\?/g, '.');
+        pat = pat.replace(/\\\[(!|)(.*)\\]/g, function(match, p1, p2, offset, string) {
             const first = p1 && '^' || '';
             const second = p2.replace(/\\\-/, '-');
             return '[' + first + second + ']';
@@ -291,6 +304,17 @@ function PushProcessor(client) {
         return actionObj;
     };
 
+    this.ruleMatchesEvent = function(rule, ev) {
+        let ret = true;
+        for (let i = 0; i < rule.conditions.length; ++i) {
+            const cond = rule.conditions[i];
+            ret &= eventFulfillsCondition(cond, ev);
+        }
+        //console.log("Rule "+rule.rule_id+(ret ? " matches" : " doesn't match"));
+        return ret;
+    };
+
+
     /**
      * Get the user's push actions for the given event
      *
@@ -301,6 +325,27 @@ function PushProcessor(client) {
     this.actionsForEvent = function(ev) {
         return pushActionsForEventAndRulesets(ev, client.pushRules);
     };
+
+    /**
+     * Get one of the users push rules by its ID
+     *
+     * @param {string} ruleId The ID of the rule to search for
+     * @return {object} The push rule, or null if no such rule was found
+     */
+    this.getPushRuleById = function(ruleId) {
+        for (const scope of ['device', 'global']) {
+            if (client.pushRules[scope] === undefined) continue;
+
+            for (const kind of RULEKINDS_IN_ORDER) {
+                if (client.pushRules[scope][kind] === undefined) continue;
+
+                for (const rule of client.pushRules[scope][kind]) {
+                    if (rule.rule_id === ruleId) return rule;
+                }
+            }
+        }
+        return null;
+    };
 }
 
 /**

src/scheduler.js

@@ -20,7 +20,7 @@ limitations under the License.
  * @module scheduler
  */
 const utils = require("./utils");
-const q = require("q");
+import Promise from 'bluebird';
 
 const DEBUG = false;  // set true to enable console logging.
 
@@ -83,6 +83,8 @@ MatrixScheduler.prototype.removeEventFromQueue = function(event) {
     let removed = false;
     utils.removeElement(this._queues[name], function(element) {
         if (element.event.getId() === event.getId()) {
+            // XXX we should probably reject the promise?
+            // https://github.com/matrix-org/matrix-js-sdk/issues/496
             removed = true;
             return true;
         }
@@ -118,7 +120,7 @@ MatrixScheduler.prototype.queueEvent = function(event) {
     if (!this._queues[queueName]) {
         this._queues[queueName] = [];
     }
-    const defer = q.defer();
+    const defer = Promise.defer();
     this._queues[queueName].push({
         event: event,
         defer: defer,

src/store/indexeddb-local-backend.js

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import q from "q";
+import Promise from 'bluebird';
 import SyncAccumulator from "../sync-accumulator";
 import utils from "../utils";
 
@@ -44,7 +44,7 @@ function createDatabase(db) {
  */
 function selectQuery(store, keyRange, resultMapper) {
     const query = store.openCursor(keyRange);
-    return q.Promise((resolve, reject) => { /*eslint new-cap: 0*/
+    return new Promise((resolve, reject) => {
         const results = [];
         query.onerror = (event) => {
             reject(new Error("Query failed: " + event.target.errorCode));
@@ -63,7 +63,7 @@ function selectQuery(store, keyRange, resultMapper) {
 }
 
 function promiseifyTxn(txn) {
-    return new q.Promise((resolve, reject) => {
+    return new Promise((resolve, reject) => {
         txn.oncomplete = function(event) {
             resolve(event);
         };
@@ -74,7 +74,7 @@ function promiseifyTxn(txn) {
 }
 
 function promiseifyRequest(req) {
-    return new q.Promise((resolve, reject) => {
+    return new Promise((resolve, reject) => {
         req.onsuccess = function(event) {
             resolve(event);
         };
@@ -113,19 +113,41 @@ LocalIndexedDBStoreBackend.prototype = {
      */
     connect: function() {
         if (this.db) {
-            return q();
+            console.log(
+                `LocalIndexedDBStoreBackend.connect: already connected`,
+            );
+            return Promise.resolve();
         }
+
+        console.log(
+            `LocalIndexedDBStoreBackend.connect: connecting`,
+        );
         const req = this.indexedDB.open(this._dbName, VERSION);
         req.onupgradeneeded = (ev) => {
             const db = ev.target.result;
             const oldVersion = ev.oldVersion;
+            console.log(
+                `LocalIndexedDBStoreBackend.connect: upgrading from ${oldVersion}`,
+            );
             if (oldVersion < 1) { // The database did not previously exist.
                 createDatabase(db);
             }
             // Expand as needed.
         };
 
+        req.onblocked = () => {
+            console.log(
+                `can't yet open LocalIndexedDBStoreBackend because it is open elsewhere`,
+            );
+        };
+
+        console.log(
+            `LocalIndexedDBStoreBackend.connect: awaiting connection`,
+        );
         return promiseifyRequest(req).then((ev) => {
+            console.log(
+                `LocalIndexedDBStoreBackend.connect: connected`,
+            );
             this.db = ev.target.result;
 
             // add a poorly-named listener for when deleteDatabase is called
@@ -143,13 +165,17 @@ LocalIndexedDBStoreBackend.prototype = {
      * @return {Promise} Resolves on success
      */
     _init: function() {
-        return q.all([
+        return Promise.all([
             this._loadAccountData(),
             this._loadSyncData(),
         ]).then(([accountData, syncData]) => {
+            console.log(
+                `LocalIndexedDBStoreBackend: loaded initial data`,
+            );
             this._syncAccumulator.accumulate({
                 next_batch: syncData.nextBatch,
                 rooms: syncData.roomsData,
+                groups: syncData.groupsData,
                 account_data: {
                     events: accountData,
                 },
@@ -163,8 +189,32 @@ LocalIndexedDBStoreBackend.prototype = {
      * @return {Promise} Resolved when the database is cleared.
      */
     clearDatabase: function() {
-        console.log("Removing indexeddb instance: ", this._dbName);
-        return promiseifyRequest(this.indexedDB.deleteDatabase(this._dbName));
+        return new Promise((resolve, reject) => {
+            console.log(`Removing indexeddb instance: ${this._dbName}`);
+            const req = this.indexedDB.deleteDatabase(this._dbName);
+
+            req.onblocked = () => {
+                console.log(
+                    `can't yet delete indexeddb ${this._dbName}` +
+                    ` because it is open elsewhere`,
+                );
+            };
+
+            req.onerror = (ev) => {
+                // in firefox, with indexedDB disabled, this fails with a
+                // DOMError. We treat this as non-fatal, so that we can still
+                // use the app.
+                console.warn(
+                    `unable to delete js-sdk store indexeddb: ${ev.target.error}`,
+                );
+                resolve();
+            };
+
+            req.onsuccess = () => {
+                console.log(`Removed indexeddb instance: ${this._dbName}`);
+                resolve();
+            };
+        });
     },
 
     /**
@@ -180,18 +230,18 @@ LocalIndexedDBStoreBackend.prototype = {
         if (copy === undefined) copy = true;
 
         const data = this._syncAccumulator.getJSON();
-        if (!data.nextBatch) return q(null);
+        if (!data.nextBatch) return Promise.resolve(null);
         if (copy) {
             // We must deep copy the stored data so that the /sync processing code doesn't
             // corrupt the internal state of the sync accumulator (it adds non-clonable keys)
-            return q(utils.deepCopy(data));
+            return Promise.resolve(utils.deepCopy(data));
         } else {
-            return q(data);
+            return Promise.resolve(data);
         }
     },
 
     setSyncData: function(syncData) {
-        return q().then(() => {
+        return Promise.resolve().then(() => {
             this._syncAccumulator.accumulate(syncData);
         });
     },
@@ -199,10 +249,12 @@ LocalIndexedDBStoreBackend.prototype = {
     syncToDatabase: function(userTuples) {
         const syncData = this._syncAccumulator.getJSON();
 
-        return q.all([
+        return Promise.all([
             this._persistUserPresenceEvents(userTuples),
             this._persistAccountData(syncData.accountData),
-            this._persistSyncData(syncData.nextBatch, syncData.roomsData),
+            this._persistSyncData(
+                syncData.nextBatch, syncData.roomsData, syncData.groupsData,
+            ),
         ]);
     },
 
@@ -210,17 +262,19 @@ LocalIndexedDBStoreBackend.prototype = {
      * Persist rooms /sync data along with the next batch token.
      * @param {string} nextBatch The next_batch /sync value.
      * @param {Object} roomsData The 'rooms' /sync data from a SyncAccumulator
+     * @param {Object} groupsData The 'groups' /sync data from a SyncAccumulator
      * @return {Promise} Resolves if the data was persisted.
      */
-    _persistSyncData: function(nextBatch, roomsData) {
+    _persistSyncData: function(nextBatch, roomsData, groupsData) {
         console.log("Persisting sync data up to ", nextBatch);
-        return q.try(() => {
+        return Promise.try(() => {
             const txn = this.db.transaction(["sync"], "readwrite");
             const store = txn.objectStore("sync");
             store.put({
                 clobber: "-", // constant key so will always clobber
                 nextBatch: nextBatch,
                 roomsData: roomsData,
+                groupsData: groupsData,
             }); // put == UPSERT
             return promiseifyTxn(txn);
         });
@@ -233,7 +287,7 @@ LocalIndexedDBStoreBackend.prototype = {
      * @return {Promise} Resolves if the events were persisted.
      */
     _persistAccountData: function(accountData) {
-        return q.try(() => {
+        return Promise.try(() => {
             const txn = this.db.transaction(["accountData"], "readwrite");
             const store = txn.objectStore("accountData");
             for (let i = 0; i < accountData.length; i++) {
@@ -252,7 +306,7 @@ LocalIndexedDBStoreBackend.prototype = {
      * @return {Promise} Resolves if the users were persisted.
      */
     _persistUserPresenceEvents: function(tuples) {
-        return q.try(() => {
+        return Promise.try(() => {
             const txn = this.db.transaction(["users"], "readwrite");
             const store = txn.objectStore("users");
             for (const tuple of tuples) {
@@ -272,7 +326,7 @@ LocalIndexedDBStoreBackend.prototype = {
      * @return {Promise<Object[]>} A list of presence events in their raw form.
      */
     getUserPresenceEvents: function() {
-        return q.try(() => {
+        return Promise.try(() => {
             const txn = this.db.transaction(["users"], "readonly");
             const store = txn.objectStore("users");
             return selectQuery(store, undefined, (cursor) => {
@@ -286,7 +340,13 @@ LocalIndexedDBStoreBackend.prototype = {
      * @return {Promise<Object[]>} A list of raw global account events.
      */
     _loadAccountData: function() {
-        return q.try(() => {
+        console.log(
+            `LocalIndexedDBStoreBackend: loading account data`,
+        );
+        return Promise.try(() => {
+            console.log(
+                `LocalIndexedDBStoreBackend: loaded account data`,
+            );
             const txn = this.db.transaction(["accountData"], "readonly");
             const store = txn.objectStore("accountData");
             return selectQuery(store, undefined, (cursor) => {
@@ -300,7 +360,13 @@ LocalIndexedDBStoreBackend.prototype = {
      * @return {Promise<Object>} An object with "roomsData" and "nextBatch" keys.
      */
     _loadSyncData: function() {
-        return q.try(() => {
+        console.log(
+            `LocalIndexedDBStoreBackend: loaded sync data`,
+        );
+        return Promise.try(() => {
+            console.log(
+                `LocalIndexedDBStoreBackend: loaded sync data`,
+            );
             const txn = this.db.transaction(["sync"], "readonly");
             const store = txn.objectStore("sync");
             return selectQuery(store, undefined, (cursor) => {

src/store/indexeddb-remote-backend.js

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import q from "q";
+import Promise from 'bluebird';
 
 /**
  * An IndexedDB store backend where the actual backend sits in a web
@@ -40,6 +40,11 @@ const RemoteIndexedDBStoreBackend = function RemoteIndexedDBStoreBackend(
     };
 
     this._worker.onmessage = this._onWorkerMessage.bind(this);
+
+    // tell the worker the db name.
+    this._startPromise = this._doCmd('_setupWorker', [this._dbName]).then(() => {
+        console.log("IndexedDB worker is ready");
+    });
 };
 
 
@@ -50,10 +55,7 @@ RemoteIndexedDBStoreBackend.prototype = {
      * @return {Promise} Resolves if successfully connected.
      */
     connect: function() {
-        return this._doCmd('_setupWorker', [this._dbName]).then(() => {
-            console.log("IndexedDB worker is ready");
-            return this._doCmd('connect');
-        });
+        return this._startPromise.then(() => this._doCmd('connect'));
     },
 
     /**
@@ -62,7 +64,7 @@ RemoteIndexedDBStoreBackend.prototype = {
      * @return {Promise} Resolved when the database is cleared.
      */
     clearDatabase: function() {
-        return this._doCmd('clearDatabase');
+        return this._startPromise.then(() => this._doCmd('clearDatabase'));
     },
 
     /**
@@ -94,9 +96,9 @@ RemoteIndexedDBStoreBackend.prototype = {
     _doCmd: function(cmd, args) {
         // wrap in a q so if the postMessage throws,
         // the promise automatically gets rejected
-        return q().then(() => {
+        return Promise.resolve().then(() => {
             const seq = this._nextSeq++;
-            const def = q.defer();
+            const def = Promise.defer();
 
             this._inFlight[seq] = def;
 

src/store/indexeddb-store-worker.js

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import q from "q";
+import Promise from 'bluebird';
 import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";
 
 /**
@@ -61,7 +61,7 @@ class IndexedDBStoreWorker {
                     // because it's a web worker and there is no window).
                     indexedDB, msg.args[0],
                 );
-                prom = q();
+                prom = Promise.resolve();
                 break;
             case 'connect':
                 prom = this.backend.connect();

src/store/indexeddb.js

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import q from "q";
+import Promise from 'bluebird';
 import {MatrixInMemoryStore} from "./memory";
 import utils from "../utils";
 import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";
@@ -115,12 +115,16 @@ utils.inherits(IndexedDBStore, MatrixInMemoryStore);
   */
 IndexedDBStore.prototype.startup = function() {
     if (this.startedUp) {
-        return q();
+        console.log(`IndexedDBStore.startup: already started`);
+        return Promise.resolve();
     }
 
+    console.log(`IndexedDBStore.startup: connecting to backend`);
     return this.backend.connect().then(() => {
+        console.log(`IndexedDBStore.startup: loading presence events`);
         return this.backend.getUserPresenceEvents();
     }).then((userPresenceEvents) => {
+        console.log(`IndexedDBStore.startup: processing presence events`);
         userPresenceEvents.forEach(([userId, rawEvent]) => {
             const u = new User(userId);
             if (rawEvent) {
@@ -150,7 +154,7 @@ IndexedDBStore.prototype.deleteAllData = function() {
     return this.backend.clearDatabase().then(() => {
         console.log("Deleted indexeddb data.");
     }, (err) => {
-        console.error("Failed to delete indexeddb data: ", err);
+        console.error(`Failed to delete indexeddb data: ${err}`);
         throw err;
     });
 };
@@ -164,7 +168,7 @@ IndexedDBStore.prototype.save = function() {
     if (now - this._syncTs > WRITE_DELAY_MS) {
         return this._reallySave();
     }
-    return q();
+    return Promise.resolve();
 };
 
 IndexedDBStore.prototype._reallySave = function() {

src/store/memory.js

@@ -19,9 +19,9 @@ limitations under the License.
  * This is an internal module. See {@link MatrixInMemoryStore} for the public class.
  * @module store/memory
  */
- const utils = require("../utils");
- const User = require("../models/user");
- const q = require("q");
+const utils = require("../utils");
+const User = require("../models/user");
+import Promise from 'bluebird';
 
 /**
  * Construct a new in-memory data store for the Matrix Client.
@@ -35,6 +35,9 @@ module.exports.MatrixInMemoryStore = function MatrixInMemoryStore(opts) {
     this.rooms = {
         // roomId: Room
     };
+    this.groups = {
+        // groupId: Group
+    };
     this.users = {
         // userId: User
     };
@@ -69,6 +72,31 @@ module.exports.MatrixInMemoryStore.prototype = {
         this.syncToken = token;
     },
 
+    /**
+     * Store the given room.
+     * @param {Group} group The group to be stored
+     */
+    storeGroup: function(group) {
+        this.groups[group.groupId] = group;
+    },
+
+    /**
+     * Retrieve a group by its group ID.
+     * @param {string} groupId The group ID.
+     * @return {Group} The group or null.
+     */
+    getGroup: function(groupId) {
+        return this.groups[groupId] || null;
+    },
+
+    /**
+     * Retrieve all known groups.
+     * @return {Group[]} A list of groups, which may be empty.
+     */
+    getGroups: function() {
+        return utils.values(this.groups);
+    },
+
     /**
      * Store the given room.
      * @param {Room} room The room to be stored. All properties must be stored.
@@ -284,7 +312,7 @@ module.exports.MatrixInMemoryStore.prototype = {
      * @return {Promise} An immediately resolved promise.
      */
     setSyncData: function(syncData) {
-        return q();
+        return Promise.resolve();
     },
 
     /**
@@ -297,7 +325,7 @@ module.exports.MatrixInMemoryStore.prototype = {
      * @return {Promise} An immediately resolved promise.
      */
     startup: function() {
-        return q();
+        return Promise.resolve();
     },
 
     /**
@@ -306,7 +334,7 @@ module.exports.MatrixInMemoryStore.prototype = {
      * is no saved sync data.
      */
     getSavedSync: function() {
-        return q(null);
+        return Promise.resolve(null);
     },
 
     /**
@@ -329,6 +357,6 @@ module.exports.MatrixInMemoryStore.prototype = {
         this.accountData = {
             // type : content
         };
-        return q();
+        return Promise.resolve();
     },
 };

src/store/session/webstorage.js

@@ -65,22 +65,6 @@ WebStorageSessionStore.prototype = {
         return this.store.getItem(KEY_END_TO_END_ACCOUNT);
     },
 
-    /**
-     * Store a flag indicating that we have announced the new device.
-     */
-    setDeviceAnnounced: function() {
-        this.store.setItem(KEY_END_TO_END_ANNOUNCED, "true");
-    },
-
-    /**
-     * Check if the "device announced" flag is set
-     *
-     * @return {boolean} true if the "device announced" flag has been set.
-     */
-    getDeviceAnnounced: function() {
-        return this.store.getItem(KEY_END_TO_END_ANNOUNCED) == "true";
-    },
-
     /**
      * Stores the known devices for a user.
      * @param {string} userId The user's ID.
@@ -208,7 +192,6 @@ WebStorageSessionStore.prototype = {
 };
 
 const KEY_END_TO_END_ACCOUNT = E2E_PREFIX + "account";
-const KEY_END_TO_END_ANNOUNCED = E2E_PREFIX + "announced";
 const KEY_END_TO_END_DEVICE_SYNC_TOKEN = E2E_PREFIX + "device_sync_token";
 const KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS = E2E_PREFIX + "device_tracking";
 

src/store/stub.js

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 "use strict";
-import q from "q";
+import Promise from 'bluebird';
 /**
  * This is an internal module.
  * @module store/stub
@@ -47,6 +47,30 @@ StubStore.prototype = {
         this.fromToken = token;
     },
 
+    /**
+     * No-op.
+     * @param {Group} group
+     */
+    storeGroup: function(group) {
+    },
+
+    /**
+     * No-op.
+     * @param {string} groupId
+     * @return {null}
+     */
+    getGroup: function(groupId) {
+        return null;
+    },
+
+    /**
+     * No-op.
+     * @return {Array} An empty array.
+     */
+    getGroups: function() {
+        return [];
+    },
+
     /**
      * No-op.
      * @param {Room} room
@@ -189,7 +213,7 @@ StubStore.prototype = {
      * @return {Promise} An immediately resolved promise.
      */
     setSyncData: function(syncData) {
-        return q();
+        return Promise.resolve();
     },
 
     /**
@@ -202,7 +226,7 @@ StubStore.prototype = {
      * @return {Promise} An immediately resolved promise.
      */
     startup: function() {
-        return q();
+        return Promise.resolve();
     },
 
     /**
@@ -211,7 +235,7 @@ StubStore.prototype = {
      * is no saved sync data.
      */
     getSavedSync: function() {
-        return q(null);
+        return Promise.resolve(null);
     },
 
     /**
@@ -220,7 +244,7 @@ StubStore.prototype = {
      * @return {Promise} An immediately resolved promise.
      */
     deleteAllData: function() {
-        return q();
+        return Promise.resolve();
     },
 };
 

src/sync-accumulator.js

@@ -72,10 +72,18 @@ class SyncAccumulator {
         // coherent /sync response and know at what point they should be
         // streaming from without losing events.
         this.nextBatch = null;
+
+        // { ('invite'|'join'|'leave'): $groupId: { ... sync 'group' data } }
+        this.groups = {
+            invite: {},
+            join: {},
+            leave: {},
+        };
     }
 
     accumulate(syncResponse) {
         this._accumulateRooms(syncResponse);
+        this._accumulateGroups(syncResponse);
         this._accumulateAccountData(syncResponse);
         this.nextBatch = syncResponse.next_batch;
     }
@@ -336,6 +344,44 @@ class SyncAccumulator {
         }
     }
 
+    /**
+     * Accumulate incremental /sync group data.
+     * @param {Object} syncResponse the complete /sync JSON
+     */
+    _accumulateGroups(syncResponse) {
+        if (!syncResponse.groups) {
+            return;
+        }
+        if (syncResponse.groups.invite) {
+            Object.keys(syncResponse.groups.invite).forEach((groupId) => {
+                this._accumulateGroup(
+                    groupId, "invite", syncResponse.groups.invite[groupId],
+                );
+            });
+        }
+        if (syncResponse.groups.join) {
+            Object.keys(syncResponse.groups.join).forEach((groupId) => {
+                this._accumulateGroup(
+                    groupId, "join", syncResponse.groups.join[groupId],
+                );
+            });
+        }
+        if (syncResponse.groups.leave) {
+            Object.keys(syncResponse.groups.leave).forEach((groupId) => {
+                this._accumulateGroup(
+                    groupId, "leave", syncResponse.groups.leave[groupId],
+                );
+            });
+        }
+    }
+
+    _accumulateGroup(groupId, category, data) {
+        for (const cat of ['invite', 'join', 'leave']) {
+            delete this.groups[cat][groupId];
+        }
+        this.groups[category][groupId] = data;
+    }
+
     /**
      * Return everything under the 'rooms' key from a /sync response which
      * represents all room data that should be stored. This should be paired
@@ -470,6 +516,7 @@ class SyncAccumulator {
         return {
             nextBatch: this.nextBatch,
             roomsData: data,
+            groupsData: this.groups,
             accountData: accData,
         };
     }

src/sync.js

@@ -24,9 +24,10 @@ limitations under the License.
  * an alternative syncing API, we may want to have a proper syncing interface
  * for HTTP and WS at some point.
  */
-const q = require("q");
+import Promise from 'bluebird';
 const User = require("./models/user");
 const Room = require("./models/room");
+const Group = require('./models/group');
 const utils = require("./utils");
 const Filter = require("./filter");
 const EventTimeline = require("./models/event-timeline");
@@ -50,11 +51,11 @@ function getFilterName(userId, suffix) {
     return "FILTER_SYNC_" + userId + (suffix ? "_" + suffix : "");
 }
 
-function debuglog() {
+function debuglog(...params) {
     if (!DEBUG) {
         return;
     }
-    console.log(...arguments);
+    console.log(...params);
 }
 
 
@@ -97,7 +98,7 @@ function SyncApi(client, opts) {
     this._failedSyncCount = 0; // Number of consecutive failed /sync requests
 
     if (client.getNotifTimelineSet()) {
-        reEmit(client, client.getNotifTimelineSet(),
+        client.reEmitter.reEmit(client.getNotifTimelineSet(),
                ["Room.timeline", "Room.timelineReset"]);
     }
 }
@@ -112,7 +113,7 @@ SyncApi.prototype.createRoom = function(roomId) {
         pendingEventOrdering: this.opts.pendingEventOrdering,
         timelineSupport: client.timelineSupport,
     });
-    reEmit(client, room, ["Room.name", "Room.timeline", "Room.redaction",
+    client.reEmitter.reEmit(room, ["Room.name", "Room.timeline", "Room.redaction",
                           "Room.receipt", "Room.tags",
                           "Room.timelineReset",
                           "Room.localEchoUpdated",
@@ -122,6 +123,18 @@ SyncApi.prototype.createRoom = function(roomId) {
     return room;
 };
 
+/**
+ * @param {string} groupId
+ * @return {Group}
+ */
+SyncApi.prototype.createGroup = function(groupId) {
+    const client = this.client;
+    const group = new Group(groupId);
+    client.reEmitter.reEmit(group, ["Group.profile", "Group.myMembership"]);
+    client.store.storeGroup(group);
+    return group;
+};
+
 /**
  * @param {Room} room
  * @private
@@ -131,13 +144,13 @@ SyncApi.prototype._registerStateListeners = function(room) {
     // we need to also re-emit room state and room member events, so hook it up
     // to the client now. We need to add a listener for RoomState.members in
     // order to hook them correctly. (TODO: find a better way?)
-    reEmit(client, room.currentState, [
+    client.reEmitter.reEmit(room.currentState, [
         "RoomState.events", "RoomState.members", "RoomState.newMember",
     ]);
     room.currentState.on("RoomState.newMember", function(event, state, member) {
         member.user = client.getUser(member.userId);
-        reEmit(
-            client, member,
+        client.reEmitter.reEmit(
+            member,
             [
                 "RoomMember.name", "RoomMember.typing", "RoomMember.powerLevel",
                 "RoomMember.membership",
@@ -218,6 +231,8 @@ SyncApi.prototype.syncLeftRooms = function() {
             room.recalculate(client.credentials.userId);
             client.store.storeRoom(room);
             client.emit("Room", room);
+
+            self._processEventsForNotifs(room, timelineEvents);
         });
         return rooms;
     });
@@ -296,7 +311,7 @@ SyncApi.prototype.peek = function(roomId) {
         client.store.storeRoom(peekRoom);
         client.emit("Room", peekRoom);
 
-        self._peekPoll(roomId);
+        self._peekPoll(peekRoom);
         return peekRoom;
     });
 };
@@ -311,22 +326,26 @@ SyncApi.prototype.stopPeeking = function() {
 
 /**
  * Do a peek room poll.
- * @param {string} roomId
+ * @param {Room} peekRoom
  * @param {string} token from= token
  */
-SyncApi.prototype._peekPoll = function(roomId, token) {
-    if (this._peekRoomId !== roomId) {
-        debuglog("Stopped peeking in room %s", roomId);
+SyncApi.prototype._peekPoll = function(peekRoom, token) {
+    if (this._peekRoomId !== peekRoom.roomId) {
+        debuglog("Stopped peeking in room %s", peekRoom.roomId);
         return;
     }
 
     const self = this;
     // FIXME: gut wrenching; hard-coded timeout values
     this.client._http.authedRequest(undefined, "GET", "/events", {
-        room_id: roomId,
+        room_id: peekRoom.roomId,
         timeout: 30 * 1000,
         from: token,
     }, undefined, 50 * 1000).done(function(res) {
+        if (self._peekRoomId !== peekRoom.roomId) {
+            debuglog("Stopped peeking in room %s", peekRoom.roomId);
+            return;
+        }
         // We have a problem that we get presence both from /events and /sync
         // however, /sync only returns presence for users in rooms
         // you're actually joined to.
@@ -352,15 +371,15 @@ SyncApi.prototype._peekPoll = function(roomId, token) {
 
         // strip out events which aren't for the given room_id (e.g presence)
         const events = res.chunk.filter(function(e) {
-            return e.room_id === roomId;
+            return e.room_id === peekRoom.roomId;
         }).map(self.client.getEventMapper());
-        const room = self.client.getRoom(roomId);
-        room.addLiveEvents(events);
-        self._peekPoll(roomId, res.end);
+
+        peekRoom.addLiveEvents(events);
+        self._peekPoll(peekRoom, res.end);
     }, function(err) {
-        console.error("[%s] Peek poll failed: %s", roomId, err);
+        console.error("[%s] Peek poll failed: %s", peekRoom.roomId, err);
         setTimeout(function() {
-            self._peekPoll(roomId, token);
+            self._peekPoll(peekRoom, token);
         }, 30 * 1000);
     });
 };
@@ -480,15 +499,14 @@ SyncApi.prototype.retryImmediately = function() {
  * @param {string} syncOptions.filterId
  * @param {boolean} syncOptions.hasSyncedBefore
  */
-SyncApi.prototype._sync = function(syncOptions) {
+SyncApi.prototype._sync = async function(syncOptions) {
     const client = this.client;
-    const self = this;
 
     if (!this._running) {
         debuglog("Sync no longer running: exiting.");
-        if (self._connectionReturnedDefer) {
-            self._connectionReturnedDefer.reject();
-            self._connectionReturnedDefer = null;
+        if (this._connectionReturnedDefer) {
+            this._connectionReturnedDefer.reject();
+            this._connectionReturnedDefer = null;
         }
         this._updateSyncState("STOPPED");
         return;
@@ -544,126 +562,134 @@ SyncApi.prototype._sync = function(syncOptions) {
         qps.timeout = 0;
     }
 
-    let isCachedResponse = false;
-
-    let syncPromise;
+    let savedSync;
     if (!syncOptions.hasSyncedBefore) {
         // Don't do an HTTP hit to /sync. Instead, load up the persisted /sync data,
         // if there is data there.
-        syncPromise = client.store.getSavedSync();
-    } else {
-        syncPromise = q(null);
+        savedSync = await client.store.getSavedSync();
     }
 
-    syncPromise.then((savedSync) => {
-        if (savedSync) {
-            debuglog("sync(): not doing HTTP hit, instead returning stored /sync data");
-            isCachedResponse = true;
-            return {
-                next_batch: savedSync.nextBatch,
-                rooms: savedSync.roomsData,
-                account_data: {
-                    events: savedSync.accountData,
-                },
-            };
-        } else {
+    let isCachedResponse = false;
+    let data;
+
+    if (savedSync) {
+        debuglog("sync(): not doing HTTP hit, instead returning stored /sync data");
+        isCachedResponse = true;
+        data = {
+            next_batch: savedSync.nextBatch,
+            rooms: savedSync.roomsData,
+            groups: savedSync.groupsData,
+            account_data: {
+                events: savedSync.accountData,
+            },
+        };
+    } else {
+        try {
             //debuglog('Starting sync since=' + syncToken);
             this._currentSyncRequest = client._http.authedRequest(
                 undefined, "GET", "/sync", qps, undefined, clientSideTimeoutMs,
             );
-            return this._currentSyncRequest;
-        }
-    }).then(function(data) {
-        //debuglog('Completed sync, next_batch=' + data.next_batch);
-
-        // set the sync token NOW *before* processing the events. We do this so
-        // if something barfs on an event we can skip it rather than constantly
-        // polling with the same token.
-        client.store.setSyncToken(data.next_batch);
-
-        // Reset after a successful sync
-        self._failedSyncCount = 0;
-
-        // We need to wait until the sync data has been sent to the backend
-        // because it appears that the sync data gets modified somewhere in
-        // processing it in such a way as to make it no longer cloneable.
-        // XXX: Find out what is modifying it!
-        if (!isCachedResponse) {
-            // Don't give the store back its own cached data
-            return client.store.setSyncData(data).then(() => {
-                return data;
-            });
-        } else {
-            return q(data);
-        }
-    }).done((data) => {
-        try {
-            self._processSyncResponse(syncToken, data);
+            data = await this._currentSyncRequest;
         } catch (e) {
-            // log the exception with stack if we have it, else fall back
-            // to the plain description
-            console.error("Caught /sync error", e.stack || e);
+            this._onSyncError(e, syncOptions);
+            return;
         }
+    }
 
-        // emit synced events
-        const syncEventData = {
-            oldSyncToken: syncToken,
-            nextSyncToken: data.next_batch,
-            catchingUp: self._catchingUp,
-        };
+    //debuglog('Completed sync, next_batch=' + data.next_batch);
 
-        if (!syncOptions.hasSyncedBefore) {
-            self._updateSyncState("PREPARED", syncEventData);
-            syncOptions.hasSyncedBefore = true;
+    // set the sync token NOW *before* processing the events. We do this so
+    // if something barfs on an event we can skip it rather than constantly
+    // polling with the same token.
+    client.store.setSyncToken(data.next_batch);
+
+    // Reset after a successful sync
+    this._failedSyncCount = 0;
+
+    // We need to wait until the sync data has been sent to the backend
+    // because it appears that the sync data gets modified somewhere in
+    // processing it in such a way as to make it no longer cloneable.
+    // XXX: Find out what is modifying it!
+    if (!isCachedResponse) {
+        // Don't give the store back its own cached data
+        await client.store.setSyncData(data);
+    }
+
+    try {
+        await this._processSyncResponse(syncToken, data, isCachedResponse);
+    } catch(e) {
+        // log the exception with stack if we have it, else fall back
+        // to the plain description
+        console.error("Caught /sync error", e.stack || e);
+    }
+
+    // emit synced events
+    const syncEventData = {
+        oldSyncToken: syncToken,
+        nextSyncToken: data.next_batch,
+        catchingUp: this._catchingUp,
+    };
+
+    if (!syncOptions.hasSyncedBefore) {
+        this._updateSyncState("PREPARED", syncEventData);
+        syncOptions.hasSyncedBefore = true;
+    }
+
+    if (!isCachedResponse) {
+        // tell the crypto module to do its processing. It may block (to do a
+        // /keys/changes request).
+        if (this.opts.crypto) {
+            await this.opts.crypto.onSyncCompleted(syncEventData);
         }
 
         // keep emitting SYNCING -> SYNCING for clients who want to do bulk updates
-        if (!isCachedResponse) {
-            self._updateSyncState("SYNCING", syncEventData);
+        this._updateSyncState("SYNCING", syncEventData);
 
-            // tell databases that everything is now in a consistent state and can be
-            // saved (no point doing so if we only have the data we just got out of the
-            // store).
-            client.store.save();
+        // tell databases that everything is now in a consistent state and can be
+        // saved (no point doing so if we only have the data we just got out of the
+        // store).
+        client.store.save();
+    }
+
+    // Begin next sync
+    this._sync(syncOptions);
+};
+
+SyncApi.prototype._onSyncError = function(err, syncOptions) {
+    if (!this._running) {
+        debuglog("Sync no longer running: exiting");
+        if (this._connectionReturnedDefer) {
+            this._connectionReturnedDefer.reject();
+            this._connectionReturnedDefer = null;
         }
+        this._updateSyncState("STOPPED");
+        return;
+    }
 
+    console.error("/sync error %s", err);
+    console.error(err);
 
-        // Begin next sync
-        self._sync(syncOptions);
-    }, function(err) {
-        if (!self._running) {
-            debuglog("Sync no longer running: exiting");
-            if (self._connectionReturnedDefer) {
-                self._connectionReturnedDefer.reject();
-                self._connectionReturnedDefer = null;
-            }
-            self._updateSyncState("STOPPED");
-            return;
-        }
-        console.error("/sync error %s", err);
-        console.error(err);
+    this._failedSyncCount++;
+    console.log('Number of consecutive failed sync requests:', this._failedSyncCount);
 
-        self._failedSyncCount++;
-        console.log('Number of consecutive failed sync requests:', self._failedSyncCount);
-
-        debuglog("Starting keep-alive");
-        // Note that we do *not* mark the sync connection as
-        // lost yet: we only do this if a keepalive poke
-        // fails, since long lived HTTP connections will
-        // go away sometimes and we shouldn't treat this as
-        // erroneous. We set the state to 'reconnecting'
-        // instead, so that clients can onserve this state
-        // if they wish.
-        self._startKeepAlives().done(function() {
-            self._sync(syncOptions);
-        });
-        self._currentSyncRequest = null;
-        // Transition from RECONNECTING to ERROR after a given number of failed syncs
-        self._updateSyncState(
-            self._failedSyncCount >= FAILED_SYNC_ERROR_THRESHOLD ?
-                "ERROR" : "RECONNECTING",
-        );
+    debuglog("Starting keep-alive");
+    // Note that we do *not* mark the sync connection as
+    // lost yet: we only do this if a keepalive poke
+    // fails, since long lived HTTP connections will
+    // go away sometimes and we shouldn't treat this as
+    // erroneous. We set the state to 'reconnecting'
+    // instead, so that clients can onserve this state
+    // if they wish.
+    this._startKeepAlives().then(() => {
+        this._sync(syncOptions);
     });
+
+    this._currentSyncRequest = null;
+    // Transition from RECONNECTING to ERROR after a given number of failed syncs
+    this._updateSyncState(
+        this._failedSyncCount >= FAILED_SYNC_ERROR_THRESHOLD ?
+            "ERROR" : "RECONNECTING",
+    );
 };
 
 /**
@@ -673,8 +699,11 @@ SyncApi.prototype._sync = function(syncOptions) {
  * @param {string} syncToken the old next_batch token sent to this
  *    sync request.
  * @param {Object} data The response from /sync
+ * @param {bool} isCachedResponse True if this response is from our local cache
  */
-SyncApi.prototype._processSyncResponse = function(syncToken, data) {
+SyncApi.prototype._processSyncResponse = async function(
+    syncToken, data, isCachedResponse,
+) {
     const client = this.client;
     const self = this;
 
@@ -685,6 +714,7 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
     //    account_data: { events: [] },
     //    device_lists: { changed: ["@user:server", ... ]},
     //    to_device: { events: [] },
+    //    device_one_time_keys_count: { signed_curve25519: 42 },
     //    rooms: {
     //      invite: {
     //        $roomid: {
@@ -710,6 +740,19 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
     //        }
     //      }
     //    },
+    //    groups: {
+    //        invite: {
+    //            $groupId: {
+    //                inviter: $inviter,
+    //                profile: {
+    //                    avatar_url: $avatarUrl,
+    //                    name: $groupName,
+    //                },
+    //            },
+    //        },
+    //        join: {},
+    //        leave: {},
+    //    },
     // }
 
     // TODO-arch:
@@ -739,7 +782,13 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
         client.store.storeAccountDataEvents(events);
         events.forEach(
             function(accountDataEvent) {
-                if (accountDataEvent.getType() == 'm.push_rules') {
+                // XXX: This is awful: ignore push rules from our cached sync. We fetch the
+                // push rules before syncing so we actually have up-to-date ones. We do want
+                // to honour new push rules that come down the sync but synapse doesn't
+                // put new push rules in the sync stream when the base rules change, so
+                // if the base rules change, we do need to refresh. We therefore ignore
+                // the push rules in our cached sync response.
+                if (accountDataEvent.getType() == 'm.push_rules' && !isCachedResponse) {
                     client.pushRules = accountDataEvent.getContent();
                 }
                 client.emit("accountData", accountDataEvent);
@@ -777,6 +826,20 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
         this._catchingUp = false;
     }
 
+    if (data.groups) {
+        if (data.groups.invite) {
+            this._processGroupSyncEntry(data.groups.invite, 'invite');
+        }
+
+        if (data.groups.join) {
+            this._processGroupSyncEntry(data.groups.join, 'join');
+        }
+
+        if (data.groups.leave) {
+            this._processGroupSyncEntry(data.groups.leave, 'leave');
+        }
+    }
+
     // the returned json structure is a bit crap, so make it into a
     // nicer form (array) after applying sanity to make sure we don't fail
     // on missing keys (on the off chance)
@@ -815,7 +878,7 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
     });
 
     // Handle joins
-    joinRooms.forEach(function(joinObj) {
+    await Promise.mapSeries(joinRooms, async function(joinObj) {
         const room = joinObj.room;
         const stateEvents = self._mapSyncEventsFormat(joinObj.state, room);
         const timelineEvents = self._mapSyncEventsFormat(joinObj.timeline, room);
@@ -878,14 +941,10 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
             }
 
             if (limited) {
-                // save the old 'next_batch' token as the
-                // forward-pagination token for the previously-active
-                // timeline.
-                room.currentState.paginationToken = syncToken;
                 self._deregisterStateListeners(room);
                 room.resetLiveTimeline(
                     joinObj.timeline.prev_batch,
-                    self.opts.canResetEntireTimeline(room.roomId),
+                    self.opts.canResetEntireTimeline(room.roomId) ? null : syncToken,
                 );
 
                 // We have to assume any gap in any timeline is
@@ -912,12 +971,18 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
             client.store.storeRoom(room);
             client.emit("Room", room);
         }
-        stateEvents.forEach(function(e) {
+
+        self._processEventsForNotifs(room, timelineEvents);
+
+        async function processRoomEvent(e) {
             client.emit("event", e);
-        });
-        timelineEvents.forEach(function(e) {
-            client.emit("event", e);
-        });
+            if (e.isState() && e.getType() == "m.room.encryption" && self.opts.crypto) {
+                await self.opts.crypto.onCryptoEvent(e);
+            }
+        }
+
+        await Promise.mapSeries(stateEvents, processRoomEvent);
+        await Promise.mapSeries(timelineEvents, processRoomEvent);
         ephemeralEvents.forEach(function(e) {
             client.emit("event", e);
         });
@@ -945,6 +1010,8 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
             client.emit("Room", room);
         }
 
+        self._processEventsForNotifs(room, timelineEvents);
+
         stateEvents.forEach(function(e) {
             client.emit("event", e);
         });
@@ -971,10 +1038,20 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
     }
 
     // Handle device list updates
-    if (this.opts.crypto && data.device_lists && data.device_lists.changed) {
-        data.device_lists.changed.forEach((u) => {
-            this.opts.crypto.userDeviceListChanged(u);
-        });
+    if (data.device_lists) {
+        if (this.opts.crypto) {
+            await this.opts.crypto.handleDeviceListChanges(data.device_lists);
+        } else {
+            // FIXME if we *don't* have a crypto module, we still need to
+            // invalidate the device lists. But that would require a
+            // substantial bit of rework :/.
+        }
+    }
+
+    // Handle one_time_keys_count
+    if (this.opts.crypto && data.device_one_time_keys_count) {
+        const currentCount = data.device_one_time_keys_count.signed_curve25519 || 0;
+        this.opts.crypto.updateOneTimeKeyCount(currentCount);
     }
 };
 
@@ -1003,7 +1080,7 @@ SyncApi.prototype._startKeepAlives = function(delay) {
         self._pokeKeepAlive();
     }
     if (!this._connectionReturnedDefer) {
-        this._connectionReturnedDefer = q.defer();
+        this._connectionReturnedDefer = Promise.defer();
     }
     return this._connectionReturnedDefer.promise;
 };
@@ -1060,6 +1137,35 @@ SyncApi.prototype._pokeKeepAlive = function() {
     });
 };
 
+/**
+ * @param {Object} groupsSection Groups section object, eg. response.groups.invite
+ * @param {string} sectionName Which section this is ('invite', 'join' or 'leave')
+ */
+SyncApi.prototype._processGroupSyncEntry = function(groupsSection, sectionName) {
+    // Processes entries from 'groups' section of the sync stream
+    for (const groupId of Object.keys(groupsSection)) {
+        const groupInfo = groupsSection[groupId];
+        let group = this.client.store.getGroup(groupId);
+        const isBrandNew = group === null;
+        if (group === null) {
+            group = this.createGroup(groupId);
+        }
+        if (groupInfo.profile) {
+            group.setProfile(
+                groupInfo.profile.name, groupInfo.profile.avatar_url,
+            );
+        }
+        if (groupInfo.inviter) {
+            group.setInviter({userId: groupInfo.inviter});
+        }
+        group.setMyMembership(sectionName);
+        if (isBrandNew) {
+            // Now we've filled in all the fields, emit the Group event
+            this.client.emit("Group", group);
+        }
+    }
+};
+
 /**
  * @param {Object} obj
  * @return {Object[]}
@@ -1121,7 +1227,7 @@ SyncApi.prototype._resolveInvites = function(room) {
         const user = client.getUser(member.userId);
         let promise;
         if (user) {
-            promise = q({
+            promise = Promise.resolve({
                 avatar_url: user.avatarUrl,
                 displayname: user.displayName,
             });
@@ -1186,20 +1292,33 @@ SyncApi.prototype._processRoomEvents = function(room, stateEventList,
     // may make notifications appear which should have the right name.
     room.recalculate(this.client.credentials.userId);
 
+    // execute the timeline events, this will begin to diverge the current state
+    // if the timeline has any state events in it.
+    // This also needs to be done before running push rules on the events as they need
+    // to be decorated with sender etc.
+    room.addLiveEvents(timelineEventList);
+};
+
+/**
+ * Takes a list of timelineEvents and adds and adds to _notifEvents
+ * as appropriate.
+ * This must be called after the room the events belong to has been stored.
+ *
+ * @param {Room} room
+ * @param {MatrixEvent[]} [timelineEventList] A list of timeline events. Lower index
+ * is earlier in time. Higher index is later.
+ */
+SyncApi.prototype._processEventsForNotifs = function(room, timelineEventList) {
     // gather our notifications into this._notifEvents
-    if (client.getNotifTimelineSet()) {
+    if (this.client.getNotifTimelineSet()) {
         for (let i = 0; i < timelineEventList.length; i++) {
-            const pushActions = client.getPushActionsForEvent(timelineEventList[i]);
+            const pushActions = this.client.getPushActionsForEvent(timelineEventList[i]);
             if (pushActions && pushActions.notify &&
                 pushActions.tweaks && pushActions.tweaks.highlight) {
                 this._notifEvents.push(timelineEventList[i]);
             }
         }
     }
-
-    // execute the timeline events, this will begin to diverge the current state
-    // if the timeline has any state events in it.
-    room.addLiveEvents(timelineEventList);
 };
 
 /**
@@ -1245,30 +1364,12 @@ SyncApi.prototype._onOnline = function() {
 
 function createNewUser(client, userId) {
     const user = new User(userId);
-    reEmit(client, user, [
+    client.reEmitter.reEmit(user, [
         "User.avatarUrl", "User.displayName", "User.presence",
         "User.currentlyActive", "User.lastPresenceTs",
     ]);
     return user;
 }
 
-function reEmit(reEmitEntity, emittableEntity, eventNames) {
-    utils.forEach(eventNames, function(eventName) {
-        // setup a listener on the entity (the Room, User, etc) for this event
-        emittableEntity.on(eventName, function() {
-            // take the args from the listener and reuse them, adding the
-            // event name to the arg list so it works with .emit()
-            // Transformation Example:
-            // listener on "foo" => function(a,b) { ... }
-            // Re-emit on "thing" => thing.emit("foo", a, b)
-            const newArgs = [eventName];
-            for (let i = 0; i < arguments.length; i++) {
-                newArgs.push(arguments[i]);
-            }
-            reEmitEntity.emit(...newArgs);
-        });
-    });
-}
-
 /** */
 module.exports = SyncApi;

src/timeline-window.js

@@ -17,7 +17,7 @@ limitations under the License.
 
 /** @module timeline-window */
 
-const q = require("q");
+import Promise from 'bluebird';
 const EventTimeline = require("./models/event-timeline");
 
 /**
@@ -95,10 +95,30 @@ TimelineWindow.prototype.load = function(initialEventId, initialWindowSize) {
     const self = this;
     initialWindowSize = initialWindowSize || 20;
 
-    // given an EventTimeline, and an event index within it, initialise our
+    // given an EventTimeline, find the event we were looking for, and initialise our
     // fields so that the event in question is in the middle of the window.
-    const initFields = function(timeline, eventIndex) {
-        const endIndex = Math.min(timeline.getEvents().length,
+    const initFields = function(timeline) {
+        let eventIndex;
+
+        const events = timeline.getEvents();
+
+        if (!initialEventId) {
+            // we were looking for the live timeline: initialise to the end
+            eventIndex = events.length;
+        } else {
+            for (let i = 0; i < events.length; i++) {
+                if (events[i].getId() == initialEventId) {
+                    eventIndex = i;
+                    break;
+                }
+            }
+
+            if (eventIndex === undefined) {
+                throw new Error("getEventTimeline result didn't include requested event");
+            }
+        }
+
+        const endIndex = Math.min(events.length,
                                 eventIndex + Math.ceil(initialWindowSize / 2));
         const startIndex = Math.max(0, endIndex - initialWindowSize);
         self._start = new TimelineIndex(timeline, startIndex - timeline.getBaseIndex());
@@ -110,25 +130,19 @@ TimelineWindow.prototype.load = function(initialEventId, initialWindowSize) {
     // we already have the data we need, which is important to keep room-switching
     // feeling snappy.
     //
-    // TODO: ideally we'd spot getEventTimeline returning a resolved promise and
-    // skip straight to the find-event loop.
     if (initialEventId) {
-        return this._client.getEventTimeline(this._timelineSet, initialEventId)
-            .then(function(tl) {
-                // make sure that our window includes the event
-                for (let i = 0; i < tl.getEvents().length; i++) {
-                    if (tl.getEvents()[i].getId() == initialEventId) {
-                        initFields(tl, i);
-                        return;
-                    }
-                }
-                throw new Error("getEventTimeline result didn't include requested event");
-            });
+        const prom = this._client.getEventTimeline(this._timelineSet, initialEventId);
+
+        if (prom.isFulfilled()) {
+            initFields(prom.value());
+            return Promise.resolve();
+        } else {
+            return prom.then(initFields);
+        }
     } else {
-        // start with the most recent events
         const tl = this._timelineSet.getLiveTimeline();
-        initFields(tl, tl.getEvents().length);
-        return q();
+        initFields(tl);
+        return Promise.resolve();
     }
 };
 
@@ -220,7 +234,7 @@ TimelineWindow.prototype.paginate = function(direction, size, makeRequest,
 
     if (!tl) {
         debuglog("TimelineWindow: no timeline yet");
-        return q(false);
+        return Promise.resolve(false);
     }
 
     if (tl.pendingPaginate) {
@@ -240,20 +254,20 @@ TimelineWindow.prototype.paginate = function(direction, size, makeRequest,
         if (excess > 0) {
             this.unpaginate(excess, direction != EventTimeline.BACKWARDS);
         }
-        return q(true);
+        return Promise.resolve(true);
     }
 
     if (!makeRequest || requestLimit === 0) {
         // todo: should we return something different to indicate that there
         // might be more events out there, but we haven't found them yet?
-        return q(false);
+        return Promise.resolve(false);
     }
 
     // try making a pagination request
     const token = tl.timeline.getPaginationToken(direction);
     if (!token) {
         debuglog("TimelineWindow: no token");
-        return q(false);
+        return Promise.resolve(false);
     }
 
     debuglog("TimelineWindow: starting request");

src/webrtc/call.js

@@ -71,6 +71,7 @@ const DEBUG = true;  // set true to enable console logging.
  * @param {Object} opts Config options.
  * @param {string} opts.roomId The room ID for this call.
  * @param {Object} opts.webRtc The WebRTC globals from the browser.
+ * @param {boolean} opts.forceTURN whether relay through TURN should be forced.
  * @param {Object} opts.URL The URL global.
  * @param {Array<Object>} opts.turnServers Optional. A list of TURN servers.
  * @param {MatrixClient} opts.client The Matrix Client instance to send events to.
@@ -79,6 +80,7 @@ function MatrixCall(opts) {
     this.roomId = opts.roomId;
     this.client = opts.client;
     this.webRtc = opts.webRtc;
+    this.forceTURN = opts.forceTURN;
     this.URL = opts.URL;
     // Array of Objects with urls, username, credential keys
     this.turnServers = opts.turnServers || [];
@@ -1184,6 +1186,7 @@ const _createPeerConnection = function(self) {
     }
 
     const pc = new self.webRtc.RtcPeerConnection({
+        iceTransportPolicy: self.forceTURN ? 'relay' : undefined,
         iceServers: servers,
     });
     pc.oniceconnectionstatechange = hookCallback(self, self._onIceConnectionStateChanged);
@@ -1219,6 +1222,8 @@ const _getScreenSharingConstraints = function(call) {
 };
 
 const _getUserMediaVideoContraints = function(callType) {
+    const isWebkit = !!global.window.navigator.webkitGetUserMedia;
+
     switch (callType) {
         case 'voice':
             return {
@@ -1232,12 +1237,11 @@ const _getUserMediaVideoContraints = function(callType) {
                     deviceId: audioInput ? {exact: audioInput} : undefined,
                 }, video: {
                     deviceId: videoInput ? {exact: videoInput} : undefined,
-                    mandatory: {
-                        minWidth: 640,
-                        maxWidth: 640,
-                        minHeight: 360,
-                        maxHeight: 360,
-                    },
+                    /* We want 640x360.  Chrome will give it only if we ask exactly,
+                       FF refuses entirely if we ask exactly, so have to ask for ideal
+                       instead */
+                    width: isWebkit ? { exact: 640 } : { ideal: 640 },
+                    height: isWebkit ? { exact: 360 } : { ideal: 360 },
                 },
             };
     }
@@ -1292,9 +1296,11 @@ module.exports.setVideoInput = function(deviceId) { videoInput = deviceId; };
  * Create a new Matrix call for the browser.
  * @param {MatrixClient} client The client instance to use.
  * @param {string} roomId The room the call is in.
+ * @param {Object?} options DEPRECATED optional options map.
+ * @param {boolean} options.forceTURN DEPRECATED whether relay through TURN should be forced. This option is deprecated - use opts.forceTURN when creating the matrix client since it's only possible to set this option on outbound calls.
  * @return {MatrixCall} the call or null if the browser doesn't support calling.
  */
-module.exports.createNewMatrixCall = function(client, roomId) {
+module.exports.createNewMatrixCall = function(client, roomId, options) {
     const w = global.window;
     const doc = global.document;
     if (!w || !doc) {
@@ -1344,12 +1350,17 @@ module.exports.createNewMatrixCall = function(client, roomId) {
             !webRtc.RtcPeerConnection || !webRtc.getUserMedia) {
         return null; // WebRTC is not supported.
     }
+
+    const optionsForceTURN = options ? options.forceTURN : false;
+
     const opts = {
         webRtc: webRtc,
         client: client,
         URL: w.URL,
         roomId: roomId,
         turnServers: client.getTurnServers(),
+        // call level options
+        forceTURN: client._forceTURN || optionsForceTURN,
     };
     return new MatrixCall(opts);
 };

travis.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -ex
+
+npm run lint
+
+# install Olm so that we can run the crypto tests.
+npm install https://matrix.org/packages/npm/olm/olm-2.2.2.tgz
+
+npm run test
+
+npm run gendoc
+