v0.7.11

Add a bunch of logging around sending messages

.editorconfig

@@ -0,0 +1,23 @@
+#   Copyright 2017 Aviral Dasgupta
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+root = true
+
+[*]
+charset=utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = true

.gitignore

@@ -1,4 +1,6 @@
-.jsdoc
+/.jsdocbuild
+/.jsdoc
+
 node_modules
 .lock-wscript
 build/Release

CHANGELOG.md

@@ -1,3 +1,224 @@
+Changes in [0.7.11](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.11) (2017-06-12)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.11-rc.1...v0.7.11)
+
+ * Add a bunch of logging around sending messages
+   [\#460](https://github.com/matrix-org/matrix-js-sdk/pull/460)
+
+Changes in [0.7.11-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.11-rc.1) (2017-06-09)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.10...v0.7.11-rc.1)
+
+ * Make TimelineWindow.load resolve quicker if we have the events
+   [\#458](https://github.com/matrix-org/matrix-js-sdk/pull/458)
+ * Stop peeking when a matrix client is stopped
+   [\#451](https://github.com/matrix-org/matrix-js-sdk/pull/451)
+ * Update README: Clarify how to install libolm
+   [\#450](https://github.com/matrix-org/matrix-js-sdk/pull/450)
+
+Changes in [0.7.10](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.10) (2017-06-02)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.9...v0.7.10)
+
+ * BREAKING CHANGE: The SDK no longer ``require``s ``olm`` - instead it expects
+   libolm to be provided as an ``Olm`` global. This will only affect
+   applications which use end-to-end encryption. See the
+   [README](README.md#end-to-end-encryption-support) for details.
+
+ * indexeddb-crypto-store: fix db deletion
+   [\#447](https://github.com/matrix-org/matrix-js-sdk/pull/447)
+ * Load Olm from the global rather than requiring it.
+   [\#446](https://github.com/matrix-org/matrix-js-sdk/pull/446)
+
+Changes in [0.7.9](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.9) (2017-06-01)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.8...v0.7.9)
+
+ * Initial framework for indexeddb-backed crypto store
+   [\#445](https://github.com/matrix-org/matrix-js-sdk/pull/445)
+ * Factor out reEmit to a common module
+   [\#444](https://github.com/matrix-org/matrix-js-sdk/pull/444)
+ * crypto/algorithms/base.js: Convert to es6
+   [\#443](https://github.com/matrix-org/matrix-js-sdk/pull/443)
+ * maySendRedactionForEvent for userId
+   [\#435](https://github.com/matrix-org/matrix-js-sdk/pull/435)
+ * MatrixClient: add getUserId()
+   [\#441](https://github.com/matrix-org/matrix-js-sdk/pull/441)
+ * Run jsdoc on a custom babeling of the source
+   [\#442](https://github.com/matrix-org/matrix-js-sdk/pull/442)
+ * Add in a public api getStoredDevice allowing clients to get a specific
+   device
+   [\#439](https://github.com/matrix-org/matrix-js-sdk/pull/439)
+
+Changes in [0.7.8](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.8) (2017-05-22)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.8-rc.1...v0.7.8)
+
+ * No changes
+
+
+Changes in [0.7.8-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.8-rc.1) (2017-05-19)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.7...v0.7.8-rc.1)
+
+ * Attempt to rework the release-tarball-signing stuff
+   [\#438](https://github.com/matrix-org/matrix-js-sdk/pull/438)
+ * ability to specify webrtc audio/video inputs for the lib to request
+   [\#427](https://github.com/matrix-org/matrix-js-sdk/pull/427)
+ * make screen sharing call FF friendly :D
+   [\#434](https://github.com/matrix-org/matrix-js-sdk/pull/434)
+ * Fix race in device list updates
+   [\#431](https://github.com/matrix-org/matrix-js-sdk/pull/431)
+ * WebRTC: Support recvonly for video for those without a webcam
+   [\#424](https://github.com/matrix-org/matrix-js-sdk/pull/424)
+ * Update istanbul to remove minimatch DoS Warning
+   [\#422](https://github.com/matrix-org/matrix-js-sdk/pull/422)
+ * webrtc/call: Make it much less likely that callIds collide locally
+   [\#423](https://github.com/matrix-org/matrix-js-sdk/pull/423)
+ * Automatically complete dummy auth
+   [\#420](https://github.com/matrix-org/matrix-js-sdk/pull/420)
+ * Don't leave the gh-pages branch checked out
+   [\#418](https://github.com/matrix-org/matrix-js-sdk/pull/418)
+
+Changes in [0.7.7](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.7) (2017-04-25)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.7-rc.1...v0.7.7)
+
+ * No changes
+
+
+Changes in [0.7.7-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.7-rc.1) (2017-04-21)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.6...v0.7.7-rc.1)
+
+ * Automatically complete dummy auth
+   [\#420](https://github.com/matrix-org/matrix-js-sdk/pull/420)
+
+
+Changes in [0.7.6](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.6) (2017-04-12)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.6-rc.2...v0.7.6)
+
+ * No changes
+
+Changes in [0.7.6-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.6-rc.2) (2017-04-10)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.6-rc.1...v0.7.6-rc.2)
+
+ * Add feature detection for webworkers
+   [\#416](https://github.com/matrix-org/matrix-js-sdk/pull/416)
+ * Fix release script
+   [\#415](https://github.com/matrix-org/matrix-js-sdk/pull/415)
+
+Changes in [0.7.6-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.6-rc.1) (2017-04-07)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.5...v0.7.6-rc.1)
+
+ * Make indexeddb save after the first sync
+   [\#414](https://github.com/matrix-org/matrix-js-sdk/pull/414)
+ * Make indexeddb startup faster
+   [\#413](https://github.com/matrix-org/matrix-js-sdk/pull/413)
+ * Add ability to do indexeddb sync work in webworker
+   [\#412](https://github.com/matrix-org/matrix-js-sdk/pull/412)
+ * Move more functionality to the indexeddb backend
+   [\#409](https://github.com/matrix-org/matrix-js-sdk/pull/409)
+ * Indicate syncState ERROR after many failed /syncs
+   [\#410](https://github.com/matrix-org/matrix-js-sdk/pull/410)
+ * Further reorganising of indexeddb sync code
+   [\#407](https://github.com/matrix-org/matrix-js-sdk/pull/407)
+ * Change interface of IndexedDBStore: hide internals
+   [\#406](https://github.com/matrix-org/matrix-js-sdk/pull/406)
+ * Don't be SYNCING until updating from the server
+   [\#405](https://github.com/matrix-org/matrix-js-sdk/pull/405)
+ * Don't log the entire /sync response
+   [\#403](https://github.com/matrix-org/matrix-js-sdk/pull/403)
+ * webrtc/call: Assign MediaStream to video element srcObject
+   [\#402](https://github.com/matrix-org/matrix-js-sdk/pull/402)
+ * Fix undefined reference in http-api
+   [\#400](https://github.com/matrix-org/matrix-js-sdk/pull/400)
+ * Add copyright header to event-timeline.js
+   [\#382](https://github.com/matrix-org/matrix-js-sdk/pull/382)
+ * client: fix docs for user-scoped account_data events
+   [\#397](https://github.com/matrix-org/matrix-js-sdk/pull/397)
+ * Add a CONTRIBUTING for js-sdk
+   [\#399](https://github.com/matrix-org/matrix-js-sdk/pull/399)
+ * Fix leaking room state objects on limited sync responses
+   [\#395](https://github.com/matrix-org/matrix-js-sdk/pull/395)
+ * Extend 'ignoreFailure' to be 'background'
+   [\#396](https://github.com/matrix-org/matrix-js-sdk/pull/396)
+ * Add x_show_msisdn parameter to register calls
+   [\#388](https://github.com/matrix-org/matrix-js-sdk/pull/388)
+ * Update event redaction to keep sender and origin_server_ts
+   [\#394](https://github.com/matrix-org/matrix-js-sdk/pull/394)
+ * Handle 'limited' timeline responses in the SyncAccumulator
+   [\#393](https://github.com/matrix-org/matrix-js-sdk/pull/393)
+ * Give a better error message if the HS doesn't support msisdn registeration
+   [\#391](https://github.com/matrix-org/matrix-js-sdk/pull/391)
+ * Add getEmailSid
+   [\#383](https://github.com/matrix-org/matrix-js-sdk/pull/383)
+ * Add m.login.email.identity support to UI auth
+   [\#380](https://github.com/matrix-org/matrix-js-sdk/pull/380)
+ * src/client.js: Fix incorrect roomId reference in VoIP glare code
+   [\#381](https://github.com/matrix-org/matrix-js-sdk/pull/381)
+ * add .editorconfig
+   [\#379](https://github.com/matrix-org/matrix-js-sdk/pull/379)
+ * Store account data in the same way as room data
+   [\#377](https://github.com/matrix-org/matrix-js-sdk/pull/377)
+ * Upload one-time keys on /sync rather than a timer
+   [\#376](https://github.com/matrix-org/matrix-js-sdk/pull/376)
+ * Increase the WRITE_DELAY on database syncing
+   [\#374](https://github.com/matrix-org/matrix-js-sdk/pull/374)
+ * Make deleteAllData() return a Promise
+   [\#373](https://github.com/matrix-org/matrix-js-sdk/pull/373)
+ * Don't include banned users in the room name
+   [\#372](https://github.com/matrix-org/matrix-js-sdk/pull/372)
+ * Support IndexedDB as a backing store
+   [\#363](https://github.com/matrix-org/matrix-js-sdk/pull/363)
+ * Poll /sync with a short timeout while catching up
+   [\#370](https://github.com/matrix-org/matrix-js-sdk/pull/370)
+ * Make test coverage work again
+   [\#368](https://github.com/matrix-org/matrix-js-sdk/pull/368)
+ * Add docs to event
+   [\#367](https://github.com/matrix-org/matrix-js-sdk/pull/367)
+ * Keep the device-sync token more up-to-date
+   [\#366](https://github.com/matrix-org/matrix-js-sdk/pull/366)
+ * Fix race conditions in device list download
+   [\#365](https://github.com/matrix-org/matrix-js-sdk/pull/365)
+ * Fix the unban method
+   [\#364](https://github.com/matrix-org/matrix-js-sdk/pull/364)
+ * Spread out device verification work
+   [\#362](https://github.com/matrix-org/matrix-js-sdk/pull/362)
+ * Clean up/improve e2e logging
+   [\#361](https://github.com/matrix-org/matrix-js-sdk/pull/361)
+ * Fix decryption of events whose key arrives later
+   [\#360](https://github.com/matrix-org/matrix-js-sdk/pull/360)
+ * Invalidate device lists when encryption is enabled in a room
+   [\#359](https://github.com/matrix-org/matrix-js-sdk/pull/359)
+ * Switch from jasmine to mocha + expect + lolex
+   [\#358](https://github.com/matrix-org/matrix-js-sdk/pull/358)
+ * Install source-map-support in each test
+   [\#356](https://github.com/matrix-org/matrix-js-sdk/pull/356)
+ * searchMessageText: avoid setting keys=undefined
+   [\#357](https://github.com/matrix-org/matrix-js-sdk/pull/357)
+ * realtime-callbacks: pass `global` as `this`
+   [\#355](https://github.com/matrix-org/matrix-js-sdk/pull/355)
+ * Make the tests work without olm
+   [\#354](https://github.com/matrix-org/matrix-js-sdk/pull/354)
+ * Tests: Factor out TestClient and use it in crypto tests
+   [\#353](https://github.com/matrix-org/matrix-js-sdk/pull/353)
+ * Fix some lint
+   [\#352](https://github.com/matrix-org/matrix-js-sdk/pull/352)
+ * Make a sig for source tarballs when releasing
+   [\#351](https://github.com/matrix-org/matrix-js-sdk/pull/351)
+ * When doing a pre-release, don't bother merging to master and develop.
+   [\#350](https://github.com/matrix-org/matrix-js-sdk/pull/350)
+
+Changes in [0.7.5](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.5) (2017-02-04)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.5-rc.3...v0.7.5)
+
+No changes from 0.7.5-rc.3
+
 Changes in [0.7.5-rc.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.5-rc.3) (2017-02-03)
 ==========================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.5-rc.2...v0.7.5-rc.3)

CONTRIBUTING.rst

@@ -1,4 +1,115 @@
 Contributing code to matrix-js-sdk
 ==================================
 
-matrix-js-sdk follows the same pattern as https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst
+Everyone is welcome to contribute code to matrix-js-sdk, provided that they are
+willing to license their contributions under the same license as the project
+itself. We follow a simple 'inbound=outbound' model for contributions: the act
+of submitting an 'inbound' contribution means that the contributor agrees to
+license the code under the same terms as the project's overall 'outbound'
+license - in this case, Apache Software License v2 (see `<LICENSE>`_).
+
+How to contribute
+~~~~~~~~~~~~~~~~~
+
+The preferred and easiest way to contribute changes to the project is to fork
+it on github, and then create a pull request to ask us to pull your changes
+into our repo (https://help.github.com/articles/using-pull-requests/)
+
+**The single biggest thing you need to know is: please base your changes on
+the develop branch - /not/ master.**
+
+We use the master branch to track the most recent release, so that folks who
+blindly clone the repo and automatically check out master get something that
+works. Develop is the unstable branch where all the development actually
+happens: the workflow is that contributors should fork the develop branch to
+make a 'feature' branch for a particular contribution, and then make a pull
+request to merge this back into the matrix.org 'official' develop branch. We
+use github's pull request workflow to review the contribution, and either ask
+you to make any refinements needed or merge it and make them ourselves. The
+changes will then land on master when we next do a release.
+
+We use Travis for continuous integration, and all pull requests get
+automatically tested by Travis: if your change breaks the build, then the PR
+will show that there are failed checks, so please check back after a few
+minutes.
+
+Code style
+~~~~~~~~~~
+
+The code-style for matrix-js-sdk is not formally documented, but contributors
+are encouraged to read the code style document for matrix-react-sdk
+(`<https://github.com/matrix-org/matrix-react-sdk/blob/master/code_style.md>`_)
+and follow the principles set out there.
+
+Please ensure your changes match the cosmetic style of the existing project,
+and **never** mix cosmetic and functional changes in the same commit, as it
+makes it horribly hard to review otherwise.
+
+Attribution
+~~~~~~~~~~~
+
+Everyone who contributes anything to Matrix is welcome to be listed in the
+AUTHORS.rst file for the project in question. Please feel free to include a
+change to AUTHORS.rst in your pull request to list yourself and a short
+description of the area(s) you've worked on. Also, we sometimes have swag to
+give away to contributors - if you feel that Matrix-branded apparel is missing
+from your life, please mail us your shipping address to matrix at matrix.org
+and we'll try to fix it :)
+
+Sign off
+~~~~~~~~
+
+In order to have a concrete record that your contribution is intentional
+and you agree to license it under the same terms as the project's license, we've adopted the
+same lightweight approach that the Linux Kernel
+(https://www.kernel.org/doc/Documentation/SubmittingPatches), Docker
+(https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other
+projects use: the DCO (Developer Certificate of Origin:
+http://developercertificate.org/). This is a simple declaration that you wrote
+the contribution or otherwise have the right to contribute it to Matrix::
+
+    Developer Certificate of Origin
+    Version 1.1
+
+    Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+    660 York Street, Suite 102,
+    San Francisco, CA 94110 USA
+
+    Everyone is permitted to copy and distribute verbatim copies of this
+    license document, but changing it is not allowed.
+
+    Developer's Certificate of Origin 1.1
+
+    By making a contribution to this project, I certify that:
+
+    (a) The contribution was created in whole or in part by me and I
+        have the right to submit it under the open source license
+        indicated in the file; or
+
+    (b) The contribution is based upon previous work that, to the best
+        of my knowledge, is covered under an appropriate open source
+        license and I have the right under that license to submit that
+        work with modifications, whether created in whole or in part
+        by me, under the same open source license (unless I am
+        permitted to submit under a different license), as indicated
+        in the file; or
+
+    (c) The contribution was provided directly to me by some other
+        person who certified (a), (b) or (c) and I have not modified
+        it.
+
+    (d) I understand and agree that this project and the contribution
+        are public and that a record of the contribution (including all
+        personal information I submit with it, including my sign-off) is
+        maintained indefinitely and may be redistributed consistent with
+        this project or the open source license(s) involved.
+
+If you agree to this for your contribution, then all that's needed is to
+include the line in your commit or pull request comment::
+
+    Signed-off-by: Your Name <your@email.example.org>
+
+...using your real name; unfortunately pseudonyms and anonymous contributions
+can't be accepted. Git makes this trivial - just use the -s flag when you do
+``git commit``, having first set ``user.name`` and ``user.email`` git configs
+(which you should have done anyway :)

README.md

@@ -13,7 +13,8 @@ In a browser
 Download either the full or minified version from
 https://github.com/matrix-org/matrix-js-sdk/releases/latest and add that as a
 ``<script>`` to your page. There will be a global variable ``matrixcs``
-attached to ``window`` through which you can access the SDK.
+attached to ``window`` through which you can access the SDK. See below for how to
+include libolm to enable end-to-end-encryption.
 
 Please check [the working browser example](examples/browser) for more information.
 
@@ -29,8 +30,9 @@ In Node.js
     console.log("Public Rooms: %s", JSON.stringify(data));
   });
 ```
+See below for how to include libolm to enable end-to-end-encryption. Please check
+[the Node.js terminal app](examples/node) for a more complex example.
 
-Please check [the Node.js terminal app](examples/node) for a more complex example.
 
 What does this SDK do?
 ----------------------
@@ -236,6 +238,40 @@ host the API reference from the source files like this:
 
 Then visit ``http://localhost:8005`` to see the API docs.
 
+End-to-end encryption support
+=============================
+
+The SDK supports end-to-end encryption via the Olm and Megolm protocols, using
+[libolm](http://matrix.org/git/olm). It is left up to the application to make
+libolm available, via the ``Olm`` global.
+
+If the ``Olm`` global is not available, the SDK will show a warning:
+
+```
+Unable to load crypto module: crypto will be disabled: Error: global.Olm is not defined
+```
+
+The SDK will continue to work for unencrypted rooms, but it will not support
+the E2E parts of the Matrix specification.
+
+To enable E2E support in a browser application:
+
+ * download the transpiled libolm (from https://matrix.org/packages/npm/olm/).
+ * load ``olm.js`` as a ``<script>`` *before* ``browser-matrix.js``.
+ 
+To enable E2E support in a node.js application:
+
+ * ``npm install https://matrix.org/packages/npm/olm/olm-2.2.2.tgz``
+   (replace the URL with the latest version you want to use from
+    https://matrix.org/packages/npm/olm/)
+ * ``global.Olm = require('olm');`` *before* loading ``matrix-js-sdk``.
+
+If you want to package Olm as dependency for your node.js application, you
+can use ``npm install https://matrix.org/packages/npm/olm/olm-2.2.2.tgz 
+--save-optional`` (if your application also works without e2e crypto enabled)
+or ``--save`` (if it doesn't) to do so.
+
+
 Contributing
 ============
 *This section is for people who want to modify the SDK. If you just

browser-index.js

@@ -1,4 +1,13 @@
 var matrixcs = require("./lib/matrix");
 matrixcs.request(require("browser-request"));
+
+matrixcs.setCryptoStoreFactory(
+    function() {
+        return new matrixcs.IndexedDBCryptoStore(
+            global.indexedDB, "matrix-js-sdk:crypto"
+        );
+    }
+);
+
 module.exports = matrixcs; // keep export for browserify package deps
 global.matrixcs = matrixcs;

examples/voip/browserTest.js

@@ -44,7 +44,7 @@ window.onload = function() {
     disableButtons(true, true, true);
 };
 
-matrixClient.on("sync", function(state, prevState, data) {
+client.on("sync", function(state, prevState, data) {
     switch (state) {
         case "PREPARED":
           syncComplete();

jenkins.sh

@@ -2,7 +2,7 @@
 
 set -x
 
-export NVM_DIR="/home/jenkins/.nvm"
+export NVM_DIR="$HOME/.nvm"
 [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
 
 nvm use 6 || exit $?
@@ -15,6 +15,9 @@ function fail {
     RC=1
 }
 
+# don't use last time's test reports
+rm -rf reports coverage || exit $?
+
 npm test || fail "npm test finished with return code $?"
 
 npm run -s lint -- -f checkstyle > eslint.xml ||

package.json

@@ -1,18 +1,19 @@
 {
   "name": "matrix-js-sdk",
-  "version": "0.7.5-rc.3",
+  "version": "0.7.11",
   "description": "Matrix Client-Server SDK for Javascript",
   "main": "index.js",
   "scripts": {
-    "buildtest": "babel -s -d specbuild spec",
-    "test": "npm run buildtest && istanbul cover --report cobertura --config .istanbul.yml -i \"lib/**/*.js\" jasmine-node -- specbuild --verbose --junitreport --captureExceptions",
-    "check": "npm run buildtest && jasmine-node specbuild --verbose --junitreport --captureExceptions",
-    "gendoc": "jsdoc -r lib -P package.json -R README.md -d .jsdoc",
+    "test:build": "babel -s -d specbuild spec",
+    "test:run": "istanbul cover --report text --report cobertura --config .istanbul.yml -i \"lib/**/*.js\" _mocha -- --recursive specbuild --colors --reporter mocha-jenkins-reporter --reporter-options junit_report_path=reports/test-results.xml",
+    "test": "npm run test:build && npm run test:run",
+    "check": "npm run test:build && _mocha --recursive specbuild --colors",
+    "gendoc": "babel --no-babelrc -d .jsdocbuild src && jsdoc -r .jsdocbuild -P package.json -R README.md -d .jsdoc",
     "start": "babel -s -w -d lib src",
     "build": "babel -s -d lib src && rimraf dist && mkdir dist && browserify -d browser-index.js | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js && uglifyjs -c -m -o dist/browser-matrix.min.js --source-map dist/browser-matrix.min.js.map --in-source-map dist/browser-matrix.js.map dist/browser-matrix.js",
     "dist": "npm run build",
     "watch": "watchify -d browser-index.js -o 'exorcist dist/browser-matrix.js.map > dist/browser-matrix.js' -v",
-    "lint": "eslint --max-warnings 121 src spec",
+    "lint": "eslint --max-warnings 113 src spec",
     "prepublish": "npm run build && git rev-parse HEAD > git-revision.txt"
   },
   "repository": {
@@ -60,24 +61,21 @@
     "eslint": "^3.13.1",
     "eslint-config-google": "^0.7.1",
     "exorcist": "^0.4.0",
-    "istanbul": "^0.3.13",
-    "jasmine-node": "^1.14.5",
+    "expect": "^1.20.2",
+    "istanbul": "^0.4.5",
     "jsdoc": "^3.4.0",
+    "lolex": "^1.5.2",
+    "mocha": "^3.2.0",
+    "mocha-jenkins-reporter": "^0.3.6",
     "rimraf": "^2.5.4",
+    "source-map-support": "^0.4.11",
     "sourceify": "^0.1.0",
-    "uglifyjs": "^2.4.10",
+    "uglify-js": "^2.8.26",
     "watchify": "^3.2.1"
   },
-  "optionalDependencies": {
-    "olm": "https://matrix.org/packages/npm/olm/olm-2.2.1.tgz"
-  },
   "browserify": {
     "transform": [
-      "sourceify",
-      "browserify-shim"
+      "sourceify"
     ]
-  },
-  "browserify-shim": {
-    "olm": "global:Olm"
   }
 }

release.sh

@@ -178,10 +178,6 @@ if [ $dodist -eq 0 ]; then
     done
 fi
 
-# push the release branch (github can't release from
-# a branch it doesn't have)
-git push origin "$rel_branch"
-
 if [ -n "$signing_id" ]; then
     # make a signed tag
     # gnupg seems to fail to get the right tty device unless we set it here
@@ -190,8 +186,55 @@ else
     git tag -a -F "${latest_changes}" "$tag"
 fi
 
-# push the tag
-git push origin "$tag"
+# push the tag and the release branch
+git push origin "$rel_branch" "$tag"
+
+if [ -n "$signing_id" ]; then
+    # make a signature for the source tarball.
+    #
+    # github will make us a tarball from the tag - we want to create a
+    # signature for it, which means that first of all we need to check that
+    # it's correct.
+    #
+    # we can't deterministically build exactly the same tarball, due to
+    # differences in gzip implementation - but we *can* build the same tar - so
+    # the easiest way to check the validity of the tarball from git is to unzip
+    # it and compare it with our own idea of what the tar should look like.
+
+    # the name of the sig file we want to create
+    source_sigfile="${tag}-src.tar.gz.asc"
+
+    tarfile="$tag.tar.gz"
+    gh_project_url=$(git remote get-url origin |
+                            sed -e 's#^git@github\.com:#https://github.com/#' \
+                                -e 's#^git\+ssh://git@github\.com/#https://github.com/#' \
+                                -e 's/\.git$//')
+    project_name="${gh_project_url##*/}"
+    curl -L "${gh_project_url}/archive/${tarfile}" -o "${tarfile}"
+
+    # unzip it and compare it with the tar we would generate
+    if ! cmp --silent <(gunzip -c $tarfile) \
+         <(git archive --format tar --prefix="${project_name}-${release}/" "$tag"); then
+
+        # we don't bail out here, because really it's more likely that our comparison
+        # screwed up and it's super annoying to abort the script at this point.
+        cat >&2 <<EOF
+!!!!!!!!!!!!!!!!!
+!!!! WARNING !!!!
+
+Mismatch between our own tarfile and that generated by github: not signing
+source tarball.
+
+To resolve, determine if $tarfile is correct, and if so sign it with gpg and
+attach it to the release as $source_sigfile.
+
+!!!!!!!!!!!!!!!!!
+EOF
+    else
+        gpg -u "$signing_id" --armor --output "$source_sigfile" --detach-sig "$tarfile"
+        assets="$assets -a $source_sigfile"
+    fi
+fi
 
 hubflags=''
 if [ $prerelease -eq 1 ]; then
@@ -210,6 +253,9 @@ fi
 rm "${release_text}"
 rm "${latest_changes}"
 
+# publish to npmjs
+npm publish
+
 if [ -z "$skip_jsdoc" ]; then
     echo "generating jsdocs"
     npm run gendoc
@@ -225,6 +271,12 @@ if [ -z "$skip_jsdoc" ]; then
     git commit --no-verify -m "Add jsdoc for $release" index.html "$release"
 fi
 
+# if it is a pre-release, leave it on the release branch for now.
+if [ $prerelease -eq 1 ]; then
+    git checkout "$rel_branch"
+    exit 0
+fi
+
 # merge release branch to master
 echo "updating master branch"
 git checkout master
@@ -237,9 +289,6 @@ if [ -z "$skip_jsdoc" ]; then
     git push origin gh-pages
 fi
 
-# publish to npmjs
-npm publish
-
 # finally, merge master back onto develop
 git checkout develop
 git pull

spec/.eslintrc.js

@@ -1,5 +1,5 @@
 module.exports = {
     env: {
-        jasmine: true,
+        mocha: true,
     },
 }

spec/TestClient.js

@@ -0,0 +1,186 @@
+/*
+Copyright 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+"use strict";
+
+import sdk from '..';
+import testUtils from './test-utils';
+import MockHttpBackend from './mock-request';
+import expect from 'expect';
+import q from 'q';
+
+/**
+ * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
+ *
+ * @constructor
+ * @param {string} userId
+ * @param {string} deviceId
+ * @param {string} accessToken
+ */
+export default function TestClient(userId, deviceId, accessToken) {
+    this.userId = userId;
+    this.deviceId = deviceId;
+
+    this.storage = new sdk.WebStorageSessionStore(new testUtils.MockStorageApi());
+    this.httpBackend = new MockHttpBackend();
+    this.client = sdk.createClient({
+        baseUrl: "http://" + userId + ".test.server",
+        userId: userId,
+        accessToken: accessToken,
+        deviceId: deviceId,
+        sessionStore: this.storage,
+        request: this.httpBackend.requestFn,
+    });
+
+    this.deviceKeys = null;
+    this.oneTimeKeys = {};
+}
+
+TestClient.prototype.toString = function() {
+    return 'TestClient[' + this.userId + ']';
+};
+
+/**
+ * start the client, and wait for it to initialise.
+ *
+ * @return {Promise}
+ */
+TestClient.prototype.start = function() {
+    console.log(this + ': starting');
+    this.httpBackend.when("GET", "/pushrules").respond(200, {});
+    this.httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+    this.expectDeviceKeyUpload();
+
+    // we let the client do a very basic initial sync, which it needs before
+    // it will upload one-time keys.
+    this.httpBackend.when("GET", "/sync").respond(200, { next_batch: 1 });
+
+    this.client.startClient({
+        // set this so that we can get hold of failed events
+        pendingEventOrdering: 'detached',
+    });
+
+    return this.httpBackend.flush().then(() => {
+        console.log(this + ': started');
+    });
+};
+
+/**
+ * stop the client
+ */
+TestClient.prototype.stop = function() {
+    this.client.stopClient();
+};
+
+/**
+ * Set up expectations that the client will upload device keys.
+ */
+TestClient.prototype.expectDeviceKeyUpload = function() {
+    const self = this;
+    this.httpBackend.when("POST", "/keys/upload").respond(200, function(path, content) {
+        expect(content.one_time_keys).toBe(undefined);
+        expect(content.device_keys).toBeTruthy();
+
+        console.log(self + ': received device keys');
+        // we expect this to happen before any one-time keys are uploaded.
+        expect(Object.keys(self.oneTimeKeys).length).toEqual(0);
+
+        self.deviceKeys = content.device_keys;
+        return {one_time_key_counts: {signed_curve25519: 0}};
+    });
+};
+
+
+/**
+ * If one-time keys have already been uploaded, return them. Otherwise,
+ * set up an expectation that the keys will be uploaded, and wait for
+ * that to happen.
+ *
+ * @returns {Promise} for the one-time keys
+ */
+TestClient.prototype.awaitOneTimeKeyUpload = function() {
+    if (Object.keys(this.oneTimeKeys).length != 0) {
+        // already got one-time keys
+        return q(this.oneTimeKeys);
+    }
+
+    this.httpBackend.when("POST", "/keys/upload")
+        .respond(200, (path, content) => {
+            expect(content.device_keys).toBe(undefined);
+            expect(content.one_time_keys).toBe(undefined);
+            return {one_time_key_counts: {
+                signed_curve25519: Object.keys(this.oneTimeKeys).length,
+            }};
+        });
+
+    this.httpBackend.when("POST", "/keys/upload")
+          .respond(200, (path, content) => {
+              expect(content.device_keys).toBe(undefined);
+              expect(content.one_time_keys).toBeTruthy();
+              expect(content.one_time_keys).toNotEqual({});
+              console.log('%s: received %i one-time keys', this,
+                          Object.keys(content.one_time_keys).length);
+              this.oneTimeKeys = content.one_time_keys;
+              return {one_time_key_counts: {
+                  signed_curve25519: Object.keys(this.oneTimeKeys).length,
+              }};
+          });
+
+    return this.httpBackend.flush('/keys/upload', 2).then((flushed) => {
+        expect(flushed).toEqual(2);
+        return this.oneTimeKeys;
+    });
+};
+
+/**
+ * Set up expectations that the client will query device keys.
+ *
+ * We check that the query contains each of the users in `response`.
+ *
+ * @param {Object} response   response to the query.
+ */
+TestClient.prototype.expectKeyQuery = function(response) {
+    this.httpBackend.when('POST', '/keys/query').respond(
+        200, (path, content) => {
+            Object.keys(response.device_keys).forEach((userId) => {
+                expect(content.device_keys[userId]).toEqual({});
+            });
+            return response;
+        });
+};
+
+
+/**
+ * get the uploaded curve25519 device key
+ *
+ * @return {string} base64 device key
+ */
+TestClient.prototype.getDeviceKey = function() {
+    const keyId = 'curve25519:' + this.deviceId;
+    return this.deviceKeys.keys[keyId];
+};
+
+
+/**
+ * get the uploaded ed25519 device key
+ *
+ * @return {string} base64 device key
+ */
+TestClient.prototype.getSigningKey = function() {
+    const keyId = 'ed25519:' + this.deviceId;
+    return this.deviceKeys.keys[keyId];
+};

spec/integ/matrix-client-crypto.spec.js

@@ -1,115 +1,59 @@
+/*
+Copyright 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/* This file consists of a set of integration tests which try to simulate
+ * communication via an Olm-encrypted room between two users, Alice and Bob.
+ *
+ * Note that megolm (group) conversation is not tested here.
+ *
+ * See also `megolm.spec.js`.
+ */
+
 "use strict";
+import 'source-map-support/register';
+import expect from 'expect';
 const sdk = require("../..");
 const q = require("q");
-const HttpBackend = require("../mock-request");
 const utils = require("../../lib/utils");
 const testUtils = require("../test-utils");
+const TestClient = require('../TestClient').default;
 
-let aliHttpBackend;
-let bobHttpBackend;
-let aliClient;
+let aliTestClient;
 const roomId = "!room:localhost";
 const aliUserId = "@ali:localhost";
 const aliDeviceId = "zxcvb";
 const aliAccessToken = "aseukfgwef";
-let bobClient;
+let bobTestClient;
 const bobUserId = "@bob:localhost";
 const bobDeviceId = "bvcxz";
 const bobAccessToken = "fewgfkuesa";
-let bobOneTimeKeys;
-let aliDeviceKeys;
-let bobDeviceKeys;
-let bobDeviceCurve25519Key;
-let bobDeviceEd25519Key;
-let aliStorage;
-let bobStorage;
 let aliMessages;
 let bobMessages;
 
-
-/**
- * Set an expectation that the client will upload device keys and a number of
- * one-time keys; then flush the http requests.
- *
- * @param {string} deviceId expected device id in upload request
- * @param {object} httpBackend
- *
- * @return {promise} completes once the http requests have completed, returning combined
- * {one_time_keys: {}, device_keys: {}}
- */
-function expectKeyUpload(deviceId, httpBackend) {
-    const uploadPath = "/keys/upload/" + deviceId;
-    const keys = {};
-
-    httpBackend.when("POST", uploadPath).respond(200, function(path, content) {
-        expect(content.one_time_keys).not.toBeDefined();
-        expect(content.device_keys).toBeDefined();
-        keys.device_keys = content.device_keys;
-        return {one_time_key_counts: {signed_curve25519: 0}};
-    });
-
-    httpBackend.when("POST", uploadPath).respond(200, function(path, content) {
-        expect(content.device_keys).not.toBeDefined();
-        expect(content.one_time_keys).toBeDefined();
-        expect(content.one_time_keys).not.toEqual({});
-        let count = 0;
-        for (const key in content.one_time_keys) {
-            if (content.one_time_keys.hasOwnProperty(key)) {
-                count++;
-            }
-        }
-        expect(count).toEqual(5);
-        keys.one_time_keys = content.one_time_keys;
-        return {one_time_key_counts: {signed_curve25519: count}};
-    });
-
-    return httpBackend.flush(uploadPath, 2).then(function() {
-        return keys;
+function bobUploadsDeviceKeys() {
+    bobTestClient.expectDeviceKeyUpload();
+    return q.all([
+        bobTestClient.client.uploadKeys(),
+        bobTestClient.httpBackend.flush(),
+    ]).then(() => {
+        expect(Object.keys(bobTestClient.deviceKeys).length).toNotEqual(0);
     });
 }
 
-
-/**
- * Set an expectation that ali will upload device keys and a number of one-time keys;
- * then flush the http requests.
- *
- * <p>Updates <tt>aliDeviceKeys</tt>
- *
- * @return {promise} completes once the http requests have completed.
- */
-function expectAliKeyUpload() {
-    return expectKeyUpload(aliDeviceId, aliHttpBackend).then(function(content) {
-        aliDeviceKeys = content.device_keys;
-    });
-}
-
-
-/**
- * Set an expectation that bob will upload device keys and a number of one-time keys;
- * then flush the http requests.
- *
- * <p>Updates <tt>bobDeviceKeys</tt>, <tt>bobOneTimeKeys</tt>,
- * <tt>bobDeviceCurve25519Key</tt>, <tt>bobDeviceEd25519Key</tt>
- *
- * @return {promise} completes once the http requests have completed.
- */
-function expectBobKeyUpload() {
-    return expectKeyUpload(bobDeviceId, bobHttpBackend).then(function(content) {
-        bobDeviceKeys = content.device_keys;
-        bobOneTimeKeys = content.one_time_keys;
-        expect(bobDeviceKeys).toBeDefined();
-        expect(bobOneTimeKeys).toBeDefined();
-        bobDeviceCurve25519Key = bobDeviceKeys.keys["curve25519:bvcxz"];
-        bobDeviceEd25519Key = bobDeviceKeys.keys["ed25519:bvcxz"];
-    });
-}
-
-function bobUploadsKeys() {
-    bobClient.uploadKeys(5).catch(testUtils.failTest);
-    return expectBobKeyUpload();
-}
-
-
 /**
  * Set an expectation that ali will query bobs keys; then flush the http request.
  *
@@ -117,17 +61,18 @@ function bobUploadsKeys() {
  */
 function expectAliQueryKeys() {
     // can't query keys before bob has uploaded them
-    expect(bobDeviceKeys).toBeDefined();
+    expect(bobTestClient.deviceKeys).toBeTruthy();
 
     const bobKeys = {};
-    bobKeys[bobDeviceId] = bobDeviceKeys;
-    aliHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
+    bobKeys[bobDeviceId] = bobTestClient.deviceKeys;
+    aliTestClient.httpBackend.when("POST", "/keys/query")
+            .respond(200, function(path, content) {
         expect(content.device_keys[bobUserId]).toEqual({});
         const result = {};
         result[bobUserId] = bobKeys;
         return {device_keys: result};
     });
-    return aliHttpBackend.flush("/keys/query", 1);
+    return aliTestClient.httpBackend.flush("/keys/query", 1);
 }
 
 /**
@@ -137,17 +82,21 @@ function expectAliQueryKeys() {
  */
 function expectBobQueryKeys() {
     // can't query keys before ali has uploaded them
-    expect(aliDeviceKeys).toBeDefined();
+    expect(aliTestClient.deviceKeys).toBeTruthy();
 
     const aliKeys = {};
-    aliKeys[aliDeviceId] = aliDeviceKeys;
-    bobHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
+    aliKeys[aliDeviceId] = aliTestClient.deviceKeys;
+    console.log("query result will be", aliKeys);
+
+    bobTestClient.httpBackend.when(
+        "POST", "/keys/query",
+    ).respond(200, function(path, content) {
         expect(content.device_keys[aliUserId]).toEqual({});
         const result = {};
         result[aliUserId] = aliKeys;
         return {device_keys: result};
     });
-    return bobHttpBackend.flush("/keys/query", 1);
+    return bobTestClient.httpBackend.flush("/keys/query", 1);
 }
 
 /**
@@ -156,39 +105,45 @@ function expectBobQueryKeys() {
  * @return {promise} resolves once the http request has completed.
  */
 function expectAliClaimKeys() {
-    // can't query keys before bob has uploaded them
-    expect(bobOneTimeKeys).toBeDefined();
-
-    aliHttpBackend.when("POST", "/keys/claim").respond(200, function(path, content) {
-        const claimType = content.one_time_keys[bobUserId][bobDeviceId];
-        expect(claimType).toEqual("signed_curve25519");
-        let keyId = null;
-        for (keyId in bobOneTimeKeys) {
-            if (bobOneTimeKeys.hasOwnProperty(keyId)) {
-                if (keyId.indexOf(claimType + ":") === 0) {
-                    break;
+    return bobTestClient.awaitOneTimeKeyUpload().then((keys) => {
+        aliTestClient.httpBackend.when(
+            "POST", "/keys/claim",
+        ).respond(200, function(path, content) {
+            const claimType = content.one_time_keys[bobUserId][bobDeviceId];
+            expect(claimType).toEqual("signed_curve25519");
+            let keyId = null;
+            for (keyId in keys) {
+                if (bobTestClient.oneTimeKeys.hasOwnProperty(keyId)) {
+                    if (keyId.indexOf(claimType + ":") === 0) {
+                        break;
+                    }
                 }
             }
-        }
-        const result = {};
-        result[bobUserId] = {};
-        result[bobUserId][bobDeviceId] = {};
-        result[bobUserId][bobDeviceId][keyId] = bobOneTimeKeys[keyId];
-        return {one_time_keys: result};
+            const result = {};
+            result[bobUserId] = {};
+            result[bobUserId][bobDeviceId] = {};
+            result[bobUserId][bobDeviceId][keyId] = keys[keyId];
+            return {one_time_keys: result};
+        });
+    }).then(() => {
+        // it can take a while to process the key query, so give it some extra
+        // time, and make sure the claim actually happens rather than ploughing on
+        // confusingly.
+        return aliTestClient.httpBackend.flush("/keys/claim", 1, 20).then((r) => {
+            expect(r).toEqual(1);
+        });
     });
-
-    return aliHttpBackend.flush("/keys/claim", 1);
 }
 
 
 function aliDownloadsKeys() {
     // can't query keys before bob has uploaded them
-    expect(bobDeviceEd25519Key).toBeDefined();
+    expect(bobTestClient.getSigningKey()).toBeTruthy();
 
-    const p1 = aliClient.downloadKeys([bobUserId]).then(function() {
-        expect(aliClient.listDeviceKeys(bobUserId)).toEqual([{
+    const p1 = aliTestClient.client.downloadKeys([bobUserId]).then(function() {
+        expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([{
             id: "bvcxz",
-            key: bobDeviceEd25519Key,
+            key: bobTestClient.getSigningKey(),
             verified: false,
             blocked: false,
             display_name: null,
@@ -199,26 +154,26 @@ function aliDownloadsKeys() {
     // check that the localStorage is updated as we expect (not sure this is
     // an integration test, but meh)
     return q.all([p1, p2]).then(function() {
-        const devices = aliStorage.getEndToEndDevicesForUser(bobUserId);
-        expect(devices[bobDeviceId].keys).toEqual(bobDeviceKeys.keys);
+        const devices = aliTestClient.storage.getEndToEndDevicesForUser(bobUserId);
+        expect(devices[bobDeviceId].keys).toEqual(bobTestClient.deviceKeys.keys);
         expect(devices[bobDeviceId].verified).
             toBe(0); // DeviceVerification.UNVERIFIED
     });
 }
 
 function aliEnablesEncryption() {
-    return aliClient.setRoomEncryption(roomId, {
+    return aliTestClient.client.setRoomEncryption(roomId, {
         algorithm: "m.olm.v1.curve25519-aes-sha2",
     }).then(function() {
-        expect(aliClient.isRoomEncrypted(roomId)).toBeTruthy();
+        expect(aliTestClient.client.isRoomEncrypted(roomId)).toBeTruthy();
     });
 }
 
 function bobEnablesEncryption() {
-    return bobClient.setRoomEncryption(roomId, {
+    return bobTestClient.client.setRoomEncryption(roomId, {
         algorithm: "m.olm.v1.curve25519-aes-sha2",
     }).then(function() {
-       expect(bobClient.isRoomEncrypted(roomId)).toBeTruthy();
+       expect(bobTestClient.client.isRoomEncrypted(roomId)).toBeTruthy();
     });
 }
 
@@ -230,7 +185,7 @@ function bobEnablesEncryption() {
  */
 function aliSendsFirstMessage() {
     return q.all([
-        sendMessage(aliClient),
+        sendMessage(aliTestClient.client),
         expectAliQueryKeys()
             .then(expectAliClaimKeys)
             .then(expectAliSendMessageRequest),
@@ -247,7 +202,7 @@ function aliSendsFirstMessage() {
  */
 function aliSendsMessage() {
     return q.all([
-        sendMessage(aliClient),
+        sendMessage(aliTestClient.client),
         expectAliSendMessageRequest(),
     ]).spread(function(_, ciphertext) {
         return ciphertext;
@@ -262,7 +217,7 @@ function aliSendsMessage() {
  */
 function bobSendsReplyMessage() {
     return q.all([
-        sendMessage(bobClient),
+        sendMessage(bobTestClient.client),
         expectBobQueryKeys()
             .then(expectBobSendMessageRequest),
     ]).spread(function(_, ciphertext) {
@@ -276,11 +231,11 @@ function bobSendsReplyMessage() {
  * @return {promise} which resolves to the ciphertext for Bob's device.
  */
 function expectAliSendMessageRequest() {
-    return expectSendMessageRequest(aliHttpBackend).then(function(content) {
+    return expectSendMessageRequest(aliTestClient.httpBackend).then(function(content) {
         aliMessages.push(content);
-        expect(utils.keys(content.ciphertext)).toEqual([bobDeviceCurve25519Key]);
-        const ciphertext = content.ciphertext[bobDeviceCurve25519Key];
-        expect(ciphertext).toBeDefined();
+        expect(utils.keys(content.ciphertext)).toEqual([bobTestClient.getDeviceKey()]);
+        const ciphertext = content.ciphertext[bobTestClient.getDeviceKey()];
+        expect(ciphertext).toBeTruthy();
         return ciphertext;
     });
 }
@@ -291,13 +246,13 @@ function expectAliSendMessageRequest() {
  * @return {promise} which resolves to the ciphertext for Bob's device.
  */
 function expectBobSendMessageRequest() {
-    return expectSendMessageRequest(bobHttpBackend).then(function(content) {
+    return expectSendMessageRequest(bobTestClient.httpBackend).then(function(content) {
         bobMessages.push(content);
         const aliKeyId = "curve25519:" + aliDeviceId;
-        const aliDeviceCurve25519Key = aliDeviceKeys.keys[aliKeyId];
+        const aliDeviceCurve25519Key = aliTestClient.deviceKeys.keys[aliKeyId];
         expect(utils.keys(content.ciphertext)).toEqual([aliDeviceCurve25519Key]);
         const ciphertext = content.ciphertext[aliDeviceCurve25519Key];
-        expect(ciphertext).toBeDefined();
+        expect(ciphertext).toBeTruthy();
         return ciphertext;
     });
 }
@@ -310,26 +265,30 @@ function sendMessage(client) {
 
 function expectSendMessageRequest(httpBackend) {
     const path = "/send/m.room.encrypted/";
-    let sent;
+    const deferred = q.defer();
     httpBackend.when("PUT", path).respond(200, function(path, content) {
-        sent = content;
+        deferred.resolve(content);
         return {
             event_id: "asdfgh",
         };
     });
-    return httpBackend.flush(path, 1).then(function() {
-        return sent;
-    });
+
+    // it can take a while to process the key query, so give it 20ms
+    return httpBackend.flush(path, 1, 20).then(() => deferred.promise);
 }
 
 function aliRecvMessage() {
     const message = bobMessages.shift();
-    return recvMessage(aliHttpBackend, aliClient, bobUserId, message);
+    return recvMessage(
+        aliTestClient.httpBackend, aliTestClient.client, bobUserId, message,
+    );
 }
 
 function bobRecvMessage() {
     const message = aliMessages.shift();
-    return recvMessage(bobHttpBackend, bobClient, aliUserId, message);
+    return recvMessage(
+        bobTestClient.httpBackend, bobTestClient.client, aliUserId, message,
+    );
 }
 
 function recvMessage(httpBackend, client, sender, message) {
@@ -382,52 +341,14 @@ function recvMessage(httpBackend, client, sender, message) {
 }
 
 
-function aliStartClient() {
-    expectAliKeyUpload().catch(testUtils.failTest);
-
-    // ali will try to query her own keys on start
-    aliHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
-        expect(content.device_keys[aliUserId]).toEqual({});
-        const result = {};
-        result[aliUserId] = {};
-        return {device_keys: result};
-    });
-
-    startClient(aliHttpBackend, aliClient);
-    return aliHttpBackend.flush().then(function() {
-        console.log("Ali client started");
-    });
-}
-
-function bobStartClient() {
-    expectBobKeyUpload().catch(testUtils.failTest);
-
-    // bob will try to query his own keys on start
-    bobHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
-        expect(content.device_keys[bobUserId]).toEqual({});
-        const result = {};
-        result[bobUserId] = {};
-        return {device_keys: result};
-    });
-
-    startClient(bobHttpBackend, bobClient);
-    return bobHttpBackend.flush().then(function() {
-        console.log("Bob client started");
-    });
-}
-
-
 /**
- * Set http responses for the requests which are made when a client starts, and
- * start the client.
+ * Send an initial sync response to the client (which just includes the member
+ * list for our test room).
  *
- * @param {object} httpBackend
- * @param {MatrixClient} client
+ * @param {TestClient} testClient
+ * @returns {Promise} which resolves when the sync has been flushed.
  */
-function startClient(httpBackend, client) {
-    httpBackend.when("GET", "/pushrules").respond(200, {});
-    httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
-
+function firstSync(testClient) {
     // send a sync response including our test room.
     const syncData = {
         next_batch: "x",
@@ -452,9 +373,11 @@ function startClient(httpBackend, client) {
             events: [],
         },
     };
-    httpBackend.when("GET", "/sync").respond(200, syncData);
 
-    client.startClient();
+    testClient.httpBackend.when("GET", "/sync").respond(200, syncData);
+    return testClient.httpBackend.flush("/sync", 1).then(() => {
+        return testUtils.syncPromise(testClient.client);
+    });
 }
 
 
@@ -466,90 +389,47 @@ describe("MatrixClient crypto", function() {
     beforeEach(function() {
         testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
 
-        aliStorage = new sdk.WebStorageSessionStore(new testUtils.MockStorageApi());
-        aliHttpBackend = new HttpBackend();
-        aliClient = sdk.createClient({
-            baseUrl: "http://alis.server",
-            userId: aliUserId,
-            accessToken: aliAccessToken,
-            deviceId: aliDeviceId,
-            sessionStore: aliStorage,
-            request: aliHttpBackend.requestFn,
-        });
+        aliTestClient = new TestClient(aliUserId, aliDeviceId, aliAccessToken);
+        bobTestClient = new TestClient(bobUserId, bobDeviceId, bobAccessToken);
 
-        bobStorage = new sdk.WebStorageSessionStore(new testUtils.MockStorageApi());
-        bobHttpBackend = new HttpBackend();
-        bobClient = sdk.createClient({
-            baseUrl: "http://bobs.server",
-            userId: bobUserId,
-            accessToken: bobAccessToken,
-            deviceId: bobDeviceId,
-            sessionStore: bobStorage,
-            request: bobHttpBackend.requestFn,
-        });
-
-        bobOneTimeKeys = undefined;
-        aliDeviceKeys = undefined;
-        bobDeviceKeys = undefined;
-        bobDeviceCurve25519Key = undefined;
-        bobDeviceEd25519Key = undefined;
         aliMessages = [];
         bobMessages = [];
     });
 
     afterEach(function() {
-        aliClient.stopClient();
-        bobClient.stopClient();
+        aliTestClient.stop();
+        aliTestClient.httpBackend.verifyNoOutstandingExpectation();
+        bobTestClient.stop();
+        bobTestClient.httpBackend.verifyNoOutstandingExpectation();
     });
 
-    it('Ali knows the difference between a new user and one with no devices',
-        function(done) {
-            aliHttpBackend.when('POST', '/keys/query').respond(200, {
-                device_keys: {
-                    '@bob:id': {},
-                },
-            });
-
-            const p1 = aliClient.downloadKeys(['@bob:id']);
-            const p2 = aliHttpBackend.flush('/keys/query', 1);
-
-            q.all([p1, p2]).then(function() {
-                const devices = aliStorage.getEndToEndDevicesForUser('@bob:id');
-                expect(utils.keys(devices).length).toEqual(0);
-
-                // request again: should be no more requests
-                return aliClient.downloadKeys(['@bob:id']);
-            }).nodeify(done);
-        },
-    );
-
-    it("Bob uploads without one-time keys and with one-time keys", function(done) {
-        q()
-            .then(bobUploadsKeys)
-            .catch(testUtils.failTest).done(done);
+    it("Bob uploads device keys", function() {
+        return q()
+            .then(bobUploadsDeviceKeys);
     });
 
-    it("Ali downloads Bobs keys", function(done) {
+    it("Ali downloads Bobs device keys", function(done) {
         q()
-            .then(bobUploadsKeys)
+            .then(bobUploadsDeviceKeys)
             .then(aliDownloadsKeys)
             .catch(testUtils.failTest).done(done);
     });
 
     it("Ali gets keys with an invalid signature", function(done) {
         q()
-            .then(bobUploadsKeys)
+            .then(bobUploadsDeviceKeys)
             .then(function() {
-                // tamper bob's keys!
-                expect(bobDeviceKeys.keys["curve25519:" + bobDeviceId]).toBeDefined();
+                // tamper bob's keys
+                const bobDeviceKeys = bobTestClient.deviceKeys;
+                expect(bobDeviceKeys.keys["curve25519:" + bobDeviceId]).toBeTruthy();
                 bobDeviceKeys.keys["curve25519:" + bobDeviceId] += "abc";
 
-                return q.all(aliClient.downloadKeys([bobUserId]),
+                return q.all(aliTestClient.client.downloadKeys([bobUserId]),
                              expectAliQueryKeys());
             })
             .then(function() {
                 // should get an empty list
-                expect(aliClient.listDeviceKeys(bobUserId)).toEqual([]);
+                expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([]);
             })
             .catch(testUtils.failTest).done(done);
     });
@@ -575,19 +455,21 @@ describe("MatrixClient crypto", function() {
 
         const bobKeys = {};
         bobKeys[bobDeviceId] = bobDeviceKeys;
-        aliHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
+        aliTestClient.httpBackend.when(
+            "POST", "/keys/query",
+        ).respond(200, function(path, content) {
             const result = {};
             result[bobUserId] = bobKeys;
             return {device_keys: result};
         });
 
         q.all(
-            aliClient.downloadKeys([bobUserId, eveUserId]),
-            aliHttpBackend.flush("/keys/query", 1),
+            aliTestClient.client.downloadKeys([bobUserId, eveUserId]),
+            aliTestClient.httpBackend.flush("/keys/query", 1),
         ).then(function() {
             // should get an empty list
-            expect(aliClient.listDeviceKeys(bobUserId)).toEqual([]);
-            expect(aliClient.listDeviceKeys(eveUserId)).toEqual([]);
+            expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([]);
+            expect(aliTestClient.client.listDeviceKeys(eveUserId)).toEqual([]);
         }).catch(testUtils.failTest).done(done);
     });
 
@@ -610,33 +492,39 @@ describe("MatrixClient crypto", function() {
 
         const bobKeys = {};
         bobKeys[bobDeviceId] = bobDeviceKeys;
-        aliHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
+        aliTestClient.httpBackend.when(
+            "POST", "/keys/query",
+        ).respond(200, function(path, content) {
             const result = {};
             result[bobUserId] = bobKeys;
             return {device_keys: result};
         });
 
         q.all(
-            aliClient.downloadKeys([bobUserId]),
-            aliHttpBackend.flush("/keys/query", 1),
+            aliTestClient.client.downloadKeys([bobUserId]),
+            aliTestClient.httpBackend.flush("/keys/query", 1),
         ).then(function() {
             // should get an empty list
-            expect(aliClient.listDeviceKeys(bobUserId)).toEqual([]);
+            expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([]);
         }).catch(testUtils.failTest).done(done);
     });
 
-    it("Ali enables encryption", function(done) {
-        q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
-            .then(aliEnablesEncryption)
-            .catch(testUtils.failTest).done(done);
+
+    it("Bob starts his client and uploads device keys and one-time keys", function() {
+        return q()
+            .then(() => bobTestClient.start())
+            .then(() => bobTestClient.awaitOneTimeKeyUpload())
+            .then((keys) => {
+                expect(Object.keys(keys).length).toEqual(5);
+                expect(Object.keys(bobTestClient.deviceKeys).length).toNotEqual(0);
+            });
     });
 
     it("Ali sends a message", function(done) {
         q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+            .then(() => aliTestClient.start())
+            .then(() => bobTestClient.start())
+            .then(() => firstSync(aliTestClient))
             .then(aliEnablesEncryption)
             .then(aliSendsFirstMessage)
             .catch(testUtils.failTest).nodeify(done);
@@ -644,22 +532,22 @@ describe("MatrixClient crypto", function() {
 
     it("Bob receives a message", function(done) {
         q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+            .then(() => aliTestClient.start())
+            .then(() => bobTestClient.start())
+            .then(() => firstSync(aliTestClient))
             .then(aliEnablesEncryption)
             .then(aliSendsFirstMessage)
-            .then(bobStartClient)
             .then(bobRecvMessage)
             .catch(testUtils.failTest).done(done);
     });
 
     it("Bob receives a message with a bogus sender", function(done) {
         q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+            .then(() => aliTestClient.start())
+            .then(() => bobTestClient.start())
+            .then(() => firstSync(aliTestClient))
             .then(aliEnablesEncryption)
             .then(aliSendsFirstMessage)
-            .then(bobStartClient)
             .then(function() {
                 const message = aliMessages.shift();
                 const syncData = {
@@ -682,11 +570,11 @@ describe("MatrixClient crypto", function() {
                         ],
                     },
                 };
-                bobHttpBackend.when("GET", "/sync").respond(200, syncData);
+                bobTestClient.httpBackend.when("GET", "/sync").respond(200, syncData);
 
                 const deferred = q.defer();
                 const onEvent = function(event) {
-                    console.log(bobClient.credentials.userId + " received event",
+                    console.log(bobUserId + " received event",
                                 event);
 
                     // ignore the m.room.member events
@@ -698,13 +586,13 @@ describe("MatrixClient crypto", function() {
                     expect(event.getContent().msgtype).toEqual("m.bad.encrypted");
                     expect(event.isEncrypted()).toBeTruthy();
 
-                    bobClient.removeListener("event", onEvent);
+                    bobTestClient.client.removeListener("event", onEvent);
                     deferred.resolve();
                 };
 
-                bobClient.on("event", onEvent);
+                bobTestClient.client.on("event", onEvent);
 
-                bobHttpBackend.flush();
+                bobTestClient.httpBackend.flush();
                 return deferred.promise;
             })
             .catch(testUtils.failTest).done(done);
@@ -712,58 +600,77 @@ describe("MatrixClient crypto", function() {
 
     it("Ali blocks Bob's device", function(done) {
         q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+            .then(() => aliTestClient.start())
+            .then(() => bobTestClient.start())
+            .then(() => firstSync(aliTestClient))
             .then(aliEnablesEncryption)
             .then(aliDownloadsKeys)
             .then(function() {
-                aliClient.setDeviceBlocked(bobUserId, bobDeviceId, true);
-                const p1 = sendMessage(aliClient);
-                const p2 = expectAliQueryKeys()
-                    .then(expectAliClaimKeys)
-                    .then(function() {
-                        return expectSendMessageRequest(aliHttpBackend);
-                    }).then(function(sentContent) {
-                        // no unblocked devices, so the ciphertext should be empty
-                        expect(sentContent.ciphertext).toEqual({});
-                    });
+                aliTestClient.client.setDeviceBlocked(bobUserId, bobDeviceId, true);
+                const p1 = sendMessage(aliTestClient.client);
+                const p2 = expectSendMessageRequest(aliTestClient.httpBackend)
+                      .then(function(sentContent) {
+                          // no unblocked devices, so the ciphertext should be empty
+                          expect(sentContent.ciphertext).toEqual({});
+                      });
                 return q.all([p1, p2]);
-            }).catch(testUtils.failTest).nodeify(done);
+            }).nodeify(done);
     });
 
     it("Bob receives two pre-key messages", function(done) {
         q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+            .then(() => aliTestClient.start())
+            .then(() => bobTestClient.start())
+            .then(() => firstSync(aliTestClient))
             .then(aliEnablesEncryption)
             .then(aliSendsFirstMessage)
-            .then(bobStartClient)
             .then(bobRecvMessage)
             .then(aliSendsMessage)
             .then(bobRecvMessage)
             .catch(testUtils.failTest).done(done);
     });
 
-    it("Bob replies to the message", function(done) {
-        q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+    it("Bob replies to the message", function() {
+        return q()
+            .then(() => aliTestClient.start())
+            .then(() => bobTestClient.start())
+            .then(() => firstSync(aliTestClient))
+            .then(() => firstSync(bobTestClient))
             .then(aliEnablesEncryption)
             .then(aliSendsFirstMessage)
-            .then(bobStartClient)
             .then(bobRecvMessage)
             .then(bobEnablesEncryption)
             .then(bobSendsReplyMessage).then(function(ciphertext) {
                 expect(ciphertext.type).toEqual(1);
-            }).then(aliRecvMessage)
-            .catch(testUtils.failTest).done(done);
+            }).then(aliRecvMessage);
     });
 
 
-    it("Ali does a key query when she gets a new_device event", function(done) {
-        q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+    it("Ali does a key query when she gets a new_device event", function() {
+        return q()
+            .then(() => aliTestClient.start())
+            .then(() => firstSync(aliTestClient))
+
+            // ali will only care about bob's new_device if she is tracking
+            // bob's devices, which she will do if we enable encryption
+            .then(aliEnablesEncryption)
+
+            .then(() => {
+                aliTestClient.expectKeyQuery({
+                    device_keys: {
+                        [aliUserId]: {},
+                        [bobUserId]: {},
+                    },
+                });
+                return aliTestClient.httpBackend.flush('/keys/query', 1);
+            })
+
+            // make sure that the initial key download has completed
+            // (downloadKeys will wait until it does)
+            .then(() => {
+                return aliTestClient.client.downloadKeys([bobUserId]);
+            })
+
             .then(function() {
                 const syncData = {
                     next_batch: '2',
@@ -780,9 +687,56 @@ describe("MatrixClient crypto", function() {
                         ],
                     },
                 };
-                aliHttpBackend.when('GET', '/sync').respond(200, syncData);
-                return aliHttpBackend.flush('/sync', 1);
-            }).then(expectAliQueryKeys)
-            .nodeify(done);
+                aliTestClient.httpBackend.when('GET', '/sync').respond(200, syncData);
+                return aliTestClient.httpBackend.flush('/sync', 1);
+            }).then(() => {
+                aliTestClient.expectKeyQuery({
+                    device_keys: {
+                        [bobUserId]: {},
+                    },
+                });
+                return aliTestClient.httpBackend.flush('/keys/query', 1);
+            });
+    });
+
+    it("Ali does a key query when encryption is enabled", function() {
+        // enabling encryption in the room should make alice download devices
+        // for both members.
+        return q()
+            .then(() => aliTestClient.start())
+            .then(() => firstSync(aliTestClient))
+            .then(() => {
+                const syncData = {
+                    next_batch: '2',
+                    rooms: {
+                        join: {},
+                    },
+                };
+                syncData.rooms.join[roomId] = {
+                    state: {
+                        events: [
+                            testUtils.mkEvent({
+                                type: 'm.room.encryption',
+                                skey: '',
+                                content: {
+                                    algorithm: 'm.olm.v1.curve25519-aes-sha2',
+                                },
+                            }),
+                        ],
+                    },
+                };
+
+                aliTestClient.httpBackend.when('GET', '/sync').respond(
+                    200, syncData);
+                return aliTestClient.httpBackend.flush('/sync', 1);
+            }).then(() => {
+                aliTestClient.expectKeyQuery({
+                    device_keys: {
+                        [aliUserId]: {},
+                        [bobUserId]: {},
+                    },
+                });
+                return aliTestClient.httpBackend.flush('/keys/query', 1);
+            });
     });
 });

spec/integ/matrix-client-event-emitter.spec.js

@@ -1,8 +1,11 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const HttpBackend = require("../mock-request");
 const utils = require("../test-utils");
 
+import expect from 'expect';
+
 describe("MatrixClient events", function() {
     const baseUrl = "http://localhost.or.something";
     let client;
@@ -141,8 +144,8 @@ describe("MatrixClient events", function() {
             let fired = false;
             client.on("User.presence", function(event, user) {
                 fired = true;
-                expect(user).toBeDefined();
-                expect(event).toBeDefined();
+                expect(user).toBeTruthy();
+                expect(event).toBeTruthy();
                 if (!user || !event) {
                     return;
                 }
@@ -207,7 +210,7 @@ describe("MatrixClient events", function() {
             client.on("RoomState.events", function(event, state) {
                 eventsInvokeCount++;
                 const index = roomStateEventTypes.indexOf(event.getType());
-                expect(index).not.toEqual(
+                expect(index).toNotEqual(
                     -1, "Unexpected room state event type: " + event.getType(),
                 );
                 if (index >= 0) {

spec/integ/matrix-client-event-timeline.spec.js

@@ -1,4 +1,5 @@
 "use strict";
+import 'source-map-support/register';
 const q = require("q");
 const sdk = require("../..");
 const HttpBackend = require("../mock-request");
@@ -94,7 +95,6 @@ function startClient(httpBackend, client) {
     return deferred.promise;
 }
 
-
 describe("getEventTimeline support", function() {
     let httpBackend;
     let client;
@@ -142,7 +142,7 @@ describe("getEventTimeline support", function() {
             const timelineSet = room.getTimelineSets()[0];
             expect(function() {
                 client.getEventTimeline(timelineSet, "event");
-            }).not.toThrow();
+            }).toNotThrow();
         }).catch(utils.failTest).done(done);
 
         httpBackend.flush().catch(utils.failTest);
@@ -219,6 +219,8 @@ describe("getEventTimeline support", function() {
     });
 });
 
+import expect from 'expect';
+
 describe("MatrixClient event timelines", function() {
     let client = null;
     let httpBackend = null;
@@ -310,7 +312,7 @@ describe("MatrixClient event timelines", function() {
             httpBackend.flush().catch(utils.failTest);
         });
 
-        it("should update timelines where they overlap a previous /sync", function(done) {
+        it("should update timelines where they overlap a previous /sync", function() {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
             httpBackend.when("GET", "/sync").respond(200, {
@@ -342,6 +344,7 @@ describe("MatrixClient event timelines", function() {
                     };
                 });
 
+            const deferred = q.defer();
             client.on("sync", function() {
                 client.getEventTimeline(timelineSet, EVENTS[2].event_id,
                 ).then(function(tl) {
@@ -353,10 +356,14 @@ describe("MatrixClient event timelines", function() {
                         .toEqual("start_token");
                     // expect(tl.getPaginationToken(EventTimeline.FORWARDS))
                     //    .toEqual("s_5_4");
-                }).catch(utils.failTest).done(done);
+                }).done(() => deferred.resolve(),
+                        (e) => deferred.reject(e));
             });
 
-            httpBackend.flush().catch(utils.failTest);
+            return q.all([
+                httpBackend.flush(),
+                deferred.promise,
+            ]);
         });
 
         it("should join timelines where they overlap a previous /context",
@@ -631,6 +638,8 @@ describe("MatrixClient event timelines", function() {
 
                 // now let the sync complete, and check it again
                 return httpBackend.flush("/sync", 1);
+            }).then(function() {
+                return utils.syncPromise(client);
             }).then(function() {
                 return client.getEventTimeline(timelineSet, event.event_id);
             }).then(function(tl) {
@@ -658,6 +667,8 @@ describe("MatrixClient event timelines", function() {
             }).catch(utils.failTest).done(done);
 
             httpBackend.flush("/sync", 1).then(function() {
+                return utils.syncPromise(client);
+            }).then(function() {
                 return client.getEventTimeline(timelineSet, event.event_id);
             }).then(function(tl) {
                 console.log("getEventTimeline completed (1)");

spec/integ/matrix-client-methods.spec.js

@@ -1,4 +1,5 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const HttpBackend = require("../mock-request");
 const publicGlobals = require("../../lib/matrix");
@@ -8,6 +9,8 @@ const Filter = publicGlobals.Filter;
 const utils = require("../test-utils");
 const MockStorageApi = require("../MockStorageApi");
 
+import expect from 'expect';
+
 describe("MatrixClient", function() {
     const baseUrl = "http://localhost.or.something";
     let client = null;
@@ -60,7 +63,7 @@ describe("MatrixClient", function() {
                 type: "text/plain",
             });
 
-            expect(prom).toBeDefined();
+            expect(prom).toBeTruthy();
 
             const uploads = client.getCurrentUploads();
             expect(uploads.length).toEqual(1);
@@ -208,14 +211,14 @@ describe("MatrixClient", function() {
             const httpFilterDefinition = {
                 event_format: "federation",
             };
-            expect(store.getFilter(userId, filterId)).toBeNull();
+            expect(store.getFilter(userId, filterId)).toBe(null);
 
             httpBackend.when(
                 "GET", "/user/" + encodeURIComponent(userId) + "/filter/" + filterId,
             ).respond(200, httpFilterDefinition);
             client.getFilter(userId, filterId, true).done(function(gotFilter) {
                 expect(gotFilter.getDefinition()).toEqual(httpFilterDefinition);
-                expect(store.getFilter(userId, filterId)).toBeDefined();
+                expect(store.getFilter(userId, filterId)).toBeTruthy();
                 done();
             });
 
@@ -227,7 +230,7 @@ describe("MatrixClient", function() {
         const filterId = "f1llllllerid";
 
         it("should do an HTTP request and then store the filter", function(done) {
-            expect(store.getFilter(userId, filterId)).toBeNull();
+            expect(store.getFilter(userId, filterId)).toBe(null);
 
             const filterDefinition = {
                 event_format: "client",
@@ -296,6 +299,10 @@ describe("MatrixClient", function() {
 
 
     describe("downloadKeys", function() {
+        if (!sdk.CRYPTO_ENABLED) {
+            return;
+        }
+
         it("should do an HTTP request and then store the keys", function(done) {
             const ed25519key = "7wG2lzAqbjcyEkOP7O4gU7ItYcn+chKzh5sT/5r2l78";
             // ed25519key = client.getDeviceEd25519Key();
@@ -348,7 +355,6 @@ describe("MatrixClient", function() {
 
             httpBackend.when("POST", "/keys/query").check(function(req) {
                 expect(req.data).toEqual({device_keys: {
-                    '@alice:localhost': {},
                     'boris': {},
                     'chaz': {},
                 }});

spec/integ/matrix-client-opts.spec.js

@@ -1,9 +1,12 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const MatrixClient = sdk.MatrixClient;
 const HttpBackend = require("../mock-request");
 const utils = require("../test-utils");
 
+import expect from 'expect';
+
 describe("MatrixClient opts", function() {
     const baseUrl = "http://localhost.or.something";
     let client = null;
@@ -96,7 +99,7 @@ describe("MatrixClient opts", function() {
                 "m.room.create",
             ];
             client.on("event", function(event) {
-                expect(expectedEventTypes.indexOf(event.getType())).not.toEqual(
+                expect(expectedEventTypes.indexOf(event.getType())).toNotEqual(
                     -1, "Recv unexpected event type: " + event.getType(),
                 );
                 expectedEventTypes.splice(
@@ -111,6 +114,8 @@ describe("MatrixClient opts", function() {
                 return httpBackend.flush("/filter", 1);
             }).then(function() {
                 return httpBackend.flush("/sync", 1);
+            }).then(function() {
+                return utils.syncPromise(client);
             }).done(function() {
                 expect(expectedEventTypes.length).toEqual(
                     0, "Expected to see event types: " + expectedEventTypes,

spec/integ/matrix-client-retrying.spec.js

@@ -1,9 +1,12 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const HttpBackend = require("../mock-request");
 const utils = require("../test-utils");
 const EventStatus = sdk.EventStatus;
 
+import expect from 'expect';
+
 describe("MatrixClient retrying", function() {
     const baseUrl = "http://localhost.or.something";
     let client = null;

spec/integ/matrix-client-room-timeline.spec.js

@@ -1,9 +1,12 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const EventStatus = sdk.EventStatus;
 const HttpBackend = require("../mock-request");
 const utils = require("../test-utils");
 
+import expect from 'expect';
+
 describe("MatrixClient room timelines", function() {
     const baseUrl = "http://localhost.or.something";
     let client = null;
@@ -120,7 +123,7 @@ describe("MatrixClient room timelines", function() {
         client.startClient();
         httpBackend.flush("/pushrules").then(function() {
             return httpBackend.flush("/filter");
-        }).done(done);
+        }).nodeify(done);
     });
 
     afterEach(function() {
@@ -247,7 +250,7 @@ describe("MatrixClient room timelines", function() {
 
                 client.scrollback(room).done(function() {
                     expect(room.timeline.length).toEqual(1);
-                    expect(room.oldState.paginationToken).toBeNull();
+                    expect(room.oldState.paginationToken).toBe(null);
                     done();
                 });
 
@@ -367,7 +370,7 @@ describe("MatrixClient room timelines", function() {
                     return;
                 }
                 const room = client.getRoom(roomId);
-                expect(room.oldState.paginationToken).toBeDefined();
+                expect(room.oldState.paginationToken).toBeTruthy();
 
                 client.scrollback(room, 1).done(function() {
                     expect(room.oldState.paginationToken).toEqual(sbEndTok);
@@ -406,6 +409,8 @@ describe("MatrixClient room timelines", function() {
 
                 httpBackend.flush("/messages", 1);
                 httpBackend.flush("/sync", 1).then(function() {
+                    return utils.syncPromise(client);
+                }).then(function() {
                     expect(index).toEqual(2);
                     expect(room.timeline.length).toEqual(3);
                     expect(room.timeline[2].event).toEqual(
@@ -436,6 +441,8 @@ describe("MatrixClient room timelines", function() {
                 }
                 const room = client.getRoom(roomId);
                 httpBackend.flush("/sync", 1).then(function() {
+                    return utils.syncPromise(client);
+                }).then(function() {
                     const preNameEvent = room.timeline[room.timeline.length - 3];
                     const postNameEvent = room.timeline[room.timeline.length - 1];
                     expect(preNameEvent.sender.name).toEqual(userName);
@@ -464,7 +471,9 @@ describe("MatrixClient room timelines", function() {
                     nameEmitCount += 1;
                 });
 
-                httpBackend.flush("/sync", 1).done(function() {
+                httpBackend.flush("/sync", 1).then(() => {
+                    return utils.syncPromise(client);
+                }).done(function() {
                     expect(nameEmitCount).toEqual(1);
                     expect(room.name).toEqual("Room 2");
                     // do another round
@@ -476,7 +485,9 @@ describe("MatrixClient room timelines", function() {
                     thirdRoomNameEvent.__prev_event = secondRoomNameEvent;
                     setNextSyncData([thirdRoomNameEvent]);
                     httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
-                    httpBackend.flush("/sync", 1).done(function() {
+                    httpBackend.flush("/sync", 1).then(() => {
+                        return utils.syncPromise(client);
+                    }).done(function() {
                         expect(nameEmitCount).toEqual(2);
                         expect(room.name).toEqual("Room 3");
                         done();
@@ -507,6 +518,8 @@ describe("MatrixClient room timelines", function() {
                 }
                 const room = client.getRoom(roomId);
                 httpBackend.flush("/sync", 1).then(function() {
+                    return utils.syncPromise(client);
+                }).then(function() {
                     expect(room.currentState.getMembers().length).toEqual(4);
                     expect(room.currentState.getMember(userC).name).toEqual("C");
                     expect(room.currentState.getMember(userC).membership).toEqual(
@@ -537,7 +550,9 @@ describe("MatrixClient room timelines", function() {
                 const room = client.getRoom(roomId);
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1).done(function() {
+                httpBackend.flush("/sync", 1).then(function() {
+                    return utils.syncPromise(client);
+                }).done(function() {
                     expect(room.timeline.length).toEqual(1);
                     expect(room.timeline[0].event).toEqual(eventData[0]);
                     expect(room.currentState.getMembers().length).toEqual(2);
@@ -575,7 +590,9 @@ describe("MatrixClient room timelines", function() {
                 });
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1).done(function() {
+                httpBackend.flush("/sync", 1).then(function() {
+                    return utils.syncPromise(client);
+                }).done(function() {
                     expect(emitCount).toEqual(1);
                     done();
                 });

spec/integ/matrix-client-syncing.spec.js

@@ -1,10 +1,13 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const HttpBackend = require("../mock-request");
 const utils = require("../test-utils");
 const MatrixEvent = sdk.MatrixEvent;
 const EventTimeline = sdk.EventTimeline;
 
+import expect from 'expect';
+
 describe("MatrixClient syncing", function() {
     const baseUrl = "http://localhost.or.something";
     let client = null;
@@ -134,7 +137,7 @@ describe("MatrixClient syncing", function() {
                 expect(member.name).toEqual("The Boss");
                 expect(
                     member.getAvatarUrl("home.server.url", null, null, null, false),
-                ).toBeDefined();
+                ).toBeTruthy();
                 done();
             });
         });
@@ -211,7 +214,7 @@ describe("MatrixClient syncing", function() {
                 expect(member.name).toEqual(userC);
                 expect(
                     member.getAvatarUrl("home.server.url", null, null, null, false),
-                ).toBeNull();
+                ).toBe(null);
                 done();
             });
         });
@@ -405,10 +408,10 @@ describe("MatrixClient syncing", function() {
             httpBackend.flush().done(function() {
                 const room = client.getRoom(roomTwo);
                 let member = room.getMember(otherUserId);
-                expect(member).toBeDefined();
+                expect(member).toBeTruthy();
                 expect(member.typing).toEqual(true);
                 member = room.getMember(selfUserId);
-                expect(member).toBeDefined();
+                expect(member).toBeTruthy();
                 expect(member.typing).toEqual(false);
                 done();
             });

spec/integ/megolm.spec.js

@@ -16,6 +16,7 @@ limitations under the License.
 
 "use strict";
 
+import 'source-map-support/register';
 
 let Olm = null;
 try {
@@ -24,131 +25,33 @@ try {
 
 const anotherjson = require('another-json');
 const q = require('q');
+import expect from 'expect';
 
 const sdk = require('../..');
 const utils = require('../../lib/utils');
 const testUtils = require('../test-utils');
-const MockHttpBackend = require('../mock-request');
+const TestClient = require('../TestClient').default;
 
 const ROOM_ID = "!room:id";
 
-/**
- * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
- *
- * @constructor
- * @param {string} userId
- * @param {string} deviceId
- * @param {string} accessToken
- */
-function TestClient(userId, deviceId, accessToken) {
-    this.userId = userId;
-    this.deviceId = deviceId;
-
-    this.storage = new sdk.WebStorageSessionStore(new testUtils.MockStorageApi());
-    this.httpBackend = new MockHttpBackend();
-    this.client = sdk.createClient({
-        baseUrl: "http://test.server",
-        userId: userId,
-        accessToken: accessToken,
-        deviceId: deviceId,
-        sessionStore: this.storage,
-        request: this.httpBackend.requestFn,
-    });
-
-    this.deviceKeys = null;
-    this.oneTimeKeys = [];
-}
-
-/**
- * start the client, and wait for it to initialise.
- *
- * @param {object?} existingDevices  the list of our existing devices to return from
- *    the /query request. Defaults to empty device list
- * @return {Promise}
- */
-TestClient.prototype.start = function(existingDevices) {
-    const self = this;
-
-    this.httpBackend.when("GET", "/pushrules").respond(200, {});
-    this.httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
-
-    this.httpBackend.when('POST', '/keys/query').respond(200, function(path, content) {
-        expect(content.device_keys[self.userId]).toEqual({});
-        let res = existingDevices;
-        if (!res) {
-            res = { device_keys: {} };
-            res.device_keys[self.userId] = {};
-        }
-        return res;
-    });
-    this.httpBackend.when("POST", "/keys/upload").respond(200, function(path, content) {
-        expect(content.one_time_keys).not.toBeDefined();
-        expect(content.device_keys).toBeDefined();
-        self.deviceKeys = content.device_keys;
-        return {one_time_key_counts: {signed_curve25519: 0}};
-    });
-    this.httpBackend.when("POST", "/keys/upload").respond(200, function(path, content) {
-        expect(content.device_keys).not.toBeDefined();
-        expect(content.one_time_keys).toBeDefined();
-        expect(content.one_time_keys).not.toEqual({});
-        self.oneTimeKeys = content.one_time_keys;
-        return {one_time_key_counts: {
-            signed_curve25519: utils.keys(self.oneTimeKeys).length,
-        }};
-    });
-
-    this.client.startClient({
-        // set this so that we can get hold of failed events
-        pendingEventOrdering: 'detached',
-    });
-
-    return this.httpBackend.flush();
-};
-
-/**
- * stop the client
- */
-TestClient.prototype.stop = function() {
-    this.client.stopClient();
-};
-
-/**
- * get the uploaded curve25519 device key
- *
- * @return {string} base64 device key
- */
-TestClient.prototype.getDeviceKey = function() {
-    const keyId = 'curve25519:' + this.deviceId;
-    return this.deviceKeys.keys[keyId];
-};
-
-
-/**
- * get the uploaded ed25519 device key
- *
- * @return {string} base64 device key
- */
-TestClient.prototype.getSigningKey = function() {
-    const keyId = 'ed25519:' + this.deviceId;
-    return this.deviceKeys.keys[keyId];
-};
-
 /**
  * start an Olm session with a given recipient
  *
  * @param {Olm.Account} olmAccount
  * @param {TestClient} recipientTestClient
- * @return {Olm.Session}
+ * @return {Promise} promise for Olm.Session
  */
 function createOlmSession(olmAccount, recipientTestClient) {
-    const otkId = utils.keys(recipientTestClient.oneTimeKeys)[0];
-    const otk = recipientTestClient.oneTimeKeys[otkId];
+    return recipientTestClient.awaitOneTimeKeyUpload().then((keys) => {
+        const otkId = utils.keys(keys)[0];
+        const otk = keys[otkId];
 
-    const session = new Olm.Session();
-    session.create_outbound(
-        olmAccount, recipientTestClient.getDeviceKey(), otk.key,
-    );
-    return session;
+        const session = new Olm.Session();
+        session.create_outbound(
+            olmAccount, recipientTestClient.getDeviceKey(), otk.key,
+        );
+        return session;
+    });
 }
 
 /**
@@ -165,9 +68,9 @@ function createOlmSession(olmAccount, recipientTestClient) {
  * @return {object} event
  */
 function encryptOlmEvent(opts) {
-    expect(opts.senderKey).toBeDefined();
-    expect(opts.p2pSession).toBeDefined();
-    expect(opts.recipient).toBeDefined();
+    expect(opts.senderKey).toBeTruthy();
+    expect(opts.p2pSession).toBeTruthy();
+    expect(opts.recipient).toBeTruthy();
 
     const plaintext = {
         content: opts.plaincontent || {},
@@ -205,8 +108,8 @@ function encryptOlmEvent(opts) {
  * @return {object} event
  */
 function encryptMegolmEvent(opts) {
-    expect(opts.senderKey).toBeDefined();
-    expect(opts.groupSession).toBeDefined();
+    expect(opts.senderKey).toBeTruthy();
+    expect(opts.groupSession).toBeTruthy();
 
     const plaintext = opts.plaintext || {};
     if (!plaintext.content) {
@@ -219,7 +122,7 @@ function encryptMegolmEvent(opts) {
         plaintext.type = "m.room.message";
     }
     if (!plaintext.room_id) {
-        expect(opts.room_id).toBeDefined();
+        expect(opts.room_id).toBeTruthy();
         plaintext.room_id = opts.room_id;
     }
 
@@ -401,9 +304,9 @@ describe("megolm", function() {
     });
 
     it("Alice receives a megolm message", function(done) {
-        return aliceTestClient.start().then(function() {
-            const p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
-
+        return aliceTestClient.start().then(() => {
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((p2pSession) => {
             const groupSession = new Olm.OutboundGroupSession();
             groupSession.create();
 
@@ -441,6 +344,75 @@ describe("megolm", function() {
 
             aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
             return aliceTestClient.httpBackend.flush("/sync", 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
+        }).then(function() {
+            const room = aliceTestClient.client.getRoom(ROOM_ID);
+            const event = room.getLiveTimeline().getEvents()[0];
+            expect(event.getContent().body).toEqual('42');
+        }).nodeify(done);
+    });
+
+    it("Alice receives a megolm message before the session keys", function(done) {
+        // https://github.com/vector-im/riot-web/issues/2273
+        let roomKeyEncrypted;
+
+        return aliceTestClient.start().then(() => {
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((p2pSession) => {
+            const groupSession = new Olm.OutboundGroupSession();
+            groupSession.create();
+
+            // make the room_key event, but don't send it yet
+            roomKeyEncrypted = encryptGroupSessionKey({
+                senderKey: testSenderKey,
+                recipient: aliceTestClient,
+                p2pSession: p2pSession,
+                groupSession: groupSession,
+                room_id: ROOM_ID,
+            });
+
+            // encrypt a message with the group session
+            const messageEncrypted = encryptMegolmEvent({
+                senderKey: testSenderKey,
+                groupSession: groupSession,
+                room_id: ROOM_ID,
+            });
+
+            // Alice just gets the message event to start with
+            const syncResponse = {
+                next_batch: 1,
+                rooms: {
+                    join: {},
+                },
+            };
+            syncResponse.rooms.join[ROOM_ID] = {
+                timeline: {
+                    events: [messageEncrypted],
+                },
+            };
+
+            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
+            return aliceTestClient.httpBackend.flush("/sync", 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
+        }).then(function() {
+            const room = aliceTestClient.client.getRoom(ROOM_ID);
+            const event = room.getLiveTimeline().getEvents()[0];
+            expect(event.getContent().msgtype).toEqual('m.bad.encrypted');
+
+            // now she gets the room_key event
+            const syncResponse = {
+                next_batch: 2,
+                to_device: {
+                    events: [roomKeyEncrypted],
+                },
+            };
+
+            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
+            return aliceTestClient.httpBackend.flush("/sync", 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
         }).then(function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
             const event = room.getLiveTimeline().getEvents()[0];
@@ -449,9 +421,9 @@ describe("megolm", function() {
     });
 
     it("Alice gets a second room_key message", function(done) {
-        return aliceTestClient.start().then(function() {
-            const p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
-
+        return aliceTestClient.start().then(() => {
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((p2pSession) => {
             const groupSession = new Olm.OutboundGroupSession();
             groupSession.create();
 
@@ -509,6 +481,8 @@ describe("megolm", function() {
             aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse2);
 
             return aliceTestClient.httpBackend.flush("/sync", 2);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
         }).then(function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
             const event = room.getLiveTimeline().getEvents()[0];
@@ -519,11 +493,13 @@ describe("megolm", function() {
     it('Alice sends a megolm message', function(done) {
         let p2pSession;
 
-        return aliceTestClient.start().then(function() {
-            const syncResponse = getSyncResponse(['@bob:xyz']);
-
+        return aliceTestClient.start().then(() => {
             // establish an olm session with alice
-            p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((_p2pSession) => {
+            p2pSession = _p2pSession;
+
+            const syncResponse = getSyncResponse(['@bob:xyz']);
 
             const olmEvent = encryptOlmEvent({
                 senderKey: testSenderKey,
@@ -535,6 +511,8 @@ describe("megolm", function() {
 
             aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
             return aliceTestClient.httpBackend.flush('/sync', 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
         }).then(function() {
             // start out with the device unknown - the send should be rejected.
             aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
@@ -600,6 +578,8 @@ describe("megolm", function() {
             aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
 
             return aliceTestClient.httpBackend.flush('/sync', 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
         }).then(function() {
             console.log("Forcing alice to download our device keys");
 
@@ -631,11 +611,11 @@ describe("megolm", function() {
 
 
     it("We shouldn't attempt to send to blocked devices", function(done) {
-        return aliceTestClient.start().then(function() {
-            const syncResponse = getSyncResponse(['@bob:xyz']);
-
+        return aliceTestClient.start().then(() => {
             // establish an olm session with alice
-            const p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((p2pSession) => {
+            const syncResponse = getSyncResponse(['@bob:xyz']);
 
             const olmEvent = encryptOlmEvent({
                 senderKey: testSenderKey,
@@ -647,6 +627,8 @@ describe("megolm", function() {
             aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
 
             return aliceTestClient.httpBackend.flush('/sync', 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
         }).then(function() {
             console.log('Forcing alice to download our device keys');
 
@@ -680,11 +662,13 @@ describe("megolm", function() {
         let p2pSession;
         let megolmSessionId;
 
-        return aliceTestClient.start().then(function() {
-            const syncResponse = getSyncResponse(['@bob:xyz']);
-
+        return aliceTestClient.start().then(() => {
             // establish an olm session with alice
-            p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((_p2pSession) => {
+            p2pSession = _p2pSession;
+
+            const syncResponse = getSyncResponse(['@bob:xyz']);
 
             const olmEvent = encryptOlmEvent({
                 senderKey: testSenderKey,
@@ -696,6 +680,8 @@ describe("megolm", function() {
             aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
 
             return aliceTestClient.httpBackend.flush('/sync', 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
         }).then(function() {
             console.log("Fetching bob's devices and marking known");
 
@@ -749,7 +735,7 @@ describe("megolm", function() {
                 'PUT', '/send/',
             ).respond(200, function(path, content) {
                 console.log('/send:', content);
-                expect(content.session_id).not.toEqual(megolmSessionId);
+                expect(content.session_id).toNotEqual(megolmSessionId);
                 return {
                     event_id: '$event_id',
                 };
@@ -771,9 +757,7 @@ describe("megolm", function() {
         let inboundGroupSession;
         let decrypted;
 
-        return aliceTestClient.start(
-            getTestKeysQueryResponse(aliceTestClient.userId),
-        ).then(function() {
+        return aliceTestClient.start().then(function() {
             // an encrypted room with just alice
             const syncResponse = {
                 next_batch: 1,
@@ -800,8 +784,29 @@ describe("megolm", function() {
             };
             aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
 
+            // the completion of the first initialsync hould make Alice
+            // invalidate the device cache for all members in e2e rooms (ie,
+            // herself), and do a key query.
+            aliceTestClient.expectKeyQuery(
+                getTestKeysQueryResponse(aliceTestClient.userId),
+            );
+
             return aliceTestClient.httpBackend.flush();
         }).then(function() {
+            // start out with the device unknown - the send should be rejected.
+            return q.all([
+                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test').then(() => {
+                    throw new Error("sendTextMessage failed on an unknown device");
+                }, (e) => {
+                    expect(e.name).toEqual("UnknownDeviceError");
+                    expect(Object.keys(e.devices)).toEqual([aliceTestClient.userId]);
+                    expect(Object.keys(e.devices[aliceTestClient.userId])).
+                        toEqual(['DEVICE_ID']);
+                }),
+                aliceTestClient.httpBackend.flush(),
+            ]);
+        }).then(function() {
+            // mark the device as known, and resend.
             aliceTestClient.client.setDeviceKnown(aliceTestClient.userId, 'DEVICE_ID');
             aliceTestClient.httpBackend.when('POST', '/keys/claim').respond(
                 200, function(path, content) {
@@ -890,11 +895,11 @@ describe("megolm", function() {
             };
         });
 
-        return aliceTestClient.start().then(function() {
-            const syncResponse = getSyncResponse(['@bob:xyz']);
-
+        return aliceTestClient.start().then(() => {
             // establish an olm session with alice
-            p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((p2pSession) => {
+            const syncResponse = getSyncResponse(['@bob:xyz']);
 
             const olmEvent = encryptOlmEvent({
                 senderKey: testSenderKey,
@@ -906,6 +911,8 @@ describe("megolm", function() {
 
             aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
             return aliceTestClient.httpBackend.flush('/sync', 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
         }).then(function() {
             console.log('Forcing alice to download our device keys');
 
@@ -931,14 +938,136 @@ describe("megolm", function() {
     });
 
 
+    it("We should not get confused by out-of-order device query responses",
+       () => {
+           // https://github.com/vector-im/riot-web/issues/3126
+           return aliceTestClient.start().then(() => {
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                   200, getSyncResponse(['@bob:xyz', '@chris:abc']));
+               return aliceTestClient.httpBackend.flush('/sync', 1);
+           }).then(() => {
+               return testUtils.syncPromise(aliceTestClient.client);
+           }).then(() => {
+               // to make sure the initial device queries are flushed out, we
+               // attempt to send a message.
+
+               aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
+                   200, {
+                       device_keys: {
+                           '@bob:xyz': {},
+                           '@chris:abc': {},
+                       },
+                   },
+               );
+
+               aliceTestClient.httpBackend.when('PUT', '/send/').respond(
+                   200, {event_id: '$event1'});
+
+               return q.all([
+                   aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+                   aliceTestClient.httpBackend.flush('/keys/query', 1).then(
+                       () => aliceTestClient.httpBackend.flush('/send/', 1, 20),
+                   ),
+               ]);
+           }).then(() => {
+               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(1);
+
+               // invalidate bob's and chris's device lists in separate syncs
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                   next_batch: '2',
+                   device_lists: {
+                       changed: ['@bob:xyz'],
+                   },
+               });
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                   next_batch: '3',
+                   device_lists: {
+                       changed: ['@chris:abc'],
+                   },
+               });
+               return aliceTestClient.httpBackend.flush('/sync', 2);
+           }).then(() => {
+               return testUtils.syncPromise(aliceTestClient.client);
+           }).then(() => {
+               // check that we don't yet have a request for chris's devices.
+               aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                   device_keys: {
+                       '@chris:abc': {},
+                   },
+                   token: '3',
+               }).respond(200, {
+                   device_keys: {'@chris:abc': {}},
+               });
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(0);
+               const bobStat = aliceTestClient.storage
+                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
+               if (bobStat != 1 && bobStat != 2) {
+                   throw new Error('Unexpected status for bob: wanted 1 or 2, got ' +
+                                   bobStat);
+               }
+
+               const chrisStat = aliceTestClient.storage
+                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
+               if (chrisStat != 1 && chrisStat != 2) {
+                   throw new Error('Unexpected status for chris: wanted 1 or 2, got ' +
+                                   bobStat);
+               }
+
+               // now add an expectation for a query for bob's devices, and let
+               // it complete.
+               aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                   device_keys: {
+                       '@bob:xyz': {},
+                   },
+                   token: '2',
+               }).respond(200, {
+                   device_keys: {'@bob:xyz': {}},
+               });
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(1);
+
+               // wait for the client to stop processing the response
+               return aliceTestClient.client.downloadKeys(['@bob:xyz']);
+           }).then(() => {
+               const bobStat = aliceTestClient.storage
+                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
+               expect(bobStat).toEqual(3);
+               const chrisStat = aliceTestClient.storage
+                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
+               if (chrisStat != 1 && chrisStat != 2) {
+                   throw new Error('Unexpected status for chris: wanted 1 or 2, got ' +
+                                   bobStat);
+               }
+
+               // now let the query for chris's devices complete.
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(1);
+
+               // wait for the client to stop processing the response
+               return aliceTestClient.client.downloadKeys(['@chris:abc']);
+           }).then(() => {
+               const bobStat = aliceTestClient.storage
+                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
+               const chrisStat = aliceTestClient.storage
+                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
+
+               expect(bobStat).toEqual(3);
+               expect(chrisStat).toEqual(3);
+               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(3);
+           });
+       });
+
     it("Alice exports megolm keys and imports them to a new device", function(done) {
         let messageEncrypted;
 
         return aliceTestClient.start().then(() => {
-            const p2pSession = createOlmSession(
-                testOlmAccount, aliceTestClient,
-            );
-
+            // establish an olm session with alice
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((p2pSession) => {
             const groupSession = new Olm.OutboundGroupSession();
             groupSession.create();
 
@@ -976,6 +1105,8 @@ describe("megolm", function() {
 
             aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
             return aliceTestClient.httpBackend.flush("/sync", 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
         }).then(function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
             const event = room.getLiveTimeline().getEvents()[0];
@@ -1008,6 +1139,8 @@ describe("megolm", function() {
 
             aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
             return aliceTestClient.httpBackend.flush("/sync", 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
         }).then(function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
             const event = room.getLiveTimeline().getEvents()[0];

spec/mock-request.js

@@ -1,5 +1,6 @@
 "use strict";
 const q = require("q");
+import expect from 'expect';
 
 /**
  * Construct a mock HTTP backend, heavily inspired by Angular.js.
@@ -35,40 +36,48 @@ HttpBackend.prototype = {
      * Respond to all of the requests (flush the queue).
      * @param {string} path The path to flush (optional) default: all.
      * @param {integer} numToFlush The number of things to flush (optional), default: all.
-     * @return {Promise} resolved when there is nothing left to flush.
+     * @param {integer=} waitTime  The time (in ms) to wait for a request to happen.
+     *    default: 5
+     *
+     * @return {Promise} resolves when there is nothing left to flush, with the
+     *    number of requests flushed
      */
-    flush: function(path, numToFlush) {
+    flush: function(path, numToFlush, waitTime) {
         const defer = q.defer();
         const self = this;
         let flushed = 0;
         let triedWaiting = false;
+        if (waitTime === undefined) {
+            waitTime = 5;
+        }
         console.log(
-            "HTTP backend flushing... (path=%s  numToFlush=%s)", path, numToFlush,
+            "HTTP backend flushing... (path=%s numToFlush=%s waitTime=%s)",
+            path, numToFlush, waitTime,
         );
         const tryFlush = function() {
             // if there's more real requests and more expected requests, flush 'em.
             console.log(
-                "  trying to flush queue => reqs=%s expected=%s [%s]",
-                self.requests.length, self.expectedRequests.length, path,
+                "  trying to flush queue => reqs=[%s] expected=[%s]",
+                self.requests, self.expectedRequests.map((er) => er.path),
             );
             if (self._takeFromQueue(path)) {
                 // try again on the next tick.
-                console.log("  flushed. Trying for more. [%s]", path);
                 flushed += 1;
                 if (numToFlush && flushed === numToFlush) {
-                    console.log("  [%s] Flushed assigned amount: %s", path, numToFlush);
-                    defer.resolve();
+                    console.log("  Flushed assigned amount: %s", numToFlush);
+                    defer.resolve(flushed);
                 } else {
+                    console.log("  flushed. Trying for more.");
                     setTimeout(tryFlush, 0);
                 }
             } else if (flushed === 0 && !triedWaiting) {
                 // we may not have made the request yet, wait a generous amount of
                 // time before giving up.
-                setTimeout(tryFlush, 5);
+                setTimeout(tryFlush, waitTime);
                 triedWaiting = true;
             } else {
-                console.log("  no more flushes. [%s]", path);
-                defer.resolve();
+                console.log("  no more flushes.");
+                defer.resolve(flushed);
             }
         };
 

spec/test-utils.js

@@ -1,14 +1,37 @@
 "use strict";
+import expect from 'expect';
+import q from 'q';
+
 const sdk = require("..");
 const MatrixEvent = sdk.MatrixEvent;
 
+/**
+ * Return a promise that is resolved when the client next emits a
+ * SYNCING event.
+ * @param {Object} client The client
+ * @return {Promise} Resolves once the client has emitted a SYNCING event
+ */
+module.exports.syncPromise = function(client) {
+    const def = q.defer();
+    const cb = (state) => {
+        if (state == 'SYNCING') {
+            def.resolve();
+        } else {
+            client.once('sync', cb);
+        }
+    };
+    client.once('sync', cb);
+    return def.promise;
+};
+
 /**
  * Perform common actions before each test case, e.g. printing the test case
  * name to stdout.
- * @param {TestCase} testCase The test case that is about to be run.
+ * @param {Mocha.Context} context  The test context
  */
-module.exports.beforeEach = function(testCase) {
-    const desc = testCase.suite.description + " : " + testCase.description;
+module.exports.beforeEach = function(context) {
+    const desc = context.currentTest.fullTitle();
+
     console.log(desc);
     console.log(new Array(1 + desc.length).join("="));
 };
@@ -20,18 +43,18 @@ module.exports.beforeEach = function(testCase) {
  * @return {Object} An instantiated object with spied methods/properties.
  */
 module.exports.mock = function(constr, name) {
-    // By Tim Buschtöns
+    // Based on
     // http://eclipsesource.com/blogs/2014/03/27/mocks-in-jasmine-tests/
     const HelperConstr = new Function(); // jshint ignore:line
     HelperConstr.prototype = constr.prototype;
     const result = new HelperConstr();
-    result.jasmineToString = function() {
+    result.toString = function() {
         return "mock" + (name ? " of " + name : "");
     };
     for (const key in constr.prototype) { // eslint-disable-line guard-for-in
         try {
             if (constr.prototype[key] instanceof Function) {
-                result[key] = jasmine.createSpy((name || "mock") + '.' + key);
+                result[key] = expect.createSpy();
             }
         } catch (ex) {
             // Direct access to some non-function fields of DOM prototypes may

spec/unit/content-repo.spec.js

@@ -1,7 +1,10 @@
 "use strict";
+import 'source-map-support/register';
 const ContentRepo = require("../../lib/content-repo");
 const testUtils = require("../test-utils");
 
+import expect from 'expect';
+
 describe("ContentRepo", function() {
     const baseUrl = "https://my.home.server";
 

spec/unit/crypto.spec.js

@@ -1,13 +1,19 @@
 
 "use strict";
-const Crypto = require("../../lib/crypto");
+import 'source-map-support/register';
+
 const sdk = require("../..");
+let Crypto;
+if (sdk.CRYPTO_ENABLED) {
+    Crypto = require("../../lib/crypto");
+}
+
+import expect from 'expect';
 
 describe("Crypto", function() {
     if (!sdk.CRYPTO_ENABLED) {
         return;
     }
-
     it("Crypto exposes the correct olm library version", function() {
         expect(Crypto.getOlmVersion()[0]).toEqual(2);
     });

spec/unit/crypto/DeviceList.spec.js

@@ -0,0 +1,128 @@
+import DeviceList from '../../../lib/crypto/DeviceList';
+import MockStorageApi from '../../MockStorageApi';
+import WebStorageSessionStore from '../../../lib/store/session/webstorage';
+import testUtils from '../../test-utils';
+import utils from '../../../lib/utils';
+
+import expect from 'expect';
+import q from 'q';
+
+const signedDeviceList = {
+    "failures": {},
+    "device_keys": {
+        "@test1:sw1v.org": {
+            "HGKAWHRVJQ": {
+                "signatures": {
+                    "@test1:sw1v.org": {
+                        "ed25519:HGKAWHRVJQ":
+                            "8PB450fxKDn5s8IiRZ2N2t6MiueQYVRLHFEzqIi1eLdxx1w" +
+                            "XEPC1/1Uz9T4gwnKlMVAKkhB5hXQA/3kjaeLABw",
+                    },
+                },
+                "user_id": "@test1:sw1v.org",
+                "keys": {
+                    "ed25519:HGKAWHRVJQ":
+                        "0gI/T6C+mn1pjtvnnW2yB2l1IIBb/5ULlBXi/LXFSEQ",
+                    "curve25519:HGKAWHRVJQ":
+                        "mbIZED1dBsgIgkgzxDpxKkJmsr4hiWlGzQTvUnQe3RY",
+                },
+                "algorithms": [
+                    "m.olm.v1.curve25519-aes-sha2",
+                    "m.megolm.v1.aes-sha2",
+                ],
+                "device_id": "HGKAWHRVJQ",
+                "unsigned": {},
+            },
+        },
+    },
+};
+
+describe('DeviceList', function() {
+    let downloadSpy;
+    let sessionStore;
+
+    beforeEach(function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+
+        downloadSpy = expect.createSpy();
+        const mockStorage = new MockStorageApi();
+        sessionStore = new WebStorageSessionStore(mockStorage);
+    });
+
+    function createTestDeviceList() {
+        const baseApis = {
+            downloadKeysForUsers: downloadSpy,
+        };
+        const mockOlm = {
+            verifySignature: function(key, message, signature) {},
+        };
+        return new DeviceList(baseApis, sessionStore, mockOlm);
+    }
+
+    it("should successfully download and store device keys", function() {
+        const dl = createTestDeviceList();
+
+        dl.startTrackingDeviceList('@test1:sw1v.org');
+
+        const queryDefer1 = q.defer();
+        downloadSpy.andReturn(queryDefer1.promise);
+
+        const prom1 = dl.refreshOutdatedDeviceLists();
+        expect(downloadSpy).toHaveBeenCalledWith(['@test1:sw1v.org'], {});
+        queryDefer1.resolve(utils.deepCopy(signedDeviceList));
+
+        return prom1.then(() => {
+            const storedKeys = sessionStore.getEndToEndDevicesForUser('@test1:sw1v.org');
+            expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
+        });
+    });
+
+    it("should have an outdated devicelist on an invalidation while an " +
+       "update is in progress", function() {
+        const dl = createTestDeviceList();
+
+        dl.startTrackingDeviceList('@test1:sw1v.org');
+
+        const queryDefer1 = q.defer();
+        downloadSpy.andReturn(queryDefer1.promise);
+
+        const prom1 = dl.refreshOutdatedDeviceLists();
+        expect(downloadSpy).toHaveBeenCalledWith(['@test1:sw1v.org'], {});
+        downloadSpy.reset();
+
+        // outdated notif arrives while the request is in flight.
+        const queryDefer2 = q.defer();
+        downloadSpy.andReturn(queryDefer2.promise);
+
+        dl.invalidateUserDeviceList('@test1:sw1v.org');
+        dl.refreshOutdatedDeviceLists();
+
+        // the first request completes
+        queryDefer1.resolve({
+            device_keys: {
+                '@test1:sw1v.org': {},
+            },
+        });
+
+        return prom1.then(() => {
+            // uh-oh; user restarts before second request completes. The new instance
+            // should know we never got a complete device list.
+            console.log("Creating new devicelist to simulate app reload");
+            downloadSpy.reset();
+            const dl2 = createTestDeviceList();
+            const queryDefer3 = q.defer();
+            downloadSpy.andReturn(queryDefer3.promise);
+
+            const prom3 = dl2.refreshOutdatedDeviceLists();
+            expect(downloadSpy).toHaveBeenCalledWith(['@test1:sw1v.org'], {});
+
+            queryDefer3.resolve(utils.deepCopy(signedDeviceList));
+
+            // allow promise chain to complete
+            return prom3;
+        }).then(() => {
+            const storedKeys = sessionStore.getEndToEndDevicesForUser('@test1:sw1v.org');
+            expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
+        });
+    });
+});

spec/unit/event-timeline.spec.js

@@ -1,4 +1,5 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const EventTimeline = sdk.EventTimeline;
 const utils = require("../test-utils");
@@ -8,6 +9,8 @@ function mockRoomStates(timeline) {
     timeline._endState = utils.mock(sdk.RoomState, "endState");
 }
 
+import expect from 'expect';
+
 describe("EventTimeline", function() {
     const roomId = "!foo:bar";
     const userA = "@alice:bar";
@@ -75,7 +78,7 @@ describe("EventTimeline", function() {
 
             expect(function() {
                 timeline.initialiseState(state);
-            }).not.toThrow();
+            }).toNotThrow();
             timeline.addEvent(event, false);
             expect(function() {
                 timeline.initialiseState(state);
@@ -118,7 +121,7 @@ describe("EventTimeline", function() {
             const next = {b: "b"};
             expect(function() {
                 timeline.setNeighbouringTimeline(prev, EventTimeline.BACKWARDS);
-            }).not.toThrow();
+            }).toNotThrow();
             expect(timeline.getNeighbouringTimeline(EventTimeline.BACKWARDS))
                 .toBe(prev);
             expect(function() {
@@ -127,7 +130,7 @@ describe("EventTimeline", function() {
 
             expect(function() {
                 timeline.setNeighbouringTimeline(next, EventTimeline.FORWARDS);
-            }).not.toThrow();
+            }).toNotThrow();
             expect(timeline.getNeighbouringTimeline(EventTimeline.FORWARDS))
                 .toBe(next);
             expect(function() {
@@ -184,14 +187,14 @@ describe("EventTimeline", function() {
                 name: "Old Alice",
             };
             timeline.getState(EventTimeline.FORWARDS).getSentinelMember
-                .andCallFake(function(uid) {
+                .andCall(function(uid) {
                     if (uid === userA) {
                         return sentinel;
                     }
                     return null;
                 });
             timeline.getState(EventTimeline.BACKWARDS).getSentinelMember
-                .andCallFake(function(uid) {
+                .andCall(function(uid) {
                     if (uid === userA) {
                         return oldSentinel;
                     }
@@ -226,14 +229,14 @@ describe("EventTimeline", function() {
                 name: "Old Alice",
             };
             timeline.getState(EventTimeline.FORWARDS).getSentinelMember
-                .andCallFake(function(uid) {
+                .andCall(function(uid) {
                     if (uid === userA) {
                         return sentinel;
                     }
                     return null;
                 });
             timeline.getState(EventTimeline.BACKWARDS).getSentinelMember
-                .andCallFake(function(uid) {
+                .andCall(function(uid) {
                     if (uid === userA) {
                         return oldSentinel;
                     }
@@ -278,7 +281,7 @@ describe("EventTimeline", function() {
             expect(events[1].forwardLooking).toBe(true);
 
             expect(timeline.getState(EventTimeline.BACKWARDS).setStateEvents).
-                not.toHaveBeenCalled();
+                toNotHaveBeenCalled();
         });
 
 
@@ -308,7 +311,7 @@ describe("EventTimeline", function() {
             expect(events[1].forwardLooking).toBe(false);
 
             expect(timeline.getState(EventTimeline.FORWARDS).setStateEvents).
-                not.toHaveBeenCalled();
+                toNotHaveBeenCalled();
         });
     });
 
@@ -373,4 +376,3 @@ describe("EventTimeline", function() {
            });
     });
 });
-

spec/unit/filter.spec.js

@@ -1,8 +1,11 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const Filter = sdk.Filter;
 const utils = require("../test-utils");
 
+import expect from 'expect';
+
 describe("Filter", function() {
     const filterId = "f1lt3ring15g00d4ursoul";
     const userId = "@sir_arthur_david:humming.tiger";

spec/unit/interactive-auth.spec.js

@@ -15,6 +15,7 @@ limitations under the License.
 */
 "use strict";
 
+import 'source-map-support/register';
 const q = require("q");
 const sdk = require("../..");
 const utils = require("../test-utils");
@@ -22,18 +23,29 @@ const utils = require("../test-utils");
 const InteractiveAuth = sdk.InteractiveAuth;
 const MatrixError = sdk.MatrixError;
 
+import expect from 'expect';
+
+// Trivial client object to test interactive auth
+// (we do not need TestClient here)
+class FakeClient {
+    generateClientSecret() {
+        return "testcl1Ent5EcreT";
+    }
+}
+
 describe("InteractiveAuth", function() {
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
     });
 
     it("should start an auth stage and complete it", function(done) {
-        const doRequest = jasmine.createSpy('doRequest');
-        const startAuthStage = jasmine.createSpy('startAuthStage');
+        const doRequest = expect.createSpy();
+        const stateUpdated = expect.createSpy();
 
         const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
             doRequest: doRequest,
-            startAuthStage: startAuthStage,
+            stateUpdated: stateUpdated,
             authData: {
                 session: "sessionId",
                 flows: [
@@ -51,7 +63,8 @@ describe("InteractiveAuth", function() {
         });
 
         // first we expect a call here
-        startAuthStage.andCallFake(function(stage) {
+        stateUpdated.andCall(function(stage) {
+            console.log('aaaa');
             expect(stage).toEqual("logintype");
             ia.submitAuthDict({
                 type: "logintype",
@@ -61,7 +74,8 @@ describe("InteractiveAuth", function() {
 
         // .. which should trigger a call here
         const requestRes = {"a": "b"};
-        doRequest.andCallFake(function(authData) {
+        doRequest.andCall(function(authData) {
+            console.log('cccc');
             expect(authData).toEqual({
                 session: "sessionId",
                 type: "logintype",
@@ -73,26 +87,27 @@ describe("InteractiveAuth", function() {
         ia.attemptAuth().then(function(res) {
             expect(res).toBe(requestRes);
             expect(doRequest.calls.length).toEqual(1);
-            expect(startAuthStage.calls.length).toEqual(1);
+            expect(stateUpdated.calls.length).toEqual(1);
         }).catch(utils.failTest).done(done);
     });
 
     it("should make a request if no authdata is provided", function(done) {
-        const doRequest = jasmine.createSpy('doRequest');
-        const startAuthStage = jasmine.createSpy('startAuthStage');
+        const doRequest = expect.createSpy();
+        const stateUpdated = expect.createSpy();
 
         const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
+            stateUpdated: stateUpdated,
             doRequest: doRequest,
-            startAuthStage: startAuthStage,
         });
 
         expect(ia.getSessionId()).toBe(undefined);
         expect(ia.getStageParams("logintype")).toBe(undefined);
 
         // first we expect a call to doRequest
-        doRequest.andCallFake(function(authData) {
+        doRequest.andCall(function(authData) {
             console.log("request1", authData);
-            expect(authData).toBe(null);
+            expect(authData).toEqual({});
             const err = new MatrixError({
                 session: "sessionId",
                 flows: [
@@ -106,9 +121,9 @@ describe("InteractiveAuth", function() {
             throw err;
         });
 
-        // .. which should be followed by a call to startAuthStage
+        // .. which should be followed by a call to stateUpdated
         const requestRes = {"a": "b"};
-        startAuthStage.andCallFake(function(stage) {
+        stateUpdated.andCall(function(stage) {
             expect(stage).toEqual("logintype");
             expect(ia.getSessionId()).toEqual("sessionId");
             expect(ia.getStageParams("logintype")).toEqual({
@@ -116,7 +131,7 @@ describe("InteractiveAuth", function() {
             });
 
             // submitAuthDict should trigger another call to doRequest
-            doRequest.andCallFake(function(authData) {
+            doRequest.andCall(function(authData) {
                 console.log("request2", authData);
                 expect(authData).toEqual({
                     session: "sessionId",
@@ -135,7 +150,7 @@ describe("InteractiveAuth", function() {
         ia.attemptAuth().then(function(res) {
             expect(res).toBe(requestRes);
             expect(doRequest.calls.length).toEqual(2);
-            expect(startAuthStage.calls.length).toEqual(1);
+            expect(stateUpdated.calls.length).toEqual(1);
         }).catch(utils.failTest).done(done);
     });
 });

spec/unit/matrix-client.spec.js

@@ -1,9 +1,13 @@
 "use strict";
+import 'source-map-support/register';
 const q = require("q");
 const sdk = require("../..");
 const MatrixClient = sdk.MatrixClient;
 const utils = require("../test-utils");
 
+import expect from 'expect';
+import lolex from 'lolex';
+
 describe("MatrixClient", function() {
     const userId = "@alice:bar";
     const identityServerUrl = "https://identity.server";
@@ -11,6 +15,7 @@ describe("MatrixClient", function() {
     let client;
     let store;
     let scheduler;
+    let clock;
 
     const KEEP_ALIVE_PATH = "/_matrix/client/versions";
 
@@ -120,16 +125,19 @@ describe("MatrixClient", function() {
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        jasmine.Clock.useMock();
-        scheduler = jasmine.createSpyObj("scheduler", [
+        clock = lolex.install();
+        scheduler = [
             "getQueueForEvent", "queueEvent", "removeEventFromQueue",
             "setProcessFunction",
-        ]);
-        store = jasmine.createSpyObj("store", [
+        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
+        store = [
             "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
-            "setSyncToken", "storeEvents", "storeRoom", "storeUser",
+            "save", "setSyncToken", "storeEvents", "storeRoom", "storeUser",
             "getFilterIdByName", "setFilterIdByName", "getFilter", "storeFilter",
-        ]);
+            "getSyncAccumulator", "startup", "deleteAllData",
+        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
+        store.getSavedSync = expect.createSpy().andReturn(q(null));
+        store.setSyncData = expect.createSpy().andReturn(q(null));
         client = new MatrixClient({
             baseUrl: "https://my.home.server",
             idBaseUrl: identityServerUrl,
@@ -140,14 +148,14 @@ describe("MatrixClient", function() {
             userId: userId,
         });
         // FIXME: We shouldn't be yanking _http like this.
-        client._http = jasmine.createSpyObj("httpApi", [
+        client._http = [
             "authedRequest", "authedRequestWithPrefix", "getContentUri",
             "request", "requestWithPrefix", "uploadContent",
-        ]);
-        client._http.authedRequest.andCallFake(httpReq);
-        client._http.authedRequestWithPrefix.andCallFake(httpReq);
-        client._http.requestWithPrefix.andCallFake(httpReq);
-        client._http.request.andCallFake(httpReq);
+        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
+        client._http.authedRequest.andCall(httpReq);
+        client._http.authedRequestWithPrefix.andCall(httpReq);
+        client._http.requestWithPrefix.andCall(httpReq);
+        client._http.request.andCall(httpReq);
 
         // set reasonable working defaults
         acceptKeepalives = true;
@@ -159,15 +167,16 @@ describe("MatrixClient", function() {
     });
 
     afterEach(function() {
+        clock.uninstall();
         // need to re-stub the requests with NOPs because there are no guarantees
         // clients from previous tests will be GC'd before the next test. This
         // means they may call /events and then fail an expect() which will fail
         // a DIFFERENT test (pollution between tests!) - we return unresolved
         // promises to stop the client from continuing to run.
-        client._http.authedRequest.andCallFake(function() {
+        client._http.authedRequest.andCall(function() {
             return q.defer().promise;
         });
-        client._http.authedRequestWithPrefix.andCallFake(function() {
+        client._http.authedRequestWithPrefix.andCall(function() {
             return q.defer().promise;
         });
     });
@@ -194,7 +203,7 @@ describe("MatrixClient", function() {
 
     describe("getSyncState", function() {
         it("should return null if the client isn't started", function() {
-            expect(client.getSyncState()).toBeNull();
+            expect(client.getSyncState()).toBe(null);
         });
 
         it("should return the same sync state as emitted sync events", function(done) {
@@ -266,7 +275,7 @@ describe("MatrixClient", function() {
                 if (state === "ERROR" && httpLookups.length > 0) {
                     expect(httpLookups.length).toEqual(2);
                     expect(client.retryImmediately()).toBe(true);
-                    jasmine.Clock.tick(1);
+                    clock.tick(1);
                 } else if (state === "PREPARED" && httpLookups.length === 0) {
                     client.removeListener("sync", syncListener);
                     done();
@@ -292,9 +301,9 @@ describe("MatrixClient", function() {
                     expect(client.retryImmediately()).toBe(
                         true, "retryImmediately returned false",
                     );
-                    jasmine.Clock.tick(1);
+                    clock.tick(1);
                 } else if (state === "RECONNECTING" && httpLookups.length > 0) {
-                    jasmine.Clock.tick(10000);
+                    clock.tick(10000);
                 } else if (state === "SYNCING" && httpLookups.length === 0) {
                     client.removeListener("sync", syncListener);
                     done();
@@ -316,7 +325,7 @@ describe("MatrixClient", function() {
                 if (state === "ERROR" && httpLookups.length > 0) {
                     expect(httpLookups.length).toEqual(3);
                     expect(client.retryImmediately()).toBe(true);
-                    jasmine.Clock.tick(1);
+                    clock.tick(1);
                 } else if (state === "PREPARED" && httpLookups.length === 0) {
                     client.removeListener("sync", syncListener);
                     done();
@@ -347,7 +356,7 @@ describe("MatrixClient", function() {
                     done();
                 }
                 // standard retry time is 5 to 10 seconds
-                jasmine.Clock.tick(10000);
+                clock.tick(10000);
             };
         }
 

spec/unit/pushprocessor.spec.js

@@ -1,7 +1,10 @@
 "use strict";
+import 'source-map-support/register';
 const PushProcessor = require("../../lib/pushprocessor");
 const utils = require("../test-utils");
 
+import expect from 'expect';
+
 describe('NotificationService', function() {
     const testUserId = "@ali:matrix.org";
     const testDisplayName = "Alice M";

spec/unit/realtime-callbacks.spec.js

@@ -1,74 +1,75 @@
 "use strict";
 
+import 'source-map-support/register';
 const callbacks = require("../../lib/realtime-callbacks");
 const testUtils = require("../test-utils.js");
 
+import expect from 'expect';
+import lolex from 'lolex';
+
 describe("realtime-callbacks", function() {
-    const clock = jasmine.Clock;
-    let fakeDate;
+    let clock;
 
     function tick(millis) {
-        // make sure we tick the fakedate first, otherwise nothing will happen!
-        fakeDate += millis;
         clock.tick(millis);
     }
 
     beforeEach(function() {
         testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
-        clock.useMock();
-        fakeDate = Date.now();
-        callbacks.setNow(function() {
-            return fakeDate;
-        });
+        clock = lolex.install();
+        const fakeDate = clock.Date;
+        callbacks.setNow(fakeDate.now.bind(fakeDate));
     });
 
     afterEach(function() {
         callbacks.setNow();
+        clock.uninstall();
     });
 
     describe("setTimeout", function() {
         it("should call the callback after the timeout", function() {
-            const callback = jasmine.createSpy();
+            const callback = expect.createSpy();
             callbacks.setTimeout(callback, 100);
 
-            expect(callback).not.toHaveBeenCalled();
+            expect(callback).toNotHaveBeenCalled();
             tick(100);
             expect(callback).toHaveBeenCalled();
         });
 
 
         it("should default to a zero timeout", function() {
-            const callback = jasmine.createSpy();
+            const callback = expect.createSpy();
             callbacks.setTimeout(callback);
 
-            expect(callback).not.toHaveBeenCalled();
+            expect(callback).toNotHaveBeenCalled();
             tick(0);
             expect(callback).toHaveBeenCalled();
         });
 
         it("should pass any parameters to the callback", function() {
-            const callback = jasmine.createSpy();
+            const callback = expect.createSpy();
             callbacks.setTimeout(callback, 0, "a", "b", "c");
             tick(0);
             expect(callback).toHaveBeenCalledWith("a", "b", "c");
         });
 
         it("should set 'this' to the global object", function() {
-            const callback = jasmine.createSpy();
-            callback.andCallFake(function() {
+            let passed = false;
+            const callback = function() {
                 expect(this).toBe(global); // eslint-disable-line no-invalid-this
-                expect(this.console).toBeDefined(); // eslint-disable-line no-invalid-this
-            });
+                expect(this.console).toBeTruthy(); // eslint-disable-line no-invalid-this
+                passed = true;
+            };
             callbacks.setTimeout(callback);
             tick(0);
-            expect(callback).toHaveBeenCalled();
+            expect(passed).toBe(true);
         });
 
         it("should handle timeouts of several seconds", function() {
-            const callback = jasmine.createSpy();
+            const callback = expect.createSpy();
             callbacks.setTimeout(callback, 2000);
 
-            expect(callback).not.toHaveBeenCalled();
+            expect(callback).toNotHaveBeenCalled();
             for (let i = 0; i < 4; i++) {
                 tick(500);
             }
@@ -76,24 +77,24 @@ describe("realtime-callbacks", function() {
         });
 
         it("should call multiple callbacks in the right order", function() {
-            const callback1 = jasmine.createSpy("callback1");
-            const callback2 = jasmine.createSpy("callback2");
-            const callback3 = jasmine.createSpy("callback3");
+            const callback1 = expect.createSpy();
+            const callback2 = expect.createSpy();
+            const callback3 = expect.createSpy();
             callbacks.setTimeout(callback2, 200);
             callbacks.setTimeout(callback1, 100);
             callbacks.setTimeout(callback3, 300);
 
-            expect(callback1).not.toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
-            expect(callback3).not.toHaveBeenCalled();
+            expect(callback1).toNotHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
+            expect(callback3).toNotHaveBeenCalled();
             tick(100);
             expect(callback1).toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
-            expect(callback3).not.toHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
+            expect(callback3).toNotHaveBeenCalled();
             tick(100);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
-            expect(callback3).not.toHaveBeenCalled();
+            expect(callback3).toNotHaveBeenCalled();
             tick(100);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
@@ -101,51 +102,54 @@ describe("realtime-callbacks", function() {
         });
 
         it("should treat -ve timeouts the same as a zero timeout", function() {
-            const callback1 = jasmine.createSpy("callback1");
-            const callback2 = jasmine.createSpy("callback2");
+            const callback1 = expect.createSpy();
+            const callback2 = expect.createSpy();
 
             // check that cb1 is called before cb2
-            callback1.andCallFake(function() {
-                expect(callback2).not.toHaveBeenCalled();
+            callback1.andCall(function() {
+                expect(callback2).toNotHaveBeenCalled();
             });
 
             callbacks.setTimeout(callback1);
             callbacks.setTimeout(callback2, -100);
 
-            expect(callback1).not.toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback1).toNotHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
             tick(0);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
         });
 
         it("should not get confused by chained calls", function() {
-            const callback2 = jasmine.createSpy("callback2");
-            const callback1 = jasmine.createSpy("callback1");
-            callback1.andCallFake(function() {
+            const callback2 = expect.createSpy();
+            const callback1 = expect.createSpy();
+            callback1.andCall(function() {
                 callbacks.setTimeout(callback2, 0);
-                expect(callback2).not.toHaveBeenCalled();
+                expect(callback2).toNotHaveBeenCalled();
             });
 
             callbacks.setTimeout(callback1);
-            expect(callback1).not.toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback1).toNotHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
             tick(0);
             expect(callback1).toHaveBeenCalled();
+            // the fake timer won't actually run callbacks registered during
+            // one tick until the next tick.
+            tick(1);
             expect(callback2).toHaveBeenCalled();
         });
 
         it("should be immune to exceptions", function() {
-            const callback1 = jasmine.createSpy("callback1");
-            callback1.andCallFake(function() {
+            const callback1 = expect.createSpy();
+            callback1.andCall(function() {
                 throw new Error("prepare to die");
             });
-            const callback2 = jasmine.createSpy("callback2");
+            const callback2 = expect.createSpy();
             callbacks.setTimeout(callback1, 0);
             callbacks.setTimeout(callback2, 0);
 
-            expect(callback1).not.toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback1).toNotHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
             tick(0);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
@@ -154,16 +158,16 @@ describe("realtime-callbacks", function() {
 
     describe("cancelTimeout", function() {
         it("should cancel a pending timeout", function() {
-            const callback = jasmine.createSpy();
+            const callback = expect.createSpy();
             const k = callbacks.setTimeout(callback);
             callbacks.clearTimeout(k);
             tick(0);
-            expect(callback).not.toHaveBeenCalled();
+            expect(callback).toNotHaveBeenCalled();
         });
 
         it("should not affect sooner timeouts", function() {
-            const callback1 = jasmine.createSpy("callback1");
-            const callback2 = jasmine.createSpy("callback2");
+            const callback1 = expect.createSpy();
+            const callback2 = expect.createSpy();
 
             callbacks.setTimeout(callback1, 100);
             const k = callbacks.setTimeout(callback2, 200);
@@ -171,10 +175,10 @@ describe("realtime-callbacks", function() {
 
             tick(100);
             expect(callback1).toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
 
             tick(150);
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
         });
     });
 });

spec/unit/room-member.spec.js

@@ -1,8 +1,11 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const RoomMember = sdk.RoomMember;
 const utils = require("../test-utils");
 
+import expect from 'expect';
+
 describe("RoomMember", function() {
     const roomId = "!foo:bar";
     const userA = "@alice:bar";
@@ -33,7 +36,7 @@ describe("RoomMember", function() {
             const url = member.getAvatarUrl(hsUrl);
             // we don't care about how the mxc->http conversion is done, other
             // than it contains the mxc body.
-            expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
+            expect(url.indexOf("flibble/wibble")).toNotEqual(-1);
         });
 
         it("should return an identicon HTTP URL if allowDefault was set and there " +
@@ -243,9 +246,9 @@ describe("RoomMember", function() {
             member.setMembershipEvent(joinEvent);
             expect(member.name).toEqual("Alice"); // prefer displayname
             member.setMembershipEvent(joinEvent, roomState);
-            expect(member.name).not.toEqual("Alice"); // it should disambig.
+            expect(member.name).toNotEqual("Alice"); // it should disambig.
             // user_id should be there somewhere
-            expect(member.name.indexOf(userA)).not.toEqual(-1);
+            expect(member.name.indexOf(userA)).toNotEqual(-1);
         });
 
         it("should emit 'RoomMember.membership' if the membership changes", function() {

spec/unit/room-state.spec.js

@@ -1,9 +1,12 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const RoomState = sdk.RoomState;
 const RoomMember = sdk.RoomMember;
 const utils = require("../test-utils");
 
+import expect from 'expect';
+
 describe("RoomState", function() {
     const roomId = "!foo:bar";
     const userA = "@alice:bar";
@@ -43,8 +46,8 @@ describe("RoomState", function() {
             const members = state.getMembers();
             expect(members.length).toEqual(2);
             // ordering unimportant
-            expect([userA, userB].indexOf(members[0].userId)).not.toEqual(-1);
-            expect([userA, userB].indexOf(members[1].userId)).not.toEqual(-1);
+            expect([userA, userB].indexOf(members[0].userId)).toNotEqual(-1);
+            expect([userA, userB].indexOf(members[1].userId)).toNotEqual(-1);
         });
     });
 
@@ -54,7 +57,7 @@ describe("RoomState", function() {
         });
 
         it("should return a member if they exist", function() {
-            expect(state.getMember(userB)).toBeDefined();
+            expect(state.getMember(userB)).toBeTruthy();
         });
 
         it("should return a member which changes as state changes", function() {
@@ -114,8 +117,8 @@ describe("RoomState", function() {
             const events = state.getStateEvents("m.room.member");
             expect(events.length).toEqual(2);
             // ordering unimportant
-            expect([userA, userB].indexOf(events[0].getStateKey())).not.toEqual(-1);
-            expect([userA, userB].indexOf(events[1].getStateKey())).not.toEqual(-1);
+            expect([userA, userB].indexOf(events[0].getStateKey())).toNotEqual(-1);
+            expect([userA, userB].indexOf(events[1].getStateKey())).toNotEqual(-1);
         });
 
         it("should return a single MatrixEvent if a state_key was specified",
@@ -238,7 +241,7 @@ describe("RoomState", function() {
 
             // TODO: We do this because we don't DI the RoomMember constructor
             // so we can't inject a mock :/ so we have to infer.
-            expect(state.members[userC]).toBeDefined();
+            expect(state.members[userC]).toBeTruthy();
             expect(state.members[userC].powerLevel).toEqual(10);
         });
 
@@ -252,7 +255,7 @@ describe("RoomState", function() {
             });
             state.setStateEvents([memberEvent]);
 
-            expect(state.members[userA].setMembershipEvent).not.toHaveBeenCalled();
+            expect(state.members[userA].setMembershipEvent).toNotHaveBeenCalled();
             expect(state.members[userB].setMembershipEvent).toHaveBeenCalledWith(
                 memberEvent, state,
             );

spec/unit/room.spec.js

@@ -1,4 +1,5 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const Room = sdk.Room;
 const RoomState = sdk.RoomState;
@@ -7,6 +8,8 @@ const EventStatus = sdk.EventStatus;
 const EventTimeline = sdk.EventTimeline;
 const utils = require("../test-utils");
 
+import expect from 'expect';
+
 describe("Room", function() {
     const roomId = "!foo:bar";
     const userA = "@alice:bar";
@@ -29,7 +32,7 @@ describe("Room", function() {
         const hsUrl = "https://my.home.server";
 
         it("should return the URL from m.room.avatar preferentially", function() {
-            room.currentState.getStateEvents.andCallFake(function(type, key) {
+            room.currentState.getStateEvents.andCall(function(type, key) {
                 if (type === "m.room.avatar" && key === "") {
                     return utils.mkEvent({
                         event: true,
@@ -46,7 +49,7 @@ describe("Room", function() {
             const url = room.getAvatarUrl(hsUrl);
             // we don't care about how the mxc->http conversion is done, other
             // than it contains the mxc body.
-            expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
+            expect(url.indexOf("flibble/wibble")).toNotEqual(-1);
         });
 
         it("should return an identicon HTTP URL if allowDefault was set and there " +
@@ -78,7 +81,7 @@ describe("Room", function() {
         });
 
         it("should return the member from current state", function() {
-            expect(room.getMember(userA)).not.toEqual(null);
+            expect(room.getMember(userA)).toNotEqual(null);
         });
     });
 
@@ -170,7 +173,7 @@ describe("Room", function() {
             );
             expect(events[0].forwardLooking).toBe(true);
             expect(events[1].forwardLooking).toBe(true);
-            expect(room.oldState.setStateEvents).not.toHaveBeenCalled();
+            expect(room.oldState.setStateEvents).toNotHaveBeenCalled();
         });
 
         it("should synthesize read receipts for the senders of events", function() {
@@ -179,7 +182,7 @@ describe("Room", function() {
                 membership: "join",
                 name: "Alice",
             };
-            room.currentState.getSentinelMember.andCallFake(function(uid) {
+            room.currentState.getSentinelMember.andCall(function(uid) {
                 if (uid === userA) {
                     return sentinel;
                 }
@@ -288,13 +291,13 @@ describe("Room", function() {
                 membership: "join",
                 name: "Old Alice",
             };
-            room.currentState.getSentinelMember.andCallFake(function(uid) {
+            room.currentState.getSentinelMember.andCall(function(uid) {
                 if (uid === userA) {
                     return sentinel;
                 }
                 return null;
             });
-            room.oldState.getSentinelMember.andCallFake(function(uid) {
+            room.oldState.getSentinelMember.andCall(function(uid) {
                 if (uid === userA) {
                     return oldSentinel;
                 }
@@ -327,13 +330,13 @@ describe("Room", function() {
                 membership: "join",
                 name: "Old Alice",
             };
-            room.currentState.getSentinelMember.andCallFake(function(uid) {
+            room.currentState.getSentinelMember.andCall(function(uid) {
                 if (uid === userA) {
                     return sentinel;
                 }
                 return null;
             });
-            room.oldState.getSentinelMember.andCallFake(function(uid) {
+            room.oldState.getSentinelMember.andCall(function(uid) {
                 if (uid === userA) {
                     return oldSentinel;
                 }
@@ -375,7 +378,7 @@ describe("Room", function() {
             );
             expect(events[0].forwardLooking).toBe(false);
             expect(events[1].forwardLooking).toBe(false);
-            expect(room.currentState.setStateEvents).not.toHaveBeenCalled();
+            expect(room.currentState.setStateEvents).toNotHaveBeenCalled();
         });
     });
 
@@ -538,7 +541,7 @@ describe("Room", function() {
 
     describe("getJoinedMembers", function() {
         it("should return members whose membership is 'join'", function() {
-            room.currentState.getMembers.andCallFake(function() {
+            room.currentState.getMembers.andCall(function() {
                 return [
                     { userId: "@alice:bar", membership: "join" },
                     { userId: "@bob:bar", membership: "invite" },
@@ -551,7 +554,7 @@ describe("Room", function() {
         });
 
         it("should return an empty list if no membership is 'join'", function() {
-            room.currentState.getMembers.andCallFake(function() {
+            room.currentState.getMembers.andCall(function() {
                 return [
                     { userId: "@bob:bar", membership: "invite" },
                 ];
@@ -646,7 +649,7 @@ describe("Room", function() {
 
         beforeEach(function() {
             stateLookup = {};
-            room.currentState.getStateEvents.andCallFake(function(type, key) {
+            room.currentState.getStateEvents.andCall(function(type, key) {
                 if (key === undefined) {
                     const prefix = type + "$";
                     const list = [];
@@ -663,7 +666,7 @@ describe("Room", function() {
                     return stateLookup[type + "$" + key];
                 }
             });
-            room.currentState.getMembers.andCallFake(function() {
+            room.currentState.getMembers.andCall(function() {
                 const memberEvents = room.currentState.getStateEvents("m.room.member");
                 const members = [];
                 for (let i = 0; i < memberEvents.length; i++) {
@@ -678,7 +681,7 @@ describe("Room", function() {
                 }
                 return members;
             });
-            room.currentState.getMember.andCallFake(function(userId) {
+            room.currentState.getMember.andCall(function(userId) {
                 const memberEvent = room.currentState.getStateEvents(
                     "m.room.member", userId,
                 );
@@ -712,7 +715,8 @@ describe("Room", function() {
                 room.recalculate(userA);
                 expect(room.currentState.setStateEvents).toHaveBeenCalled();
                 // first call, first arg (which is an array), first element in array
-                const fakeEvent = room.currentState.setStateEvents.calls[0].args[0][0];
+                const fakeEvent = room.currentState.setStateEvents.calls[0].
+                      arguments[0][0];
                 expect(fakeEvent.getContent()).toEqual({
                     name: roomName,
                 });
@@ -731,7 +735,7 @@ describe("Room", function() {
                 ];
 
                 room.recalculate(userA);
-                expect(room.currentState.setStateEvents).not.toHaveBeenCalled();
+                expect(room.currentState.setStateEvents).toNotHaveBeenCalled();
             });
         });
 
@@ -767,8 +771,8 @@ describe("Room", function() {
                 addMember(userC);
                 room.recalculate(userA);
                 const name = room.name;
-                expect(name.indexOf(userB)).not.toEqual(-1, name);
-                expect(name.indexOf(userC)).not.toEqual(-1, name);
+                expect(name.indexOf(userB)).toNotEqual(-1, name);
+                expect(name.indexOf(userC)).toNotEqual(-1, name);
             });
 
             it("should return the names of members in a public (public join_rules)" +
@@ -780,8 +784,8 @@ describe("Room", function() {
                 addMember(userC);
                 room.recalculate(userA);
                 const name = room.name;
-                expect(name.indexOf(userB)).not.toEqual(-1, name);
-                expect(name.indexOf(userC)).not.toEqual(-1, name);
+                expect(name.indexOf(userB)).toNotEqual(-1, name);
+                expect(name.indexOf(userC)).toNotEqual(-1, name);
             });
 
             it("should show the other user's name for public (public join_rules)" +
@@ -792,7 +796,7 @@ describe("Room", function() {
                 addMember(userB);
                 room.recalculate(userA);
                 const name = room.name;
-                expect(name.indexOf(userB)).not.toEqual(-1, name);
+                expect(name.indexOf(userB)).toNotEqual(-1, name);
             });
 
             it("should show the other user's name for private " +
@@ -803,7 +807,7 @@ describe("Room", function() {
                 addMember(userB);
                 room.recalculate(userA);
                 const name = room.name;
-                expect(name.indexOf(userB)).not.toEqual(-1, name);
+                expect(name.indexOf(userB)).toNotEqual(-1, name);
             });
 
             it("should show the other user's name for private" +
@@ -813,7 +817,7 @@ describe("Room", function() {
                 addMember(userB);
                 room.recalculate(userA);
                 const name = room.name;
-                expect(name.indexOf(userB)).not.toEqual(-1, name);
+                expect(name.indexOf(userB)).toNotEqual(-1, name);
             });
 
             it("should show the room alias if one exists for private " +
@@ -959,7 +963,7 @@ describe("Room", function() {
 
             it("should emit an event when a receipt is added",
             function() {
-                const listener = jasmine.createSpy('spy');
+                const listener = expect.createSpy();
                 room.on("Room.receipt", listener);
 
                 const ts = 13787898424;
@@ -1130,7 +1134,7 @@ describe("Room", function() {
             it("should emit Room.tags event when new tags are " +
                "received on the event stream",
             function() {
-                const listener = jasmine.createSpy('spy');
+                const listener = expect.createSpy();
                 room.on("Room.tags", listener);
 
                 const tags = { "m.foo": { "order": 0.5 } };

spec/unit/scheduler.spec.js

@@ -1,13 +1,18 @@
 // This file had a function whose name is all caps, which displeases eslint
 /* eslint new-cap: "off" */
 
+import 'source-map-support/register';
 const q = require("q");
 const sdk = require("../..");
 const MatrixScheduler = sdk.MatrixScheduler;
 const MatrixError = sdk.MatrixError;
 const utils = require("../test-utils");
 
+import expect from 'expect';
+import lolex from 'lolex';
+
 describe("MatrixScheduler", function() {
+    let clock;
     let scheduler;
     let retryFn;
     let queueFn;
@@ -22,7 +27,7 @@ describe("MatrixScheduler", function() {
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        jasmine.Clock.useMock();
+        clock = lolex.install();
         scheduler = new MatrixScheduler(function(ev, attempts, err) {
             if (retryFn) {
                 return retryFn(ev, attempts, err);
@@ -39,6 +44,10 @@ describe("MatrixScheduler", function() {
         defer = q.defer();
     });
 
+    afterEach(function() {
+        clock.uninstall();
+    });
+
     it("should process events in a queue in a FIFO manner", function(done) {
         retryFn = function() {
             return 0;
@@ -98,7 +107,7 @@ describe("MatrixScheduler", function() {
         defer.reject({});
         retryDefer.promise.done(function() {
             expect(procCount).toEqual(1);
-            jasmine.Clock.tick(waitTimeMs);
+            clock.tick(waitTimeMs);
             expect(procCount).toEqual(2);
             done();
         });
@@ -186,7 +195,7 @@ describe("MatrixScheduler", function() {
         setTimeout(function() {
             deferA.resolve({});
         }, 1000);
-        jasmine.Clock.tick(1000);
+        clock.tick(1000);
     });
 
     describe("queueEvent", function() {
@@ -202,8 +211,8 @@ describe("MatrixScheduler", function() {
                 return "yep";
             };
             const prom = scheduler.queueEvent(eventA);
-            expect(prom).toBeDefined();
-            expect(prom.then).toBeDefined();
+            expect(prom).toBeTruthy();
+            expect(prom.then).toBeTruthy();
         });
     });
 
@@ -212,14 +221,14 @@ describe("MatrixScheduler", function() {
             queueFn = function() {
                 return null;
             };
-            expect(scheduler.getQueueForEvent(eventA)).toBeNull();
+            expect(scheduler.getQueueForEvent(eventA)).toBe(null);
         });
 
         it("should return null if the mapped queue doesn't exist", function() {
             queueFn = function() {
                 return "yep";
             };
-            expect(scheduler.getQueueForEvent(eventA)).toBeNull();
+            expect(scheduler.getQueueForEvent(eventA)).toBe(null);
         });
 
         it("should return a list of events in the queue and modifications to" +

spec/unit/sync-accumulator.spec.js

@@ -0,0 +1,356 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+"use strict";
+import 'source-map-support/register';
+import utils from "../test-utils";
+import sdk from "../..";
+import expect from 'expect';
+
+const SyncAccumulator = sdk.SyncAccumulator;
+
+describe("SyncAccumulator", function() {
+    let sa;
+
+    beforeEach(function() {
+        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        sa = new SyncAccumulator({
+            maxTimelineEntries: 10,
+        });
+    });
+
+    it("should return the same /sync response if accumulated exactly once", () => {
+        // technically cheating since we also cheekily pre-populate keys we
+        // know that the sync accumulator will pre-populate.
+        // It isn't 100% transitive.
+        const res = {
+            next_batch: "abc",
+            rooms: {
+                invite: {},
+                leave: {},
+                join: {
+                    "!foo:bar": {
+                        account_data: { events: [] },
+                        ephemeral: { events: [] },
+                        unread_notifications: {},
+                        state: {
+                            events: [
+                                member("alice", "join"),
+                                member("bob", "join"),
+                            ],
+                        },
+                        timeline: {
+                            events: [msg("alice", "hi")],
+                            prev_batch: "something",
+                        },
+                    },
+                },
+            },
+        };
+        sa.accumulate(res);
+        const output = sa.getJSON();
+        expect(output.nextBatch).toEqual(res.next_batch);
+        expect(output.roomsData).toEqual(res.rooms);
+    });
+
+    it("should prune the timeline to the oldest prev_batch within the limit", () => {
+        // maxTimelineEntries is 10 so we should get back all
+        // 10 timeline messages with a prev_batch of "pinned_to_1"
+        sa.accumulate(syncSkeleton({
+            state: { events: [member("alice", "join")] },
+            timeline: {
+                events: [
+                    msg("alice", "1"),
+                    msg("alice", "2"),
+                    msg("alice", "3"),
+                    msg("alice", "4"),
+                    msg("alice", "5"),
+                    msg("alice", "6"),
+                    msg("alice", "7"),
+                ],
+                prev_batch: "pinned_to_1",
+            },
+        }));
+        sa.accumulate(syncSkeleton({
+            state: { events: [] },
+            timeline: {
+                events: [
+                    msg("alice", "8"),
+                ],
+                prev_batch: "pinned_to_8",
+            },
+        }));
+        sa.accumulate(syncSkeleton({
+            state: { events: [] },
+            timeline: {
+                events: [
+                    msg("alice", "9"),
+                    msg("alice", "10"),
+                ],
+                prev_batch: "pinned_to_10",
+            },
+        }));
+
+        let output = sa.getJSON().roomsData.join["!foo:bar"];
+
+        expect(output.timeline.events.length).toEqual(10);
+        output.timeline.events.forEach((e, i) => {
+            expect(e.content.body).toEqual(""+(i+1));
+        });
+        expect(output.timeline.prev_batch).toEqual("pinned_to_1");
+
+        // accumulate more messages. Now it can't have a prev_batch of "pinned to 1"
+        // AND give us <= 10 messages without losing messages in-between.
+        // It should try to find the oldest prev_batch which still fits into 10
+        // messages, which is "pinned to 8".
+        sa.accumulate(syncSkeleton({
+            state: { events: [] },
+            timeline: {
+                events: [
+                    msg("alice", "11"),
+                    msg("alice", "12"),
+                    msg("alice", "13"),
+                    msg("alice", "14"),
+                    msg("alice", "15"),
+                    msg("alice", "16"),
+                    msg("alice", "17"),
+                ],
+                prev_batch: "pinned_to_11",
+            },
+        }));
+
+        output = sa.getJSON().roomsData.join["!foo:bar"];
+
+        expect(output.timeline.events.length).toEqual(10);
+        output.timeline.events.forEach((e, i) => {
+            expect(e.content.body).toEqual(""+(i+8));
+        });
+        expect(output.timeline.prev_batch).toEqual("pinned_to_8");
+    });
+
+    it("should remove the stored timeline on limited syncs", () => {
+        sa.accumulate(syncSkeleton({
+            state: { events: [member("alice", "join")] },
+            timeline: {
+                events: [
+                    msg("alice", "1"),
+                    msg("alice", "2"),
+                    msg("alice", "3"),
+                ],
+                prev_batch: "pinned_to_1",
+            },
+        }));
+        // some time passes and now we get a limited sync
+        sa.accumulate(syncSkeleton({
+            state: { events: [] },
+            timeline: {
+                limited: true,
+                events: [
+                    msg("alice", "51"),
+                    msg("alice", "52"),
+                    msg("alice", "53"),
+                ],
+                prev_batch: "pinned_to_51",
+            },
+        }));
+
+        const output = sa.getJSON().roomsData.join["!foo:bar"];
+
+        expect(output.timeline.events.length).toEqual(3);
+        output.timeline.events.forEach((e, i) => {
+            expect(e.content.body).toEqual(""+(i+51));
+        });
+        expect(output.timeline.prev_batch).toEqual("pinned_to_51");
+    });
+
+    it("should drop typing notifications", () => {
+        const res = syncSkeleton({
+            ephemeral: {
+                events: [{
+                    type: "m.typing",
+                    content: {
+                        user_ids: ["@alice:localhost"],
+                    },
+                    room_id: "!foo:bar",
+                }],
+            },
+        });
+        sa.accumulate(res);
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].ephemeral.events.length,
+        ).toEqual(0);
+    });
+
+    it("should clobber account data based on event type", () => {
+        const acc1 = {
+            type: "favourite.food",
+            content: {
+                food: "banana",
+            },
+        };
+        const acc2 = {
+            type: "favourite.food",
+            content: {
+                food: "apple",
+            },
+        };
+        sa.accumulate(syncSkeleton({
+            account_data: {
+                events: [acc1],
+            },
+        }));
+        sa.accumulate(syncSkeleton({
+            account_data: {
+                events: [acc2],
+            },
+        }));
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].account_data.events.length,
+        ).toEqual(1);
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].account_data.events[0],
+        ).toEqual(acc2);
+    });
+
+    it("should clobber global account data based on event type", () => {
+        const acc1 = {
+            type: "favourite.food",
+            content: {
+                food: "banana",
+            },
+        };
+        const acc2 = {
+            type: "favourite.food",
+            content: {
+                food: "apple",
+            },
+        };
+        sa.accumulate({
+            account_data: {
+                events: [acc1],
+            },
+        });
+        sa.accumulate({
+            account_data: {
+                events: [acc2],
+            },
+        });
+        expect(
+            sa.getJSON().accountData.length,
+        ).toEqual(1);
+        expect(
+            sa.getJSON().accountData[0],
+        ).toEqual(acc2);
+    });
+
+    it("should accumulate read receipts", () => {
+        const receipt1 = {
+            type: "m.receipt",
+            room_id: "!foo:bar",
+            content: {
+                "$event1:localhost": {
+                    "m.read": {
+                        "@alice:localhost": { ts: 1 },
+                        "@bob:localhost": { ts: 2 },
+                    },
+                    "some.other.receipt.type": {
+                        "@should_be_ignored:localhost": { key: "val" },
+                    },
+                },
+            },
+        };
+        const receipt2 = {
+            type: "m.receipt",
+            room_id: "!foo:bar",
+            content: {
+                "$event2:localhost": {
+                    "m.read": {
+                        "@bob:localhost": { ts: 2 }, // clobbers event1 receipt
+                        "@charlie:localhost": { ts: 3 },
+                    },
+                },
+            },
+        };
+        sa.accumulate(syncSkeleton({
+            ephemeral: {
+                events: [receipt1],
+            },
+        }));
+        sa.accumulate(syncSkeleton({
+            ephemeral: {
+                events: [receipt2],
+            },
+        }));
+
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].ephemeral.events.length,
+        ).toEqual(1);
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].ephemeral.events[0],
+        ).toEqual({
+            type: "m.receipt",
+            room_id: "!foo:bar",
+            content: {
+                "$event1:localhost": {
+                    "m.read": {
+                        "@alice:localhost": { ts: 1 },
+                    },
+                },
+                "$event2:localhost": {
+                    "m.read": {
+                        "@bob:localhost": { ts: 2 },
+                        "@charlie:localhost": { ts: 3 },
+                    },
+                },
+            },
+        });
+    });
+});
+
+function syncSkeleton(joinObj) {
+    joinObj = joinObj || {};
+    return {
+        next_batch: "abc",
+        rooms: {
+            join: {
+                "!foo:bar": joinObj,
+            },
+        },
+    };
+}
+
+function msg(localpart, text) {
+    return {
+        content: {
+            body: text,
+        },
+        origin_server_ts: 123456789,
+        sender: "@" + localpart + ":localhost",
+        type: "m.room.message",
+    };
+}
+
+function member(localpart, membership) {
+    return {
+        content: {
+            membership: membership,
+        },
+        origin_server_ts: 123456789,
+        state_key: "@" + localpart + ":localhost",
+        sender: "@" + localpart + ":localhost",
+        type: "m.room.member",
+    };
+}

spec/unit/timeline-window.spec.js

@@ -1,4 +1,5 @@
 "use strict";
+import 'source-map-support/register';
 const q = require("q");
 const sdk = require("../..");
 const EventTimeline = sdk.EventTimeline;
@@ -6,6 +7,7 @@ const TimelineWindow = sdk.TimelineWindow;
 const TimelineIndex = require("../../lib/timeline-window").TimelineIndex;
 
 const utils = require("../test-utils");
+import expect from 'expect';
 
 const ROOM_ID = "roomId";
 const USER_ID = "userId";

spec/unit/user.spec.js

@@ -1,8 +1,11 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const User = sdk.User;
 const utils = require("../test-utils");
 
+import expect from 'expect';
+
 describe("User", function() {
     const userId = "@alice:bar";
     let user;

spec/unit/utils.spec.js

@@ -1,7 +1,10 @@
 "use strict";
+import 'source-map-support/register';
 const utils = require("../../lib/utils");
 const testUtils = require("../test-utils");
 
+import expect from 'expect';
+
 describe("utils", function() {
     beforeEach(function() {
         testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
@@ -132,7 +135,7 @@ describe("utils", function() {
                 utils.checkObjectHasKeys({
                     foo: "bar",
                 }, ["foo"]);
-            }).not.toThrow();
+            }).toNotThrow();
         });
     });
 
@@ -149,7 +152,7 @@ describe("utils", function() {
                 utils.checkObjectHasNoAdditionalKeys({
                             foo: "bar",
                         }, ["foo"]);
-            }).not.toThrow();
+            }).toNotThrow();
         });
     });
 

src/base-apis.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -119,12 +120,28 @@ MatrixBaseApis.prototype.makeTxnId = function() {
 // Registration/Login operations
 // =============================
 
+/**
+ * Check whether a username is available prior to registration. An error response
+ * indicates an invalid/unavailable username.
+ * @param {string} username The username to check the availability of.
+ * @return {module:client.Promise} Resolves: to `true`.
+ */
+MatrixBaseApis.prototype.isUsernameAvailable = function(username) {
+    return this._http.authedRequest(
+        undefined, "GET", '/register/available', { username: username },
+    ).then((response) => {
+        return response.available;
+    });
+};
+
 /**
  * @param {string} username
  * @param {string} password
  * @param {string} sessionId
  * @param {Object} auth
- * @param {boolean} bindEmail
+ * @param {Object} bindThreepids Set key 'email' to true to bind any email
+ *     threepid uses during registration in the ID server. Set 'msisdn' to
+ *     true to bind msisdn.
  * @param {string} guestAccessToken
  * @param {module:client.callback} callback Optional.
  * @return {module:client.Promise} Resolves: TODO
@@ -132,10 +149,17 @@ MatrixBaseApis.prototype.makeTxnId = function() {
  */
 MatrixBaseApis.prototype.register = function(
     username, password,
-    sessionId, auth, bindEmail, guestAccessToken,
+    sessionId, auth, bindThreepids, guestAccessToken,
     callback,
 ) {
-    if (auth === undefined) {
+    // backwards compat
+    if (bindThreepids === true) {
+        bindThreepids = {email: true};
+    } else if (bindThreepids === null || bindThreepids === undefined) {
+        bindThreepids = {};
+    }
+
+    if (auth === undefined || auth === null) {
         auth = {};
     }
     if (sessionId) {
@@ -151,12 +175,24 @@ MatrixBaseApis.prototype.register = function(
     if (password !== undefined && password !== null) {
         params.password = password;
     }
-    if (bindEmail !== undefined && bindEmail !== null) {
-        params.bind_email = bindEmail;
+    if (bindThreepids.email) {
+        params.bind_email = true;
+    }
+    if (bindThreepids.msisdn) {
+        params.bind_msisdn = true;
     }
     if (guestAccessToken !== undefined && guestAccessToken !== null) {
         params.guest_access_token = guestAccessToken;
     }
+    // Temporary parameter added to make the register endpoint advertise
+    // msisdn flows. This exists because there are clients that break
+    // when given stages they don't recognise. This parameter will cease
+    // to be necessary once these old clients are gone.
+    // Only send it if we send any params at all (the password param is
+    // mandatory, so if we send any params, we'll send the password param)
+    if (password !== undefined && password !== null) {
+        params.x_show_msisdn = true;
+    }
 
     return this.registerRequest(params, undefined, callback);
 };
@@ -448,6 +484,34 @@ MatrixBaseApis.prototype.roomInitialSync = function(roomId, limit, callback) {
     );
 };
 
+/**
+ * Set a marker to indicate the point in a room before which the user has read every
+ * event. This can be retrieved from room account data (the event type is `m.fully_read`)
+ * and displayed as a horizontal line in the timeline that is visually distinct to the
+ * position of the user's own read receipt.
+ * @param {string} roomId ID of the room that has been read
+ * @param {string} rmEventId ID of the event that has been read
+ * @param {string} rrEventId ID of the event tracked by the read receipt. This is here
+ * for convenience because the RR and the RM are commonly updated at the same time as
+ * each other. Optional.
+ * @return {module:client.Promise} Resolves: the empty object, {}.
+ */
+MatrixBaseApis.prototype.setRoomReadMarkersHttpRequest =
+                                function(roomId, rmEventId, rrEventId) {
+    const path = utils.encodeUri("/rooms/$roomId/read_markers", {
+        $roomId: roomId,
+    });
+
+    const content = {
+        "m.fully_read": rmEventId,
+        "m.read": rrEventId,
+    };
+
+    return this._http.authedRequest(
+        undefined, "POST", path, undefined, content,
+    );
+};
+
 
 // Room Directory operations
 // =========================
@@ -614,6 +678,31 @@ MatrixBaseApis.prototype.setRoomDirectoryVisibilityAppService =
     );
 };
 
+// User Directory Operations
+// =========================
+
+/**
+ * Query the user directory with a term matching user IDs, display names and domains.
+ * @param {object} opts options
+ * @param {string} opts.term the term with which to search.
+ * @param {number} opts.limit the maximum number of results to return. The server will
+ *                 apply a limit if unspecified.
+ * @return {module:client.Promise} Resolves: an array of results.
+ */
+MatrixBaseApis.prototype.searchUserDirectory = function(opts) {
+    const body = {
+        search_term: opts.term,
+    };
+
+    if (opts.limit !== undefined) {
+        body.limit = opts.limit;
+    }
+
+    return this._http.authedRequest(
+        undefined, "POST", "/user_directory/search", undefined, body,
+    );
+};
+
 
 // Media operations
 // ================
@@ -1142,6 +1231,35 @@ MatrixBaseApis.prototype.requestEmailToken = function(email, clientSecret,
     );
 };
 
+/**
+ * Submits an MSISDN token to the identity server
+ *
+ * This is used when submitting the code sent by SMS to a phone number.
+ * The ID server has an equivalent API for email but the js-sdk does
+ * not expose this, since email is normally validated by the user clicking
+ * a link rather than entering a code.
+ *
+ * @param {string} sid The sid given in the response to requestToken
+ * @param {string} clientSecret A secret binary string generated by the client.
+ *                 This must be the same value submitted in the requestToken call.
+ * @param {string} token The token, as enetered by the user.
+ *
+ * @return {module:client.Promise} Resolves: Object, currently with no parameters.
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ * @throws Error if No ID server is set
+ */
+MatrixBaseApis.prototype.submitMsisdnToken = function(sid, clientSecret, token) {
+    const params = {
+        sid: sid,
+        client_secret: clientSecret,
+        token: token,
+    };
+    return this._http.idServerRequest(
+        undefined, "POST", "/validate/msisdn/submitToken",
+        params, httpApi.PREFIX_IDENTITY_V1,
+    );
+};
+
 /**
  * Looks up the public Matrix ID mapping for a given 3rd party
  * identifier from the Identity Server

src/client.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -46,8 +47,7 @@ try {
     var Crypto = require("./crypto");
     CRYPTO_ENABLED = true;
 } catch (e) {
-    console.error("olm load error", e);
-    // Olm not installed.
+    console.warn("Unable to load crypto module: crypto will be disabled: " + e);
 }
 
 /**
@@ -103,6 +103,9 @@ try {
  * disabled by default for compatibility with older clients - in particular to
  * maintain support for back-paginating the live timeline after a '/sync'
  * result with a gap.
+ *
+ * @param {module:crypto.store.base~CryptoStore} opts.cryptoStore
+ *    crypto store implementation.
  */
 function MatrixClient(opts) {
     MatrixBaseApis.call(this, opts);
@@ -152,13 +155,16 @@ function MatrixClient(opts) {
     this._notifTimelineSet = null;
 
     this._crypto = null;
+    this._cryptoStore = opts.cryptoStore;
     if (CRYPTO_ENABLED && Boolean(opts.sessionStore) &&
+            Boolean(this._cryptoStore) &&
             userId !== null && this.deviceId !== null) {
         this._crypto = new Crypto(
             this, this,
             opts.sessionStore,
             userId, this.deviceId,
             this.store,
+            opts.cryptoStore,
         );
 
         this.olmVersion = Crypto.getOlmVersion();
@@ -167,6 +173,37 @@ function MatrixClient(opts) {
 utils.inherits(MatrixClient, EventEmitter);
 utils.extend(MatrixClient.prototype, MatrixBaseApis.prototype);
 
+/**
+ * Clear any data out of the persistent stores used by the client.
+ *
+ * @returns {Promise} Promise which resolves when the stores have been cleared.
+ */
+MatrixClient.prototype.clearStores = function() {
+    if (this._clientRunning) {
+        throw new Error("Cannot clear stores while client is running");
+    }
+
+    const promises = [];
+
+    promises.push(this.store.deleteAllData());
+    if (this._cryptoStore) {
+        promises.push(this._cryptoStore.deleteAllData());
+    }
+    return q.all(promises);
+};
+
+/**
+ * Get the user-id of the logged-in user
+ *
+ * @return {?string} MXID for the logged-in user, or null if not logged in
+ */
+MatrixClient.prototype.getUserId = function() {
+    if (this.credentials && this.credentials.userId) {
+        return this.credentials.userId;
+    }
+    return null;
+};
+
 /**
  * Get the domain for this client's MXID
  * @return {?string} Domain of this MXID
@@ -300,17 +337,15 @@ MatrixClient.prototype.getDeviceEd25519Key = function() {
 };
 
 /**
- * Upload the device keys to the homeserver and ensure that the
- * homeserver has enough one-time keys.
- * @param {number} maxKeys The maximum number of keys to generate
+ * Upload the device keys to the homeserver.
  * @return {object} A promise that will resolve when the keys are uploaded.
  */
-MatrixClient.prototype.uploadKeys = function(maxKeys) {
+MatrixClient.prototype.uploadKeys = function() {
     if (this._crypto === null) {
         throw new Error("End-to-end encryption disabled");
     }
 
-    return this._crypto.uploadKeys(maxKeys);
+    return this._crypto.uploadDeviceKeys();
 };
 
 /**
@@ -360,6 +395,21 @@ MatrixClient.prototype.getStoredDevicesForUser = function(userId) {
     return this._crypto.getStoredDevicesForUser(userId) || [];
 };
 
+/**
+ * Get the stored device key for a user id and device id
+ *
+ * @param {string} userId the user to list keys for.
+ * @param {string} deviceId unique identifier for the device
+ *
+ * @return {?module:crypto-deviceinfo} device or null
+ */
+MatrixClient.prototype.getStoredDevice = function(userId, deviceId) {
+    if (this._crypto === null) {
+        throw new Error("End-to-end encryption disabled");
+    }
+    return this._crypto.getStoredDevice(userId, deviceId) || null;
+};
+
 /**
  * Mark the given device as verified
  *
@@ -550,7 +600,10 @@ function _decryptEvent(client, event) {
     try {
         client._crypto.decryptEvent(event);
     } catch (e) {
-        if (!(e instanceof Crypto.DecryptionError)) {
+        console.warn(
+            `Error decrypting event (id=${event.getId()}): ${e}`,
+        );
+        if (e.name !== "DecryptionError") {
             throw e;
         }
         _badEncryptedMessage(event, e.message);
@@ -888,6 +941,8 @@ MatrixClient.prototype.sendEvent = function(roomId, eventType, content, txnId,
         txnId = this.makeTxnId();
     }
 
+    console.log(`sendEvent of type ${eventType} in ${roomId} with txnId ${txnId}`);
+
     // we always construct a MatrixEvent when sending because the store and
     // scheduler use them. We'll extract the params back out if it turns out
     // the client has no scheduler or store.
@@ -1012,7 +1067,12 @@ function _sendEventHttpRequest(client, event) {
 
     return client._http.authedRequest(
         undefined, "PUT", path, undefined, event.getWireContent(),
-    );
+    ).then((res) => {
+        console.log(
+            `Event sent to ${event.getRoomId()} with event id ${res.event_id}`,
+        );
+        return res;
+    });
 }
 
 /**
@@ -1199,6 +1259,33 @@ MatrixClient.prototype.sendReadReceipt = function(event, callback) {
     return this.sendReceipt(event, "m.read", callback);
 };
 
+/**
+ * Set a marker to indicate the point in a room before which the user has read every
+ * event. This can be retrieved from room account data (the event type is `m.fully_read`)
+ * and displayed as a horizontal line in the timeline that is visually distinct to the
+ * position of the user's own read receipt.
+ * @param {string} roomId ID of the room that has been read
+ * @param {string} eventId ID of the event that has been read
+ * @param {string} rrEvent the event tracked by the read receipt. This is here for
+ * convenience because the RR and the RM are commonly updated at the same time as each
+ * other. The local echo of this receipt will be done if set. Optional.
+ * @return {module:client.Promise} Resolves: the empty object, {}.
+ */
+MatrixClient.prototype.setRoomReadMarkers = function(roomId, eventId, rrEvent) {
+    const rmEventId = eventId;
+    let rrEventId;
+
+    // Add the optional RR update, do local echo like `sendReceipt`
+    if (rrEvent) {
+        rrEventId = rrEvent.getId();
+        const room = this.getRoom(roomId);
+        if (room) {
+            room._addLocalEchoReceipt(this.credentials.userId, rrEvent, "m.read");
+        }
+    }
+
+    return this.setRoomReadMarkersHttpRequest(roomId, rmEventId, rrEventId);
+};
 
 /**
  * Get a preview of the given URL as of (roughly) the given point in time,
@@ -1377,13 +1464,23 @@ MatrixClient.prototype.forget = function(roomId, deleteRoom, callback) {
  * @param {string} roomId
  * @param {string} userId
  * @param {module:client.callback} callback Optional.
- * @return {module:client.Promise} Resolves: TODO
+ * @return {module:client.Promise} Resolves: Object (currently empty)
  * @return {module:http-api.MatrixError} Rejects: with an error response.
  */
 MatrixClient.prototype.unban = function(roomId, userId, callback) {
-    // unbanning = set their state to leave
-    return _setMembershipState(
-        this, roomId, userId, "leave", undefined, callback,
+    // unbanning != set their state to leave: this used to be
+    // the case, but was then changed so that leaving was always
+    // a revoking of priviledge, otherwise two people racing to
+    // kick / ban someone could end up banning and then un-banning
+    // them.
+    const path = utils.encodeUri("/rooms/$roomId/unban", {
+        $roomId: roomId,
+    });
+    const data = {
+        user_id: userId,
+    };
+    return this._http.authedRequest(
+        callback, "POST", path, undefined, data,
     );
 };
 
@@ -2070,14 +2167,45 @@ MatrixClient.prototype.setGuestAccess = function(roomId, opts) {
  * @param {string} clientSecret As requestEmailToken
  * @param {number} sendAttempt As requestEmailToken
  * @param {string} nextLink As requestEmailToken
- * @param {module:client.callback} callback Optional. As requestEmailToken
  * @return {module:client.Promise} Resolves: As requestEmailToken
  */
 MatrixClient.prototype.requestRegisterEmailToken = function(email, clientSecret,
-                                                    sendAttempt, nextLink, callback) {
+                                                    sendAttempt, nextLink) {
     return this._requestTokenFromEndpoint(
         "/register/email/requestToken",
-        email, clientSecret, sendAttempt, nextLink, callback,
+        {
+            email: email,
+            client_secret: clientSecret,
+            send_attempt: sendAttempt,
+            next_link: nextLink,
+        },
+    );
+};
+
+/**
+ * Requests a text message verification token for the purposes of registration.
+ * This API proxies the Identity Server /validate/msisdn/requestToken API,
+ * adding registration-specific behaviour, as with requestRegisterEmailToken.
+ *
+ * @param {string} phoneCountry The ISO 3166-1 alpha-2 code for the country in which
+ *    phoneNumber should be parsed relative to.
+ * @param {string} phoneNumber The phone number, in national or international format
+ * @param {string} clientSecret As requestEmailToken
+ * @param {number} sendAttempt As requestEmailToken
+ * @param {string} nextLink As requestEmailToken
+ * @return {module:client.Promise} Resolves: As requestEmailToken
+ */
+MatrixClient.prototype.requestRegisterMsisdnToken = function(phoneCountry, phoneNumber,
+                                                    clientSecret, sendAttempt, nextLink) {
+    return this._requestTokenFromEndpoint(
+        "/register/msisdn/requestToken",
+        {
+            country: phoneCountry,
+            phone_number: phoneNumber,
+            client_secret: clientSecret,
+            send_attempt: sendAttempt,
+            next_link: nextLink,
+        },
     );
 };
 
@@ -2098,14 +2226,46 @@ MatrixClient.prototype.requestRegisterEmailToken = function(email, clientSecret,
  * @param {string} clientSecret As requestEmailToken
  * @param {number} sendAttempt As requestEmailToken
  * @param {string} nextLink As requestEmailToken
- * @param {module:client.callback} callback Optional. As requestEmailToken
  * @return {module:client.Promise} Resolves: As requestEmailToken
  */
 MatrixClient.prototype.requestAdd3pidEmailToken = function(email, clientSecret,
-                                                    sendAttempt, nextLink, callback) {
+                                                    sendAttempt, nextLink) {
     return this._requestTokenFromEndpoint(
         "/account/3pid/email/requestToken",
-        email, clientSecret, sendAttempt, nextLink, callback,
+        {
+            email: email,
+            client_secret: clientSecret,
+            send_attempt: sendAttempt,
+            next_link: nextLink,
+        },
+    );
+};
+
+/**
+ * Requests a text message verification token for the purposes of adding a
+ * third party identifier to an account.
+ * This API proxies the Identity Server /validate/email/requestToken API,
+ * adding specific behaviour for the addition of phone numbers to an
+ * account, as requestAdd3pidEmailToken.
+ *
+ * @param {string} phoneCountry As requestRegisterMsisdnToken
+ * @param {string} phoneNumber As requestRegisterMsisdnToken
+ * @param {string} clientSecret As requestEmailToken
+ * @param {number} sendAttempt As requestEmailToken
+ * @param {string} nextLink As requestEmailToken
+ * @return {module:client.Promise} Resolves: As requestEmailToken
+ */
+MatrixClient.prototype.requestAdd3pidMsisdnToken = function(phoneCountry, phoneNumber,
+                                                    clientSecret, sendAttempt, nextLink) {
+    return this._requestTokenFromEndpoint(
+        "/account/3pid/msisdn/requestToken",
+        {
+            country: phoneCountry,
+            phone_number: phoneNumber,
+            client_secret: clientSecret,
+            send_attempt: sendAttempt,
+            next_link: nextLink,
+        },
     );
 };
 
@@ -2129,10 +2289,42 @@ MatrixClient.prototype.requestAdd3pidEmailToken = function(email, clientSecret,
  * @return {module:client.Promise} Resolves: As requestEmailToken
  */
 MatrixClient.prototype.requestPasswordEmailToken = function(email, clientSecret,
-                                                    sendAttempt, nextLink, callback) {
+                                                    sendAttempt, nextLink) {
     return this._requestTokenFromEndpoint(
         "/account/password/email/requestToken",
-        email, clientSecret, sendAttempt, nextLink, callback,
+        {
+            email: email,
+            client_secret: clientSecret,
+            send_attempt: sendAttempt,
+            next_link: nextLink,
+        },
+    );
+};
+
+/**
+ * Requests a text message verification token for the purposes of resetting
+ * the password on an account.
+ * This API proxies the Identity Server /validate/email/requestToken API,
+ * adding specific behaviour for the password resetting, as requestPasswordEmailToken.
+ *
+ * @param {string} phoneCountry As requestRegisterMsisdnToken
+ * @param {string} phoneNumber As requestRegisterMsisdnToken
+ * @param {string} clientSecret As requestEmailToken
+ * @param {number} sendAttempt As requestEmailToken
+ * @param {string} nextLink As requestEmailToken
+ * @return {module:client.Promise} Resolves: As requestEmailToken
+ */
+MatrixClient.prototype.requestPasswordMsisdnToken = function(phoneCountry, phoneNumber,
+                                                    clientSecret, sendAttempt, nextLink) {
+    return this._requestTokenFromEndpoint(
+        "/account/password/msisdn/requestToken",
+        {
+            country: phoneCountry,
+            phone_number: phoneNumber,
+            client_secret: clientSecret,
+            send_attempt: sendAttempt,
+            next_link: nextLink,
+        },
     );
 };
 
@@ -2141,31 +2333,21 @@ MatrixClient.prototype.requestPasswordEmailToken = function(email, clientSecret,
  * requestToken endpoints.
  *
  * @param {string} endpoint The endpoint to send the request to
- * @param {string} email As requestEmailToken
- * @param {string} clientSecret As requestEmailToken
- * @param {number} sendAttempt As requestEmailToken
- * @param {string} nextLink As requestEmailToken
- * @param {module:client.callback} callback Optional. As requestEmailToken
+ * @param {object} params Parameters for the POST request
  * @return {module:client.Promise} Resolves: As requestEmailToken
  */
-MatrixClient.prototype._requestTokenFromEndpoint = function(endpoint,
-                                                    email, clientSecret,
-                                                    sendAttempt, nextLink, callback) {
+MatrixClient.prototype._requestTokenFromEndpoint = function(endpoint, params) {
     const id_server_url = url.parse(this.idBaseUrl);
     if (id_server_url.host === null) {
         throw new Error("Invalid ID server URL: " + this.idBaseUrl);
     }
 
-    const params = {
-        client_secret: clientSecret,
-        email: email,
-        send_attempt: sendAttempt,
-        next_link: nextLink,
+    const postParams = Object.assign({}, params, {
         id_server: id_server_url.host,
-    };
+    });
     return this._http.request(
-        callback, "POST", endpoint, undefined,
-        params,
+        undefined, "POST", endpoint, undefined,
+        postParams,
     );
 };
 
@@ -2286,13 +2468,18 @@ MatrixClient.prototype.setRoomMutePushRule = function(scope, roomId, mute) {
  * @return {module:http-api.MatrixError} Rejects: with an error response.
  */
 MatrixClient.prototype.searchMessageText = function(opts, callback) {
+    const roomEvents = {
+        search_term: opts.query,
+    };
+
+    if ('keys' in opts) {
+        roomEvents.keys = opts.keys;
+    }
+
     return this.search({
         body: {
             search_categories: {
-                room_events: {
-                    keys: opts.keys,
-                    search_term: opts.query,
-                },
+                room_events: roomEvents,
             },
         },
     }, callback);
@@ -2629,10 +2816,10 @@ MatrixClient.prototype.getTurnServers = function() {
 
 
 /**
- * High level helper method to call initialSync, emit the resulting events,
- * and then start polling the eventStream for new events. To listen for these
+ * High level helper method to begin syncing and poll for new events. To listen for these
  * events, add a listener for {@link module:client~MatrixClient#event:"event"}
- * via {@link module:client~MatrixClient#on}.
+ * via {@link module:client~MatrixClient#on}. Alternatively, listen for specific
+ * state change events.
  * @param {Object=} opts Options to apply when syncing.
  * @param {Number=} opts.initialSyncLimit The event <code>limit=</code> to apply
  * to initial sync. Default: 8.
@@ -2649,7 +2836,7 @@ MatrixClient.prototype.getTurnServers = function() {
  * accessbile via {@link module:models/room#getPendingEvents}. Default:
  * "chronological".
  *
- * @param {Number=} opts.pollTimeout The number of milliseconds to wait on /events.
+ * @param {Number=} opts.pollTimeout The number of milliseconds to wait on /sync.
  * Default: 30000 (30 seconds).
  *
  * @param {Filter=} opts.filter The filter to apply to /sync calls. This will override
@@ -2669,12 +2856,7 @@ MatrixClient.prototype.startClient = function(opts) {
     }
 
     if (this._crypto) {
-        this._crypto.uploadKeys(5).done();
-        const tenMinutes = 1000 * 60 * 10;
-        const self = this;
-        this._uploadIntervalID = global.setInterval(function() {
-            self._crypto.uploadKeys(5).done();
-        }, tenMinutes);
+        this._crypto.uploadDeviceKeys().done();
     }
 
     // periodically poll for turn servers if we support voip
@@ -2690,6 +2872,12 @@ MatrixClient.prototype.startClient = function(opts) {
     opts = Object.assign({}, opts);
 
     opts.crypto = this._crypto;
+    opts.canResetEntireTimeline = (roomId) => {
+        if (!this._canResetTimelineCallback) {
+            return false;
+        }
+        return this._canResetTimelineCallback(roomId);
+    };
     this._clientOpts = opts;
 
     this._syncApi = new SyncApi(this, opts);
@@ -2707,12 +2895,33 @@ MatrixClient.prototype.stopClient = function() {
         this._syncApi.stop();
         this._syncApi = null;
     }
-    if (this._crypto) {
-        global.clearInterval(this._uploadIntervalID);
+    if (this._peekSync) {
+        this._peekSync.stopPeeking();
     }
     global.clearTimeout(this._checkTurnServersTimeoutID);
 };
 
+/*
+ * Set a function which is called when /sync returns a 'limited' response.
+ * It is called with a room ID and returns a boolean. It should return 'true' if the SDK
+ * can SAFELY remove events from this room. It may not be safe to remove events if there
+ * are other references to the timelines for this room, e.g because the client is
+ * actively viewing events in this room.
+ * Default: returns false.
+ * @param {Function} cb The callback which will be invoked.
+ */
+MatrixClient.prototype.setCanResetTimelineCallback = function(cb) {
+    this._canResetTimelineCallback = cb;
+};
+
+/**
+ * Get the callback set via `setCanResetTimelineCallback`.
+ * @return {?Function} The callback or null
+ */
+MatrixClient.prototype.getCanResetTimelineCallback = function() {
+    return this._canResetTimelineCallback;
+};
+
 function setupCallEventHandler(client) {
     const candidatesByCall = {
         // callId: [Candidate]
@@ -2817,7 +3026,7 @@ function setupCallEventHandler(client) {
             const existingCalls = utils.values(client.callList);
             for (i = 0; i < existingCalls.length; ++i) {
                 const thisCall = existingCalls[i];
-                if (call.room_id === thisCall.room_id &&
+                if (call.roomId === thisCall.roomId &&
                         thisCall.direction === 'outbound' &&
                         (["wait_local_media", "create_offer", "invite_sent"].indexOf(
                             thisCall.state) !== -1)) {
@@ -3020,21 +3229,26 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
 /**
  * Fires whenever the SDK's syncing state is updated. The state can be one of:
  * <ul>
- * <li>PREPARED : The client has synced with the server at least once and is
+ *
+ * <li>PREPARED: The client has synced with the server at least once and is
  * ready for methods to be called on it. This will be immediately followed by
  * a state of SYNCING. <i>This is the equivalent of "syncComplete" in the
  * previous API.</i></li>
+ *
  * <li>SYNCING : The client is currently polling for new events from the server.
  * This will be called <i>after</i> processing latest events from a sync.</li>
+ *
  * <li>ERROR : The client has had a problem syncing with the server. If this is
  * called <i>before</i> PREPARED then there was a problem performing the initial
  * sync. If this is called <i>after</i> PREPARED then there was a problem polling
  * the server for updates. This may be called multiple times even if the state is
  * already ERROR. <i>This is the equivalent of "syncError" in the previous
  * API.</i></li>
- * <li>RECONNECTING: The sync connedtion has dropped, but not in a way that should
- * be considered erroneous.
+ *
+ * <li>RECONNECTING: The sync connection has dropped, but not (yet) in a way that
+ * should be considered erroneous.
  * </li>
+ *
  * <li>STOPPED: The client has stopped syncing with server due to stopClient
  * being called.
  * </li>
@@ -3044,32 +3258,46 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
  *                                          +---->STOPPED
  *                                          |
  *              +----->PREPARED -------> SYNCING <--+
- *              |        ^                  ^       |
- *              |        |                  |       |
- *              |        |                  V       |
- *   null ------+        |  +-RECONNECTING<-+       |
+ *              |        ^                |  ^      |
+ *              |        |                |  |      |
+ *              |        |                V  |      |
+ *   null ------+        |  +--------RECONNECTING   |
  *              |        |  V                       |
  *              +------->ERROR ---------------------+
  *
  * NB: 'null' will never be emitted by this event.
+ *
  * </pre>
  * Transitions:
  * <ul>
+ *
  * <li><code>null -> PREPARED</code> : Occurs when the initial sync is completed
  * first time. This involves setting up filters and obtaining push rules.
+ *
  * <li><code>null -> ERROR</code> : Occurs when the initial sync failed first time.
+ *
  * <li><code>ERROR -> PREPARED</code> : Occurs when the initial sync succeeds
  * after previously failing.
+ *
  * <li><code>PREPARED -> SYNCING</code> : Occurs immediately after transitioning
  * to PREPARED. Starts listening for live updates rather than catching up.
- * <li><code>SYNCING -> ERROR</code> : Occurs the first time a client cannot perform a
- * live update.
+ *
+ * <li><code>SYNCING -> RECONNECTING</code> : Occurs when the live update fails.
+ *
+ * <li><code>RECONNECTING -> RECONNECTING</code> : Can occur if the update calls
+ * continue to fail, but the keepalive calls (to /versions) succeed.
+ *
+ * <li><code>RECONNECTING -> ERROR</code> : Occurs when the keepalive call also fails
+ *
  * <li><code>ERROR -> SYNCING</code> : Occurs when the client has performed a
  * live update after having previously failed.
- * <li><code>ERROR -> ERROR</code> : Occurs when the client has failed to sync
+ *
+ * <li><code>ERROR -> ERROR</code> : Occurs when the client has failed to keepalive
  * for a second time or more.</li>
+ *
  * <li><code>SYNCING -> SYNCING</code> : Occurs when the client has performed a live
  * update. This is called <i>after</i> processing.</li>
+ *
  * <li><code>* -> STOPPED</code> : Occurs once the client has stopped syncing or
  * trying to sync after stopClient has been called.</li>
  * </ul>
@@ -3095,6 +3323,9 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
  *    will become the 'since' token for the next call to /sync. Only present if
  *    <code>state=PREPARED</code> or <code>state=SYNCING</code>.
  *
+ * @param {boolean} data.catchingUp True if we are working our way through a
+ *    backlog of events after connecting. Only present if <code>state=SYNCING</code>.
+ *
  * @example
  * matrixClient.on("sync", function(state, prevState, data) {
  *   switch (state) {
@@ -3170,7 +3401,7 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
 
 /**
  * Fires whenever new user-scoped account_data is added.
- * @event module:client~MatrixClient#"Room"
+ * @event module:client~MatrixClient#"accountData"
  * @param {MatrixEvent} event The event describing the account_data just added
  * @example
  * matrixClient.on("accountData", function(event){

src/crypto/DeviceList.js

@@ -26,18 +26,31 @@ import q from 'q';
 import DeviceInfo from './deviceinfo';
 import olmlib from './olmlib';
 
+// constants for DeviceList._deviceTrackingStatus
+// const TRACKING_STATUS_NOT_TRACKED = 0;
+const TRACKING_STATUS_PENDING_DOWNLOAD = 1;
+const TRACKING_STATUS_DOWNLOAD_IN_PROGRESS = 2;
+const TRACKING_STATUS_UP_TO_DATE = 3;
+
 /**
  * @alias module:crypto/DeviceList
  */
 export default class DeviceList {
     constructor(baseApis, sessionStore, olmDevice) {
-        this._baseApis = baseApis;
         this._sessionStore = sessionStore;
-        this._olmDevice = olmDevice;
+        this._serialiser = new DeviceListUpdateSerialiser(
+            baseApis, sessionStore, olmDevice,
+        );
 
-        // users with outdated device lists
-        // userId -> true
-        this._pendingUsersWithNewDevices = {};
+        // which users we are tracking device status for.
+        // userId -> TRACKING_STATUS_*
+        this._deviceTrackingStatus = sessionStore.getEndToEndDeviceTrackingStatus() || {};
+        for (const u of Object.keys(this._deviceTrackingStatus)) {
+            // if a download was in progress when we got shut down, it isn't any more.
+            if (this._deviceTrackingStatus[u] == TRACKING_STATUS_DOWNLOAD_IN_PROGRESS) {
+                this._deviceTrackingStatus[u] = TRACKING_STATUS_PENDING_DOWNLOAD;
+            }
+        }
 
         // userId -> promise
         this._keyDownloadsInProgressByUser = {};
@@ -55,26 +68,32 @@ export default class DeviceList {
      * module:crypto/deviceinfo|DeviceInfo}.
      */
     downloadKeys(userIds, forceDownload) {
-        // promises we need to wait for while the download happens
+        const usersToDownload = [];
         const promises = [];
 
-        let needsRefresh = false;
         userIds.forEach((u) => {
+            const trackingStatus = this._deviceTrackingStatus[u];
             if (this._keyDownloadsInProgressByUser[u]) {
-                // just wait for the existing download to complete
+                // already a key download in progress/queued for this user; its results
+                // will be good enough for us.
+                console.log(
+                    `downloadKeys: already have a download in progress for ` +
+                    `${u}: awaiting its result`,
+                );
                 promises.push(this._keyDownloadsInProgressByUser[u]);
-            } else {
-                if (forceDownload || !this.getStoredDevicesForUser(u)) {
-                    this.invalidateUserDeviceList(u);
-                }
-                if (this._pendingUsersWithNewDevices[u]) {
-                    needsRefresh = true;
-                }
+            } else if (forceDownload || trackingStatus != TRACKING_STATUS_UP_TO_DATE) {
+                usersToDownload.push(u);
             }
         });
 
-        if (needsRefresh) {
-            promises.push(this.refreshOutdatedDeviceLists(true));
+        if (usersToDownload.length != 0) {
+            console.log("downloadKeys: downloading for", usersToDownload);
+            const downloadPromise = this._doKeyDownload(usersToDownload);
+            promises.push(downloadPromise);
+        }
+
+        if (promises.length === 0) {
+            console.log("downloadKeys: already have all necessary keys");
         }
 
         return q.all(promises).then(() => {
@@ -146,11 +165,11 @@ export default class DeviceList {
      *
      * @param {string} userId     owner of the device
      * @param {string} algorithm  encryption algorithm
-     * @param {string} sender_key curve25519 key to match
+     * @param {string} senderKey  curve25519 key to match
      *
      * @return {module:crypto/deviceinfo?}
      */
-    getDeviceByIdentityKey(userId, algorithm, sender_key) {
+    getDeviceByIdentityKey(userId, algorithm, senderKey) {
         if (
             algorithm !== olmlib.OLM_ALGORITHM &&
             algorithm !== olmlib.MEGOLM_ALGORITHM
@@ -178,7 +197,7 @@ export default class DeviceList {
                     continue;
                 }
                 const deviceKey = device.keys[keyId];
-                if (deviceKey == sender_key) {
+                if (deviceKey == senderKey) {
                     return DeviceInfo.fromStorage(device, deviceId);
                 }
             }
@@ -189,14 +208,15 @@ export default class DeviceList {
     }
 
     /**
-     * Mark the cached device list for the given user outdated.
+     * flag the given user for device-list tracking, if they are not already.
      *
-     * This doesn't set off an update, so that several users can be batched
-     * together. Call refreshOutdatedDeviceLists() for that.
+     * This will mean that a subsequent call to refreshOutdatedDeviceLists()
+     * will download the device list for the user, and that subsequent calls to
+     * invalidateUserDeviceList will trigger more updates.
      *
      * @param {String} userId
      */
-    invalidateUserDeviceList(userId) {
+    startTrackingDeviceList(userId) {
         // sanity-check the userId. This is mostly paranoia, but if synapse
         // can't parse the userId we give it as an mxid, it 500s the whole
         // request and we can never update the device lists again (because
@@ -207,140 +227,301 @@ export default class DeviceList {
         if (typeof userId !== 'string') {
             throw new Error('userId must be a string; was '+userId);
         }
-        this._pendingUsersWithNewDevices[userId] = true;
+        if (!this._deviceTrackingStatus[userId]) {
+            console.log('Now tracking device list for ' + userId);
+            this._deviceTrackingStatus[userId] = TRACKING_STATUS_PENDING_DOWNLOAD;
+        }
+        // we don't yet persist the tracking status, since there may be a lot
+        // of calls; instead we wait for the forthcoming
+        // refreshOutdatedDeviceLists.
     }
 
     /**
-     * Start device queries for any users with outdated device lists
+     * Mark the cached device list for the given user outdated.
      *
-     * We tolerate multiple concurrent device queries, but only one query per
-     * user.
+     * If we are not tracking this user's devices, we'll do nothing. Otherwise
+     * we flag the user as needing an update.
      *
-     * If any users already have downloads in progress, they are ignored - they
-     * will be refreshed when the current download completes anyway, so
-     * each user with outdated device lists will be updated eventually.
+     * This doesn't actually set off an update, so that several users can be
+     * batched together. Call refreshOutdatedDeviceLists() for that.
      *
-     * The returned promise resolves immediately if there are no users with
-     * outdated device lists, or if all users with outdated device lists already
-     * have a query in progress.
-     *
-     * Otherwise, a new query request is made, and the promise resolves
-     * once that query completes. If the query fails, the promise will reject
-     * if rejectOnFailure was truthy, otherwise it will still resolve.
-     *
-     * @param {Boolean?} rejectOnFailure  true to make the returned promise
-     *   reject if the device list query fails.
-     *
-     * @return {Promise}
+     * @param {String} userId
      */
-    refreshOutdatedDeviceLists(rejectOnFailure) {
-        const users = Object.keys(this._pendingUsersWithNewDevices).filter(
-            (u) => !this._keyDownloadsInProgressByUser[u],
-        );
+    invalidateUserDeviceList(userId) {
+        if (this._deviceTrackingStatus[userId]) {
+            console.log("Marking device list outdated for", userId);
+            this._deviceTrackingStatus[userId] = TRACKING_STATUS_PENDING_DOWNLOAD;
+        }
+        // we don't yet persist the tracking status, since there may be a lot
+        // of calls; instead we wait for the forthcoming
+        // refreshOutdatedDeviceLists.
+    }
 
+    /**
+     * Mark all tracked device lists as outdated.
+     *
+     * This will flag each user whose devices we are tracking as in need of an
+     * update.
+     */
+    invalidateAllDeviceLists() {
+        for (const userId of Object.keys(this._deviceTrackingStatus)) {
+            this.invalidateUserDeviceList(userId);
+        }
+    }
+
+    /**
+     * If we have users who have outdated device lists, start key downloads for them
+     *
+     * @returns {Promise} which completes when the download completes; normally there
+     *    is no need to wait for this (it's mostly for the unit tests).
+     */
+    refreshOutdatedDeviceLists() {
+        const usersToDownload = [];
+        for (const userId of Object.keys(this._deviceTrackingStatus)) {
+            const stat = this._deviceTrackingStatus[userId];
+            if (stat == TRACKING_STATUS_PENDING_DOWNLOAD) {
+                usersToDownload.push(userId);
+            }
+        }
+        if (usersToDownload.length == 0) {
+            return;
+        }
+
+        // we didn't persist the tracking status during
+        // invalidateUserDeviceList, so do it now.
+        this._persistDeviceTrackingStatus();
+
+        return this._doKeyDownload(usersToDownload);
+    }
+
+
+    /**
+     * Fire off download update requests for the given users, and update the
+     * device list tracking status for them, and the
+     * _keyDownloadsInProgressByUser map for them.
+     *
+     * @param {String[]} users  list of userIds
+     *
+     * @return {module:client.Promise} resolves when all the users listed have
+     *     been updated. rejects if there was a problem updating any of the
+     *     users.
+     */
+    _doKeyDownload(users) {
         if (users.length === 0) {
+            // nothing to do
             return q();
         }
 
-        let prom = this._doKeyDownloadForUsers(users).then(() => {
-            users.forEach((u) => {
-                delete this._keyDownloadsInProgressByUser[u];
-            });
-
-            // flush out any more requests that were blocked up while that
-            // was going on, but let the initial promise complete now.
-            //
-            this.refreshOutdatedDeviceLists().done();
+        const prom = this._serialiser.updateDevicesForUsers(
+            users, this.lastKnownSyncToken,
+        ).then(() => {
+            finished(true);
         }, (e) => {
             console.error(
-                'Error updating device key cache for ' + users + ":", e,
+                'Error downloading keys for ' + users + ":", e,
             );
-
-            // reinstate the pending flags on any users which failed; this will
-            // mean that we will do another download in the future, but won't
-            // tight-loop.
-            //
-            users.forEach((u) => {
-                delete this._keyDownloadsInProgressByUser[u];
-                this._pendingUsersWithNewDevices[u] = true;
-            });
-
-            // TODO: schedule a retry.
+            finished(false);
             throw e;
         });
 
         users.forEach((u) => {
-            delete this._pendingUsersWithNewDevices[u];
             this._keyDownloadsInProgressByUser[u] = prom;
+            const stat = this._deviceTrackingStatus[u];
+            if (stat == TRACKING_STATUS_PENDING_DOWNLOAD) {
+                this._deviceTrackingStatus[u] = TRACKING_STATUS_DOWNLOAD_IN_PROGRESS;
+            }
         });
 
-        if (!rejectOnFailure) {
-            // normally we just want to swallow the exception - we've already
-            // logged it futher up.
-            prom = prom.catch((e) => {});
-        }
+        const finished = (success) => {
+            users.forEach((u) => {
+                // we may have queued up another download request for this user
+                // since we started this request. If that happens, we should
+                // ignore the completion of the first one.
+                if (this._keyDownloadsInProgressByUser[u] !== prom) {
+                    console.log('Another update in the queue for', u,
+                                '- not marking up-to-date');
+                    return;
+                }
+                delete this._keyDownloadsInProgressByUser[u];
+                const stat = this._deviceTrackingStatus[u];
+                if (stat == TRACKING_STATUS_DOWNLOAD_IN_PROGRESS) {
+                    if (success) {
+                        // we didn't get any new invalidations since this download started:
+                        // this user's device list is now up to date.
+                        this._deviceTrackingStatus[u] = TRACKING_STATUS_UP_TO_DATE;
+                        console.log("Device list for", u, "now up to date");
+                    } else {
+                        this._deviceTrackingStatus[u] = TRACKING_STATUS_PENDING_DOWNLOAD;
+                    }
+                }
+            });
+            this._persistDeviceTrackingStatus();
+        };
+
         return prom;
     }
 
-    /**
-     * @param {string[]} downloadUsers list of userIds
-     *
-     * @return {Promise}
-     */
-    _doKeyDownloadForUsers(downloadUsers) {
-        console.log('Starting key download for ' + downloadUsers);
+    _persistDeviceTrackingStatus() {
+        this._sessionStore.storeEndToEndDeviceTrackingStatus(this._deviceTrackingStatus);
+    }
+}
 
-        const token = this.lastKnownSyncToken;
-        const opts = {};
-        if (token) {
-            opts.token = token;
+/**
+ * Serialises updates to device lists
+ *
+ * Ensures that results from /keys/query are not overwritten if a second call
+ * completes *before* an earlier one.
+ *
+ * It currently does this by ensuring only one call to /keys/query happens at a
+ * time (and queuing other requests up).
+ */
+class DeviceListUpdateSerialiser {
+    constructor(baseApis, sessionStore, olmDevice) {
+        this._baseApis = baseApis;
+        this._sessionStore = sessionStore;
+        this._olmDevice = olmDevice;
+
+        this._downloadInProgress = false;
+
+        // users which are queued for download
+        // userId -> true
+        this._keyDownloadsQueuedByUser = {};
+
+        // deferred which is resolved when the queued users are downloaded.
+        //
+        // non-null indicates that we have users queued for download.
+        this._queuedQueryDeferred = null;
+
+        // sync token to be used for the next query: essentially the
+        // most recent one we know about
+        this._nextSyncToken = null;
+    }
+
+    /**
+     * Make a key query request for the given users
+     *
+     * @param {String[]} users list of user ids
+     *
+     * @param {String} syncToken sync token to pass in the query request, to
+     *     help the HS give the most recent results
+     *
+     * @return {module:client.Promise} resolves when all the users listed have
+     *     been updated. rejects if there was a problem updating any of the
+     *     users.
+     */
+    updateDevicesForUsers(users, syncToken) {
+        users.forEach((u) => {
+            this._keyDownloadsQueuedByUser[u] = true;
+        });
+        this._nextSyncToken = syncToken;
+
+        if (!this._queuedQueryDeferred) {
+            this._queuedQueryDeferred = q.defer();
         }
-        return this._baseApis.downloadKeysForUsers(
+
+        if (this._downloadInProgress) {
+            // just queue up these users
+            console.log('Queued key download for', users);
+            return this._queuedQueryDeferred.promise;
+        }
+
+        // start a new download.
+        return this._doQueuedQueries();
+    }
+
+    _doQueuedQueries() {
+        if (this._downloadInProgress) {
+            throw new Error(
+                "DeviceListUpdateSerialiser._doQueuedQueries called with request active",
+            );
+        }
+
+        const downloadUsers = Object.keys(this._keyDownloadsQueuedByUser);
+        this._keyDownloadsQueuedByUser = {};
+        const deferred = this._queuedQueryDeferred;
+        this._queuedQueryDeferred = null;
+
+        console.log('Starting key download for', downloadUsers);
+        this._downloadInProgress = true;
+
+        const opts = {};
+        if (this._nextSyncToken) {
+            opts.token = this._nextSyncToken;
+        }
+
+        this._baseApis.downloadKeysForUsers(
             downloadUsers, opts,
         ).then((res) => {
             const dk = res.device_keys || {};
 
+            // do each user in a separate promise, to avoid wedging the CPU
+            // (https://github.com/vector-im/riot-web/issues/3158)
+            //
+            // of course we ought to do this in a web worker or similar, but
+            // this serves as an easy solution for now.
+            let prom = q();
             for (const userId of downloadUsers) {
-                console.log('got keys for ' + userId + ':', dk[userId]);
-
-                // map from deviceid -> deviceinfo for this user
-                const userStore = {};
-                const devs = this._sessionStore.getEndToEndDevicesForUser(userId);
-                if (devs) {
-                    Object.keys(devs).forEach((deviceId) => {
-                        const d = DeviceInfo.fromStorage(devs[deviceId], deviceId);
-                        userStore[deviceId] = d;
-                    });
-                }
-
-                _updateStoredDeviceKeysForUser(
-                    this._olmDevice, userId, userStore, dk[userId] || {},
-                );
-
-                // update the session store
-                const storage = {};
-                Object.keys(userStore).forEach((deviceId) => {
-                    storage[deviceId] = userStore[deviceId].toStorage();
+                prom = prom.delay(5).then(() => {
+                    this._processQueryResponseForUser(userId, dk[userId]);
                 });
-
-                this._sessionStore.storeEndToEndDevicesForUser(
-                    userId, storage,
-                );
-
-                if (token) {
-                    this._sessionStore.storeEndToEndDeviceSyncToken(token);
-                }
             }
+
+            return prom;
+        }).done(() => {
+            console.log('Completed key download for ' + downloadUsers);
+
+            this._downloadInProgress = false;
+            deferred.resolve();
+
+            // if we have queued users, fire off another request.
+            if (this._queuedQueryDeferred) {
+                this._doQueuedQueries();
+            }
+        }, (e) => {
+            console.warn('Error downloading keys for ' + downloadUsers + ':', e);
+            this._downloadInProgressInProgress = false;
+            deferred.reject(e);
         });
+
+        return deferred.promise;
+    }
+
+    _processQueryResponseForUser(userId, response) {
+        console.log('got keys for ' + userId + ':', response);
+
+        // map from deviceid -> deviceinfo for this user
+        const userStore = {};
+        const devs = this._sessionStore.getEndToEndDevicesForUser(userId);
+        if (devs) {
+            Object.keys(devs).forEach((deviceId) => {
+                const d = DeviceInfo.fromStorage(devs[deviceId], deviceId);
+                userStore[deviceId] = d;
+            });
+        }
+
+        _updateStoredDeviceKeysForUser(
+            this._olmDevice, userId, userStore, response || {},
+        );
+
+        // update the session store
+        const storage = {};
+        Object.keys(userStore).forEach((deviceId) => {
+            storage[deviceId] = userStore[deviceId].toStorage();
+        });
+
+        this._sessionStore.storeEndToEndDevicesForUser(
+            userId, storage,
+        );
     }
 }
 
+
 function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
         userResult) {
     let updated = false;
 
     // remove any devices in the store which aren't in the response
-    for (var deviceId in userStore) {
+    for (const deviceId in userStore) {
         if (!userStore.hasOwnProperty(deviceId)) {
             continue;
         }
@@ -353,7 +534,7 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
         }
     }
 
-    for (deviceId in userResult) {
+    for (const deviceId in userResult) {
         if (!userResult.hasOwnProperty(deviceId)) {
             continue;
         }
@@ -398,7 +579,7 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
     const signKeyId = "ed25519:" + deviceId;
     const signKey = deviceResult.keys[signKeyId];
     if (!signKey) {
-        console.log("Device " + userId + ":" + deviceId +
+        console.warn("Device " + userId + ":" + deviceId +
             " has no ed25519 key");
         return false;
     }
@@ -408,8 +589,8 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
     try {
         olmlib.verifySignature(_olmDevice, deviceResult, userId, deviceId, signKey);
     } catch (e) {
-        console.log("Unable to verify signature on device " +
-            userId + ":" + deviceId + ":", e);
+        console.warn("Unable to verify signature on device " +
+            userId + ":" + deviceId + ":" + e);
         return false;
     }
 
@@ -425,7 +606,7 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
             // best off sticking with the original keys.
             //
             // Should we warn the user about it somehow?
-            console.warn("Ed25519 key for device" + userId + ": " +
+            console.warn("Ed25519 key for device " + userId + ":" +
                deviceId + " has changed");
             return false;
         }

src/crypto/OlmDevice.js

@@ -20,11 +20,9 @@ limitations under the License.
  *
  * @module crypto/OlmDevice
  */
-const Olm = require("olm");
+const Olm = global.Olm;
 if (!Olm) {
-    // this happens if we were loaded via browserify and the Olm module was not
-    // loaded.
-    throw new Error("Olm is not defined");
+    throw new Error("global.Olm is not defined");
 }
 const utils = require("../utils");
 

src/crypto/algorithms/base.js

@@ -13,14 +13,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
 
 /**
  * Internal module. Defines the base classes of the encryption implementations
  *
- * @module crypto/algorithms/base
+ * @module
  */
-const utils = require("../../utils");
 
 /**
  * map of registered encryption algorithm classes. A map from string to {@link
@@ -28,7 +26,7 @@ const utils = require("../../utils");
  *
  * @type {Object.<string, function(new: module:crypto/algorithms/base.EncryptionAlgorithm)>}
  */
-module.exports.ENCRYPTION_CLASSES = {};
+export const ENCRYPTION_CLASSES = {};
 
 /**
  * map of registered encryption algorithm classes. Map from string to {@link
@@ -36,12 +34,11 @@ module.exports.ENCRYPTION_CLASSES = {};
  *
  * @type {Object.<string, function(new: module:crypto/algorithms/base.DecryptionAlgorithm)>}
  */
-module.exports.DECRYPTION_CLASSES = {};
+export const DECRYPTION_CLASSES = {};
 
 /**
  * base type for encryption implementations
  *
- * @constructor
  * @alias module:crypto/algorithms/base.EncryptionAlgorithm
  *
  * @param {object} params parameters
@@ -53,45 +50,45 @@ module.exports.DECRYPTION_CLASSES = {};
  * @param {string} params.roomId  The ID of the room we will be sending to
  * @param {object} params.config  The body of the m.room.encryption event
  */
-const EncryptionAlgorithm = function(params) {
-    this._userId = params.userId;
-    this._deviceId = params.deviceId;
-    this._crypto = params.crypto;
-    this._olmDevice = params.olmDevice;
-    this._baseApis = params.baseApis;
-    this._roomId = params.roomId;
-};
-/** */
-module.exports.EncryptionAlgorithm = EncryptionAlgorithm;
+class EncryptionAlgorithm {
+    constructor(params) {
+        this._userId = params.userId;
+        this._deviceId = params.deviceId;
+        this._crypto = params.crypto;
+        this._olmDevice = params.olmDevice;
+        this._baseApis = params.baseApis;
+        this._roomId = params.roomId;
+    }
 
-/**
- * Encrypt a message event
- *
- * @method module:crypto/algorithms/base.EncryptionAlgorithm#encryptMessage
- * @abstract
- *
- * @param {module:models/room} room
- * @param {string} eventType
- * @param {object} plaintext event content
- *
- * @return {module:client.Promise} Promise which resolves to the new event body
- */
+    /**
+     * Encrypt a message event
+     *
+     * @method module:crypto/algorithms/base.EncryptionAlgorithm.encryptMessage
+     * @abstract
+     *
+     * @param {module:models/room} room
+     * @param {string} eventType
+     * @param {object} plaintext event content
+     *
+     * @return {module:client.Promise} Promise which resolves to the new event body
+     */
 
-/**
- * Called when the membership of a member of the room changes.
- *
- * @param {module:models/event.MatrixEvent} event  event causing the change
- * @param {module:models/room-member} member  user whose membership changed
- * @param {string=} oldMembership  previous membership
- */
-EncryptionAlgorithm.prototype.onRoomMembership = function(
-    event, member, oldMembership,
-) {};
+    /**
+     * Called when the membership of a member of the room changes.
+     *
+     * @param {module:models/event.MatrixEvent} event  event causing the change
+     * @param {module:models/room-member} member  user whose membership changed
+     * @param {string=} oldMembership  previous membership
+     * @public
+     */
+     onRoomMembership(event, member, oldMembership) {
+     }
+}
+export {EncryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 /**
  * base type for decryption implementations
  *
- * @constructor
  * @alias module:crypto/algorithms/base.DecryptionAlgorithm
  *
  * @param {object} params parameters
@@ -101,78 +98,107 @@ EncryptionAlgorithm.prototype.onRoomMembership = function(
  * @param {string=} params.roomId The ID of the room we will be receiving
  *     from. Null for to-device events.
  */
-const DecryptionAlgorithm = function(params) {
-    this._userId = params.userId;
-    this._crypto = params.crypto;
-    this._olmDevice = params.olmDevice;
-    this._roomId = params.roomId;
-};
-/** */
-module.exports.DecryptionAlgorithm = DecryptionAlgorithm;
+class DecryptionAlgorithm {
+    constructor(params) {
+        this._userId = params.userId;
+        this._crypto = params.crypto;
+        this._olmDevice = params.olmDevice;
+        this._roomId = params.roomId;
+    }
 
-/**
- * Decrypt an event
- *
- * @method module:crypto/algorithms/base.DecryptionAlgorithm#decryptEvent
- * @abstract
- *
- * @param {object} event raw event
- *
- * @return {null} if the event referred to an unknown megolm session
- * @return {module:crypto.DecryptionResult} decryption result
- *
- * @throws {module:crypto/algorithms/base.DecryptionError} if there is a
- *   problem decrypting the event
- */
+    /**
+     * Decrypt an event
+     *
+     * @method module:crypto/algorithms/base.DecryptionAlgorithm#decryptEvent
+     * @abstract
+     *
+     * @param {object} event raw event
+     *
+     * @return {null} if the event referred to an unknown megolm session
+     * @return {module:crypto.DecryptionResult} decryption result
+     *
+     * @throws {module:crypto/algorithms/base.DecryptionError} if there is a
+     *   problem decrypting the event
+     */
 
-/**
- * Handle a key event
- *
- * @method module:crypto/algorithms/base.DecryptionAlgorithm#onRoomKeyEvent
- *
- * @param {module:models/event.MatrixEvent} params event key event
- */
-DecryptionAlgorithm.prototype.onRoomKeyEvent = function(params) {
-    // ignore by default
-};
+    /**
+     * Handle a key event
+     *
+     * @method module:crypto/algorithms/base.DecryptionAlgorithm#onRoomKeyEvent
+     *
+     * @param {module:models/event.MatrixEvent} params event key event
+     */
+    onRoomKeyEvent(params) {
+        // ignore by default
+    }
 
-/**
- * Import a room key
- *
- * @param {module:crypto/OlmDevice.MegolmSessionData} session
- */
-DecryptionAlgorithm.prototype.importRoomKey = function(session) {
-    // ignore by default
-};
+    /**
+     * Import a room key
+     *
+     * @param {module:crypto/OlmDevice.MegolmSessionData} session
+     */
+    importRoomKey(session) {
+        // ignore by default
+    }
+}
+export {DecryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 /**
  * Exception thrown when decryption fails
  *
- * @constructor
- * @param {string} msg message describing the problem
+ * @alias module:crypto/algorithms/base.DecryptionError
+ * @param {string} msg user-visible message describing the problem
+ *
+ * @param {Object=} details key/value pairs reported in the logs but not shown
+ *   to the user.
+ *
  * @extends Error
  */
-module.exports.DecryptionError = function(msg) {
-    this.message = msg;
-};
-utils.inherits(module.exports.DecryptionError, Error);
+class DecryptionError extends Error {
+    constructor(msg, details) {
+        super(msg);
+        this.name = 'DecryptionError';
+        this.details = details;
+    }
+
+    /**
+     * override the string used when logging
+     *
+     * @returns {String}
+     */
+    toString() {
+        let result = this.name + '[msg: ' + this.message;
+
+        if (this.details) {
+            result += ', ' +
+                Object.keys(this.details).map(
+                    (k) => k + ': ' + this.details[k],
+                ).join(', ');
+        }
+
+        result += ']';
+
+        return result;
+    }
+}
+export {DecryptionError}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 /**
  * Exception thrown specifically when we want to warn the user to consider
  * the security of their conversation before continuing
  *
- * @constructor
  * @param {string} msg message describing the problem
  * @param {Object} devices userId -> {deviceId -> object}
  *      set of unknown devices per user we're warning about
  * @extends Error
  */
-module.exports.UnknownDeviceError = function(msg, devices) {
-    this.name = "UnknownDeviceError";
-    this.message = msg;
-    this.devices = devices;
-};
-utils.inherits(module.exports.UnknownDeviceError, Error);
+export class UnknownDeviceError extends Error {
+    constructor(msg, devices) {
+        super(msg);
+        this.name = "UnknownDeviceError";
+        this.devices = devices;
+    }
+}
 
 /**
  * Registers an encryption/decryption class for a particular algorithm
@@ -187,7 +213,7 @@ utils.inherits(module.exports.UnknownDeviceError, Error);
  *     module:crypto/algorithms/base.DecryptionAlgorithm|DecryptionAlgorithm}
  *     implementation
  */
-module.exports.registerAlgorithm = function(algorithm, encryptor, decryptor) {
-    module.exports.ENCRYPTION_CLASSES[algorithm] = encryptor;
-    module.exports.DECRYPTION_CLASSES[algorithm] = decryptor;
-};
+export function registerAlgorithm(algorithm, encryptor, decryptor) {
+    ENCRYPTION_CLASSES[algorithm] = encryptor;
+    DECRYPTION_CLASSES[algorithm] = decryptor;
+}

src/crypto/algorithms/megolm.js

@@ -183,6 +183,7 @@ MegolmEncryption.prototype._ensureOutboundSession = function(devicesInRoom) {
         }
 
         if (!session) {
+            console.log(`Starting new megolm session for room ${self._roomId}`);
             session = self._prepareNewSession();
         }
 
@@ -353,6 +354,8 @@ MegolmEncryption.prototype._shareKeyWithDevices = function(session, devicesByUse
         // TODO: retries
         return self._baseApis.sendToDevice("m.room.encrypted", contentMap);
     }).then(function() {
+        console.log(`Completed megolm keyshare in ${self._roomId}`);
+
         // Add the devices we have shared with to session.sharedWithDevices.
         //
         // we deliberately iterate over devicesByUser (ie, the devices we
@@ -387,6 +390,8 @@ MegolmEncryption.prototype._shareKeyWithDevices = function(session, devicesByUse
  */
 MegolmEncryption.prototype.encryptMessage = function(room, eventType, content) {
     const self = this;
+    console.log(`Starting to encrypt event for ${this._roomId}`);
+
     return this._getDevicesInRoom(room).then(function(devicesInRoom) {
         // check if any of these devices are not yet known to the user.
         // if so, warn the user so they can verify or ignore.
@@ -544,7 +549,11 @@ MegolmDecryption.prototype.decryptEvent = function(event) {
         if (e.message === 'OLM.UNKNOWN_MESSAGE_INDEX') {
             this._addEventToPendingList(event);
         }
-        throw new base.DecryptionError(e);
+        throw new base.DecryptionError(
+            e.toString(), {
+                session: content.sender_key + '|' + content.session_id,
+            },
+        );
     }
 
     if (res === null) {
@@ -552,6 +561,9 @@ MegolmDecryption.prototype.decryptEvent = function(event) {
         this._addEventToPendingList(event);
         throw new base.DecryptionError(
             "The sender's device has not sent us the keys for this message.",
+            {
+                session: content.sender_key + '|' + content.session_id,
+            },
         );
     }
 
@@ -593,24 +605,30 @@ MegolmDecryption.prototype._addEventToPendingList = function(event) {
  * @param {module:models/event.MatrixEvent} event key event
  */
 MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
-    console.log("Adding key from ", event);
     const content = event.getContent();
+    const senderKey = event.getSenderKey();
+    const sessionId = content.session_id;
 
     if (!content.room_id ||
-        !content.session_id ||
+        !sessionId ||
         !content.session_key
        ) {
         console.error("key event is missing fields");
         return;
     }
+    if (!senderKey) {
+        console.error("key event has no sender key (not encrypted?)");
+        return;
+    }
 
+    console.log(`Adding key for megolm session ${senderKey}|${sessionId}`);
     this._olmDevice.addInboundGroupSession(
-        content.room_id, event.getSenderKey(), content.session_id,
+        content.room_id, senderKey, sessionId,
         content.session_key, event.getKeysClaimed(),
     );
 
     // have another go at decrypting events sent with this session.
-    this._retryDecryption(event.getSenderKey, content.session_id);
+    this._retryDecryption(senderKey, sessionId);
 };
 
 

src/crypto/algorithms/olm.js

@@ -64,7 +64,7 @@ OlmEncryption.prototype._ensureSession = function(roomMembers) {
     }
 
     const self = this;
-    this._prepPromise = self._crypto.downloadKeys(roomMembers, true).then(function(res) {
+    this._prepPromise = self._crypto.downloadKeys(roomMembers).then(function(res) {
         return self._crypto.ensureOlmSessionsForUsers(roomMembers);
     }).then(function() {
         self._sessionPrepared = true;
@@ -174,12 +174,12 @@ OlmDecryption.prototype.decryptEvent = function(event) {
     try {
         payloadString = this._decryptMessage(deviceKey, message);
     } catch (e) {
-        console.warn(
-            "Failed to decrypt Olm event (id=" +
-                event.getId() + ") from " + deviceKey +
-                ": " + e.message,
+        throw new base.DecryptionError(
+            "Bad Encrypted Message", {
+                sender: deviceKey,
+                err: e,
+            },
         );
-        throw new base.DecryptionError("Bad Encrypted Message");
     }
 
     const payload = JSON.parse(payloadString);
@@ -187,22 +187,18 @@ OlmDecryption.prototype.decryptEvent = function(event) {
     // check that we were the intended recipient, to avoid unknown-key attack
     // https://github.com/vector-im/vector-web/issues/2483
     if (payload.recipient != this._userId) {
-        console.warn(
-            "Event " + event.getId() + ": Intended recipient " +
-            payload.recipient + " does not match our id " + this._userId,
-        );
         throw new base.DecryptionError(
             "Message was intented for " + payload.recipient,
         );
     }
 
-    if (payload.recipient_keys.ed25519 !=
-               this._olmDevice.deviceEd25519Key) {
-        console.warn(
-            "Event " + event.getId() + ": Intended recipient ed25519 key " +
-            payload.recipient_keys.ed25519 + " did not match ours",
+    if (payload.recipient_keys.ed25519 != this._olmDevice.deviceEd25519Key) {
+        throw new base.DecryptionError(
+            "Message not intended for this device", {
+                intended: payload.recipient_keys.ed25519,
+                our_key: this._olmDevice.deviceEd25519Key,
+            },
         );
-        throw new base.DecryptionError("Message not intended for this device");
     }
 
     // check that the original sender matches what the homeserver told us, to
@@ -210,23 +206,19 @@ OlmDecryption.prototype.decryptEvent = function(event) {
     // (this check is also provided via the sender's embedded ed25519 key,
     // which is checked elsewhere).
     if (payload.sender != event.getSender()) {
-        console.warn(
-            "Event " + event.getId() + ": original sender " + payload.sender +
-            " does not match reported sender " + event.getSender(),
-        );
         throw new base.DecryptionError(
-            "Message forwarded from " + payload.sender,
+            "Message forwarded from " + payload.sender, {
+                reported_sender: event.getSender(),
+            },
         );
     }
 
     // Olm events intended for a room have a room_id.
     if (payload.room_id !== event.getRoomId()) {
-        console.warn(
-            "Event " + event.getId() + ": original room " + payload.room_id +
-            " does not match reported room " + event.room_id,
-        );
         throw new base.DecryptionError(
-            "Message intended for room " + payload.room_id,
+            "Message intended for room " + payload.room_id, {
+                reported_room: event.room_id,
+            },
         );
     }
 

src/crypto/index.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,7 +16,6 @@ limitations under the License.
 */
 "use strict";
 
-
 /**
  * @module crypto
  */
@@ -50,17 +50,29 @@ const DeviceList = require('./DeviceList').default;
  * @param {string} deviceId The identifier for this device.
  *
  * @param {Object} clientStore the MatrixClient data store.
+ *
+ * @param {module:crypto/store/base~CryptoStore} cryptoStore
+ *    storage for the crypto layer.
  */
 function Crypto(baseApis, eventEmitter, sessionStore, userId, deviceId,
-                clientStore) {
+                clientStore, cryptoStore) {
     this._baseApis = baseApis;
     this._sessionStore = sessionStore;
     this._userId = userId;
     this._deviceId = deviceId;
     this._clientStore = clientStore;
+    this._cryptoStore = cryptoStore;
 
     this._olmDevice = new OlmDevice(sessionStore);
     this._deviceList = new DeviceList(baseApis, sessionStore, this._olmDevice);
+    this._initialDeviceListInvalidationPending = false;
+
+    this._clientRunning = false;
+
+    // the last time we did a check for the number of one-time-keys on the
+    // server.
+    this._lastOneTimeKeyCheck = null;
+    this._oneTimeKeyCheckInProgress = false;
 
     // EncryptionAlgorithm instance for each room
     this._roomEncryptors = {};
@@ -86,9 +98,6 @@ function Crypto(baseApis, eventEmitter, sessionStore, userId, deviceId,
     );
 
     if (!myDevices) {
-        // we don't yet have a list of our own devices; make sure we
-        // get one when we flush the pendingUsersWithNewDevices.
-        this._deviceList.invalidateUserDeviceList(this._userId);
         myDevices = {};
     }
 
@@ -113,6 +122,11 @@ function Crypto(baseApis, eventEmitter, sessionStore, userId, deviceId,
 function _registerEventHandlers(crypto, eventEmitter) {
     eventEmitter.on("sync", function(syncState, oldState, data) {
         try {
+            if (syncState === "STOPPED") {
+                crypto._clientRunning = false;
+            } else if (syncState === "PREPARED") {
+                crypto._clientRunning = true;
+            }
             if (syncState === "SYNCING") {
                 crypto._onSyncCompleted(data);
             }
@@ -189,61 +203,11 @@ Crypto.prototype.getGlobalBlacklistUnverifiedDevices = function() {
 };
 
 /**
- * Upload the device keys to the homeserver and ensure that the
- * homeserver has enough one-time keys.
- * @param {number} maxKeys The maximum number of keys to generate
+ * Upload the device keys to the homeserver.
  * @return {object} A promise that will resolve when the keys are uploaded.
  */
-Crypto.prototype.uploadKeys = function(maxKeys) {
-    const self = this;
-    return _uploadDeviceKeys(this).then(function(res) {
-        // We need to keep a pool of one time public keys on the server so that
-        // other devices can start conversations with us. But we can only store
-        // a finite number of private keys in the olm Account object.
-        // To complicate things further then can be a delay between a device
-        // claiming a public one time key from the server and it sending us a
-        // message. We need to keep the corresponding private key locally until
-        // we receive the message.
-        // But that message might never arrive leaving us stuck with duff
-        // private keys clogging up our local storage.
-        // So we need some kind of enginering compromise to balance all of
-        // these factors.
-
-        // We first find how many keys the server has for us.
-        const keyCount = res.one_time_key_counts.signed_curve25519 || 0;
-        // We then check how many keys we can store in the Account object.
-        const maxOneTimeKeys = self._olmDevice.maxNumberOfOneTimeKeys();
-        // Try to keep at most half that number on the server. This leaves the
-        // rest of the slots free to hold keys that have been claimed from the
-        // server but we haven't recevied a message for.
-        // If we run out of slots when generating new keys then olm will
-        // discard the oldest private keys first. This will eventually clean
-        // out stale private keys that won't receive a message.
-        const keyLimit = Math.floor(maxOneTimeKeys / 2);
-        // We work out how many new keys we need to create to top up the server
-        // If there are too many keys on the server then we don't need to
-        // create any more keys.
-        let numberToGenerate = Math.max(keyLimit - keyCount, 0);
-        if (maxKeys !== undefined) {
-            // Creating keys can be an expensive operation so we limit the
-            // number we generate in one go to avoid blocking the application
-            // for too long.
-            numberToGenerate = Math.min(numberToGenerate, maxKeys);
-        }
-
-        if (numberToGenerate <= 0) {
-            // If we don't need to generate any keys then we are done.
-            return;
-        }
-
-        // Ask olm to generate new one time keys, then upload them to synapse.
-        self._olmDevice.generateOneTimeKeys(numberToGenerate);
-        return _uploadOneTimeKeys(self);
-    });
-};
-
-// returns a promise which resolves to the response
-function _uploadDeviceKeys(crypto) {
+Crypto.prototype.uploadDeviceKeys = function() {
+    const crypto = this;
     const userId = crypto._userId;
     const deviceId = crypto._deviceId;
 
@@ -262,6 +226,90 @@ function _uploadDeviceKeys(crypto) {
         // same one as used in login.
         device_id: deviceId,
     });
+};
+
+// check if it's time to upload one-time keys, and do so if so.
+function _maybeUploadOneTimeKeys(crypto) {
+    // frequency with which to check & upload one-time keys
+    const uploadPeriod = 1000 * 60; // one minute
+
+    // max number of keys to upload at once
+    // Creating keys can be an expensive operation so we limit the
+    // number we generate in one go to avoid blocking the application
+    // for too long.
+    const maxKeysPerCycle = 5;
+
+    if (crypto._oneTimeKeyCheckInProgress) {
+        return;
+    }
+
+    const now = Date.now();
+    if (crypto._lastOneTimeKeyCheck !== null &&
+        now - crypto._lastOneTimeKeyCheck < uploadPeriod
+       ) {
+        // we've done a key upload recently.
+        return;
+    }
+
+    crypto._lastOneTimeKeyCheck = now;
+
+    function uploadLoop(numberToGenerate) {
+        if (numberToGenerate <= 0) {
+            // If we don't need to generate any more keys then we are done.
+            return;
+        }
+
+        const keysThisLoop = Math.min(numberToGenerate, maxKeysPerCycle);
+
+        // Ask olm to generate new one time keys, then upload them to synapse.
+        crypto._olmDevice.generateOneTimeKeys(keysThisLoop);
+        return _uploadOneTimeKeys(crypto).then(() => {
+            return uploadLoop(numberToGenerate - keysThisLoop);
+        });
+    }
+
+    crypto._oneTimeKeyCheckInProgress = true;
+    q().then(() => {
+        // ask the server how many keys we have
+        return crypto._baseApis.uploadKeysRequest({}, {
+            device_id: crypto._deviceId,
+        });
+    }).then((res) => {
+        // We need to keep a pool of one time public keys on the server so that
+        // other devices can start conversations with us. But we can only store
+        // a finite number of private keys in the olm Account object.
+        // To complicate things further then can be a delay between a device
+        // claiming a public one time key from the server and it sending us a
+        // message. We need to keep the corresponding private key locally until
+        // we receive the message.
+        // But that message might never arrive leaving us stuck with duff
+        // private keys clogging up our local storage.
+        // So we need some kind of enginering compromise to balance all of
+        // these factors.
+
+        // We first find how many keys the server has for us.
+        const keyCount = res.one_time_key_counts.signed_curve25519 || 0;
+        // We then check how many keys we can store in the Account object.
+        const maxOneTimeKeys = crypto._olmDevice.maxNumberOfOneTimeKeys();
+        // Try to keep at most half that number on the server. This leaves the
+        // rest of the slots free to hold keys that have been claimed from the
+        // server but we haven't recevied a message for.
+        // If we run out of slots when generating new keys then olm will
+        // discard the oldest private keys first. This will eventually clean
+        // out stale private keys that won't receive a message.
+        const keyLimit = Math.floor(maxOneTimeKeys / 2);
+
+        // We work out how many new keys we need to create to top up the server
+        // If there are too many keys on the server then we don't need to
+        // create any more keys.
+        const numberToGenerate = Math.max(keyLimit - keyCount, 0);
+
+        return uploadLoop(numberToGenerate);
+    }).catch((e) => {
+        console.error("Error uploading one-time keys", e.stack || e);
+    }).finally(() => {
+        crypto._oneTimeKeyCheckInProgress = false;
+    }).done();
 }
 
 // returns a promise which resolves to the response
@@ -464,19 +512,19 @@ Crypto.prototype.getOlmSessionsForUser = function(userId) {
  * @return {module:crypto/deviceinfo?}
  */
 Crypto.prototype.getEventSenderDeviceInfo = function(event) {
-    const sender_key = event.getSenderKey();
+    const senderKey = event.getSenderKey();
     const algorithm = event.getWireContent().algorithm;
 
-    if (!sender_key || !algorithm) {
+    if (!senderKey || !algorithm) {
         return null;
     }
 
-    // sender_key is the Curve25519 identity key of the device which the event
+    // senderKey is the Curve25519 identity key of the device which the event
     // was sent from. In the case of Megolm, it's actually the Curve25519
     // identity key of the device which set up the Megolm session.
 
     const device = this._deviceList.getDeviceByIdentityKey(
-        event.getSender(), algorithm, sender_key,
+        event.getSender(), algorithm, senderKey,
     );
 
     if (device === null) {
@@ -514,9 +562,13 @@ Crypto.prototype.getEventSenderDeviceInfo = function(event) {
  * Configure a room to use encryption (ie, save a flag in the sessionstore).
  *
  * @param {string} roomId The room ID to enable encryption in.
+ *
  * @param {object} config The encryption config for the room.
+ *
+ * @param {boolean=} inhibitDeviceQuery true to suppress device list query for
+ *   users in the room (for now)
  */
-Crypto.prototype.setRoomEncryption = function(roomId, config) {
+Crypto.prototype.setRoomEncryption = function(roomId, config, inhibitDeviceQuery) {
     // if we already have encryption in this room, we should ignore this event
     // (for now at least. maybe we should alert the user somehow?)
     const existingConfig = this._sessionStore.getEndToEndRoom(roomId);
@@ -545,6 +597,18 @@ Crypto.prototype.setRoomEncryption = function(roomId, config) {
         config: config,
     });
     this._roomEncryptors[roomId] = alg;
+
+    // make sure we are tracking the device lists for all users in this room.
+    console.log("Enabling encryption in " + roomId + "; " +
+                "starting to track device lists for all users therein");
+    const room = this._clientStore.getRoom(roomId);
+    const members = room.getJoinedMembers();
+    members.forEach((m) => {
+        this._deviceList.startTrackingDeviceList(m.userId);
+    });
+    if (!inhibitDeviceQuery) {
+        this._deviceList.refreshOutdatedDeviceLists();
+    }
 };
 
 
@@ -734,7 +798,9 @@ Crypto.prototype._onCryptoEvent = function(event) {
     const content = event.getContent();
 
     try {
-        this.setRoomEncryption(roomId, content);
+        // inhibit the device list refresh for now - it will happen once we've
+        // finished processing the sync, in _onSyncCompleted.
+        this.setRoomEncryption(roomId, content, true);
     } catch (e) {
         console.error("Error configuring encryption in room " + roomId +
                       ":", e);
@@ -750,31 +816,59 @@ Crypto.prototype._onCryptoEvent = function(event) {
  * @param {Object} syncData  the data from the 'MatrixClient.sync' event
  */
 Crypto.prototype._onSyncCompleted = function(syncData) {
-    this._deviceList.lastKnownSyncToken = syncData.nextSyncToken;
+    const nextSyncToken = syncData.nextSyncToken;
 
     if (!syncData.oldSyncToken) {
+        console.log("Completed initial sync");
+
         // an initialsync.
         this._sendNewDeviceEvents();
 
         // if we have a deviceSyncToken, we can tell the deviceList to
         // invalidate devices which have changed since then.
         const oldSyncToken = this._sessionStore.getEndToEndDeviceSyncToken();
-        if (oldSyncToken) {
-            this._invalidateDeviceListsSince(oldSyncToken).catch((e) => {
+        if (oldSyncToken !== null) {
+            this._initialDeviceListInvalidationPending = true;
+            this._invalidateDeviceListsSince(
+                oldSyncToken, nextSyncToken,
+            ).catch((e) => {
                 // if that failed, we fall back to invalidating everyone.
                 console.warn("Error fetching changed device list", e);
-                this._invalidateDeviceListForAllActiveUsers();
-                return this._deviceList.refreshOutdatedDeviceLists();
-            }).done();
+                this._deviceList.invalidateAllDeviceLists();
+            }).done(() => {
+                this._initialDeviceListInvalidationPending = false;
+                this._deviceList.lastKnownSyncToken = nextSyncToken;
+                this._deviceList.refreshOutdatedDeviceLists();
+            });
         } else {
             // otherwise, we have to invalidate all devices for all users we
-            // share a room with.
-            this._invalidateDeviceListForAllActiveUsers();
+            // are tracking.
+            console.log("Completed first initialsync; invalidating all " +
+                        "device list caches");
+            this._deviceList.invalidateAllDeviceLists();
         }
     }
 
+    if (!this._initialDeviceListInvalidationPending) {
+        // we can now store our sync token so that we can get an update on
+        // restart rather than having to invalidate everyone.
+        //
+        // (we don't really need to do this on every sync - we could just
+        // do it periodically)
+        this._sessionStore.storeEndToEndDeviceSyncToken(nextSyncToken);
+    }
+
     // catch up on any new devices we got told about during the sync.
-    this._deviceList.refreshOutdatedDeviceLists().done();
+    this._deviceList.lastKnownSyncToken = nextSyncToken;
+    this._deviceList.refreshOutdatedDeviceLists();
+
+    // we don't start uploading one-time keys until we've caught up with
+    // to-device messages, to help us avoid throwing away one-time-keys that we
+    // are about to receive messages for
+    // (https://github.com/vector-im/riot-web/issues/2782).
+    if (!syncData.catchingUp) {
+        _maybeUploadOneTimeKeys(this);
+    }
 };
 
 /**
@@ -830,61 +924,32 @@ Crypto.prototype._sendNewDeviceEvents = function() {
 
 /**
  * Ask the server which users have new devices since a given token,
- * invalidate them, and start an update query.
+ * and invalidate them
  *
  * @param {String} oldSyncToken
+ * @param {String} lastKnownSyncToken
  *
  * @returns {Promise} resolves once the query is complete. Rejects if the
  *   keyChange query fails.
  */
-Crypto.prototype._invalidateDeviceListsSince = function(oldSyncToken) {
+Crypto.prototype._invalidateDeviceListsSince = function(
+    oldSyncToken, lastKnownSyncToken,
+) {
     return this._baseApis.getKeyChanges(
-        oldSyncToken, this.lastKnownSyncToken,
+        oldSyncToken, lastKnownSyncToken,
     ).then((r) => {
+        console.log("got key changes since", oldSyncToken, ":", r.changed);
+
         if (!r.changed || !Array.isArray(r.changed)) {
             return;
         }
 
-        // only invalidate users we share an e2e room with - we don't
-        // care about users in non-e2e rooms.
-        const filteredUserIds = this._getE2eRoomMembers();
         r.changed.forEach((u) => {
-            if (u in filteredUserIds) {
-                this._deviceList.invalidateUserDeviceList(u);
-            }
+            this._deviceList.invalidateUserDeviceList(u);
         });
-        return this._deviceList.refreshOutdatedDeviceLists();
     });
 };
 
-/**
- * Invalidate any stored device list for any users we share an e2e room with
- *
- * @private
- */
-Crypto.prototype._invalidateDeviceListForAllActiveUsers = function() {
-    Object.keys(this._getE2eRoomMembers()).forEach((m) => {
-        this._deviceList.invalidateUserDeviceList(m);
-    });
-};
-
-/**
- * get the users we share an e2e-enabled room with
- *
- * @returns {Object<string>} userid->userid map (should be a Set but argh ES6)
- */
-Crypto.prototype._getE2eRoomMembers = function() {
-    const userIds = Object.create(null);
-
-    const rooms = this._getE2eRooms();
-    for (const r of rooms) {
-        const members = r.getJoinedMembers();
-        members.forEach((m) => { userIds[m.userId] = m.userId; });
-    }
-
-    return userIds;
-};
-
 /**
  * Get a list of the e2e-enabled rooms we are members of
  *
@@ -953,6 +1018,12 @@ Crypto.prototype._onRoomMembership = function(event, member, oldMembership) {
         return;
     }
 
+    if (member.membership == 'join') {
+        console.log('Join event for ' + member.userId + ' in ' + roomId);
+        // make sure we are tracking the deviceList for this user
+        this._deviceList.startTrackingDeviceList(member.userId);
+    }
+
     alg.onRoomMembership(event, member, oldMembership);
 };
 
@@ -1054,11 +1125,6 @@ Crypto.prototype._signObject = function(obj) {
     obj.signatures = sigs;
 };
 
-/**
- * @see module:crypto/algorithms/base.DecryptionError
- */
-Crypto.DecryptionError = algorithms.DecryptionError;
-
 
 /** */
 module.exports = Crypto;

src/crypto/store/base.js

@@ -0,0 +1,11 @@
+/**
+ * Internal module. Defintions for storage for the crypto module
+ *
+ * @module
+ */
+
+/**
+ * Abstraction of things that can store data required for end-to-end encryption
+ *
+ * @interface CryptoStore
+ */

src/crypto/store/indexeddb-crypto-store.js

@@ -0,0 +1,143 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import q from 'q';
+
+/**
+ * Internal module. indexeddb storage for e2e.
+ *
+ * @module
+ */
+
+const VERSION = 1;
+
+/**
+ * @implements {module:crypto/store/base~CryptoStore}
+ */
+export default class IndexedDBCryptoStore {
+    /**
+     * Create a new IndexedDBCryptoStore
+     *
+     * @param {IDBFactory} indexedDB  global indexedDB instance
+     * @param {string} dbName   name of db to connect to
+     */
+    constructor(indexedDB, dbName) {
+        if (!indexedDB) {
+            throw new Error("must pass indexedDB into IndexedDBCryptoStore");
+        }
+        this._indexedDB = indexedDB;
+        this._dbName = dbName;
+        this._dbPromise = null;
+    }
+
+    /**
+     * Ensure the database exists and is up-to-date
+     *
+     * @return {Promise} resolves to an instance of IDBDatabase when
+     * the database is ready
+     */
+    connect() {
+        if (this._dbPromise) {
+            return this._dbPromise;
+        }
+
+        this._dbPromise = new q.Promise((resolve, reject) => {
+            console.log(`connecting to indexeddb ${this._dbName}`);
+            const req = this._indexedDB.open(this._dbName, VERSION);
+
+            req.onupgradeneeded = (ev) => {
+                const db = ev.target.result;
+                const oldVersion = ev.oldVersion;
+                console.log(
+                    `Upgrading IndexedDBCryptoStore from version ${oldVersion}`
+                    + ` to ${VERSION}`,
+                );
+                if (oldVersion < 1) { // The database did not previously exist.
+                    createDatabase(db);
+                }
+                // Expand as needed.
+            };
+
+            req.onblocked = () => {
+                reject(new Error(
+                    "unable to upgrade indexeddb because it is open elsewhere",
+                ));
+            };
+
+            req.onerror = (ev) => {
+                reject(new Error(
+                    "unable to connect to indexeddb: " + ev.target.error,
+                ));
+            };
+
+            req.onsuccess = (r) => {
+                const db = r.target.result;
+
+                // make sure we close the db on `onversionchange` - otherwise
+                // attempts to delete the database will block (and subsequent
+                // attempts to re-create it will also block).
+                db.onversionchange = (ev) => {
+                    db.close();
+                };
+
+                resolve(db);
+            };
+        });
+        return this._dbPromise;
+    }
+
+    /**
+     * Delete all data from this store.
+     *
+     * @returns {Promise} resolves when the store has been cleared.
+     */
+    deleteAllData() {
+        return new q.Promise((resolve, reject) => {
+            console.log(`Removing indexeddb instance: ${this._dbName}`);
+            const req = this._indexedDB.deleteDatabase(this._dbName);
+
+            req.onblocked = () => {
+                reject(new Error(
+                    "unable to delete indexeddb because it is open elsewhere",
+                ));
+            };
+
+            req.onerror = (ev) => {
+                reject(new Error(
+                    "unable to delete indexeddb: " + ev.target.error,
+                ));
+            };
+
+            req.onsuccess = () => {
+                console.log(`Removed indexeddb instance: ${this._dbName}`);
+                resolve();
+            };
+        });
+    }
+}
+
+function createDatabase(db) {
+    const outgoingRoomKeyRequestsStore =
+        db.createObjectStore("outgoingRoomKeyRequests", { keyPath: "requestId" });
+
+    // we assume that the RoomKeyRequestBody will have room_id and session_id
+    // properties, to make the index efficient.
+    outgoingRoomKeyRequestsStore.createIndex("session",
+        ["requestBody.room_id", "requestBody.session_id"],
+    );
+
+    outgoingRoomKeyRequestsStore.createIndex("state", "state");
+}

src/crypto/store/memory-crypto-store.js

@@ -0,0 +1,40 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import q from 'q';
+
+/**
+ * Internal module. in-memory storage for e2e.
+ *
+ * @module
+ */
+
+/**
+ * @implements {module:crypto/store/base~CryptoStore}
+ */
+export default class MemoryCryptoStore {
+    constructor() {
+    }
+
+    /**
+     * Delete all data from this store.
+     *
+     * @returns {Promise} Promise which resolves when the store has been cleared.
+     */
+    deleteAllData() {
+        return q();
+    }
+}

src/http-api.js

@@ -631,19 +631,26 @@ module.exports.MatrixHttpApi.prototype = {
         let timedOut = false;
         let req;
         const localTimeoutMs = opts.localTimeoutMs || this.opts.localTimeoutMs;
-        if (localTimeoutMs) {
-            timeoutId = callbacks.setTimeout(function() {
-                timedOut = true;
-                if (req && req.abort) {
-                    req.abort();
+
+        const resetTimeout = () => {
+            if (localTimeoutMs) {
+                if (timeoutId) {
+                    callbacks.clearTimeout(timeoutId);
                 }
-                defer.reject(new module.exports.MatrixError({
-                    error: "Locally timed out waiting for a response",
-                    errcode: "ORG.MATRIX.JSSDK_TIMEOUT",
-                    timeout: localTimeoutMs,
-                }));
-            }, localTimeoutMs);
-        }
+                timeoutId = callbacks.setTimeout(function() {
+                    timedOut = true;
+                    if (req && req.abort) {
+                        req.abort();
+                    }
+                    defer.reject(new module.exports.MatrixError({
+                        error: "Locally timed out waiting for a response",
+                        errcode: "ORG.MATRIX.JSSDK_TIMEOUT",
+                        timeout: localTimeoutMs,
+                    }));
+                }, localTimeoutMs);
+            }
+        };
+        resetTimeout();
 
         const reqPromise = defer.promise;
 
@@ -679,10 +686,22 @@ module.exports.MatrixHttpApi.prototype = {
                     handlerFn(err, response, body);
                 },
             );
-            if (req && req.abort) {
+            if (req) {
+                // This will only work in a browser, where opts.request is the
+                // `browser-request` import. Currently `request` does not support progress
+                // updates - see https://github.com/request/request/pull/2346.
+                // `browser-request` returns an XHRHttpRequest which exposes `onprogress`
+                if ('onprogress' in req) {
+                    req.onprogress = (e) => {
+                        // Prevent the timeout from rejecting the deferred promise if progress is
+                        // seen with the request
+                        resetTimeout();
+                    };
+                }
+
                 // FIXME: This is EVIL, but I can't think of a better way to expose
                 // abort() operations on underlying HTTP requests :(
-                reqPromise.abort = req.abort.bind(req);
+                if (req.abort) reqPromise.abort = req.abort.bind(req);
             }
         } catch (ex) {
             defer.reject(ex);

src/indexeddb-worker.js

@@ -0,0 +1,24 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Separate exports file for the indexeddb web worker, which is designed
+ * to be used separately
+ */
+
+/** The {@link module:indexeddb-store-worker~IndexedDBStoreWorker} class. */
+module.exports.IndexedDBStoreWorker = require("./store/indexeddb-store-worker.js");
+

src/interactive-auth.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,9 +18,13 @@ limitations under the License.
 
 /** @module interactive-auth */
 const q = require("q");
+const url = require("url");
 
 const utils = require("./utils");
 
+const EMAIL_STAGE_TYPE = "m.login.email.identity";
+const MSISDN_STAGE_TYPE = "m.login.msisdn";
+
 /**
  * Abstracts the logic used to drive the interactive auth process.
  *
@@ -37,44 +42,123 @@ const utils = require("./utils");
  *
  * @param {object} opts  options object
  *
+ * @param {object} opts.matrixClient A matrix client to use for the auth process
+ *
  * @param {object?} opts.authData error response from the last request. If
  *    null, a request will be made with no auth before starting.
  *
- * @param {function(object?): module:client.Promise} opts.doRequest
- *     called with the new auth dict to submit the request. Should return a
+ * @param {function(object?, bool?): module:client.Promise} opts.doRequest
+ *     called with the new auth dict to submit the request and a flag set
+ *     to true if this request is a background request. Should return a
  *     promise which resolves to the successful response or rejects with a
  *     MatrixError.
  *
- * @param {function(string, object?)} opts.startAuthStage
- *     called to ask the UI to start a particular auth stage. The arguments
- *     are: the login type (eg m.login.password); and (if the last request
- *     returned an error), an error object, with fields 'errcode' and 'error'.
+ * @param {function(string, object?)} opts.stateUpdated
+ *     called when the status of the UI auth changes, ie. when the state of
+ *     an auth stage changes of when the auth flow moves to a new stage.
+ *     The arguments are: the login type (eg m.login.password); and an object
+ *     which is either an error or an informational object specific to the
+ *     login type. If the 'errcode' key is defined, the object is an error,
+ *     and has keys:
+ *         errcode: string, the textual error code, eg. M_UNKNOWN
+ *         error: string, human readable string describing the error
+ *
+ *     The login type specific objects are as follows:
+ *         m.login.email.identity:
+ *          * emailSid: string, the sid of the active email auth session
+ *
+ * @param {object?} opts.inputs Inputs provided by the user and used by different
+ *     stages of the auto process. The inputs provided will affect what flow is chosen.
+ *
+ * @param {string?} opts.inputs.emailAddress An email address. If supplied, a flow
+ *     using email verification will be chosen.
+ *
+ * @param {string?} opts.inputs.phoneCountry An ISO two letter country code. Gives
+ *     the country that opts.phoneNumber should be resolved relative to.
+ *
+ * @param {string?} opts.inputs.phoneNumber A phone number. If supplied, a flow
+ *     using phone number validation will be chosen.
+ *
+ * @param {string?} opts.sessionId If resuming an existing interactive auth session,
+ *     the sessionId of that session.
+ *
+ * @param {string?} opts.clientSecret If resuming an existing interactive auth session,
+ *     the client secret for that session
+ *
+ * @param {string?} opts.emailSid If returning from having completed m.login.email.identity
+ *     auth, the sid for the email verification session.
  *
  */
 function InteractiveAuth(opts) {
-    this._data = opts.authData;
+    this._matrixClient = opts.matrixClient;
+    this._data = opts.authData || {};
     this._requestCallback = opts.doRequest;
-    this._startAuthStageCallback = opts.startAuthStage;
+    // startAuthStage included for backwards compat
+    this._stateUpdatedCallback = opts.stateUpdated || opts.startAuthStage;
     this._completionDeferred = null;
+    this._inputs = opts.inputs || {};
+
+    if (opts.sessionId) this._data.session = opts.sessionId;
+    this._clientSecret = opts.clientSecret || this._matrixClient.generateClientSecret();
+    this._emailSid = opts.emailSid;
+    if (this._emailSid === undefined) this._emailSid = null;
+
+    this._currentStage = null;
 }
 
 InteractiveAuth.prototype = {
     /**
      * begin the authentication process.
      *
-     * @return {module:client.Promise}  which resolves to the response on success,
-     * or rejects with the error on failure.
+     * @return {module:client.Promise} which resolves to the response on success,
+     * or rejects with the error on failure. Rejects with NoAuthFlowFoundError if
+     *     no suitable authentication flow can be found
      */
     attemptAuth: function() {
         this._completionDeferred = q.defer();
 
-        if (!this._data) {
-            this._doRequest(null);
-        } else {
-            this._startNextAuthStage();
+        // wrap in a promise so that if _startNextAuthStage
+        // throws, it rejects the promise in a consistent way
+        return q().then(() => {
+            // if we have no flows, try a request (we'll have
+            // just a session ID in _data if resuming)
+            if (!this._data.flows) {
+                this._doRequest(this._data);
+            } else {
+                this._startNextAuthStage();
+            }
+            return this._completionDeferred.promise;
+        });
+    },
+
+    /**
+     * Poll to check if the auth session or current stage has been
+     * completed out-of-band. If so, the attemptAuth promise will
+     * be resolved.
+     */
+    poll: function() {
+        if (!this._data.session) return;
+
+        let authDict = {};
+        if (this._currentStage == EMAIL_STAGE_TYPE) {
+            // The email can be validated out-of-band, but we need to provide the
+            // creds so the HS can go & check it.
+            if (this._emailSid) {
+                const idServerParsedUrl = url.parse(
+                    this._matrixClient.getIdentityServerUrl(),
+                );
+                authDict = {
+                    type: EMAIL_STAGE_TYPE,
+                    threepid_creds: {
+                        sid: this._emailSid,
+                        client_secret: this._clientSecret,
+                        id_server: idServerParsedUrl.host,
+                    },
+                };
+            }
         }
 
-        return this._completionDeferred.promise;
+        this.submitAuthDict(authDict, true);
     },
 
     /**
@@ -86,6 +170,16 @@ InteractiveAuth.prototype = {
         return this._data ? this._data.session : undefined;
     },
 
+    /**
+     * get the client secret used for validation sessions
+     * with the ID server.
+     *
+     * @return {string} client secret
+     */
+    getClientSecret: function() {
+        return this._clientSecret;
+    },
+
     /**
      * get the server params for a given stage
      *
@@ -108,8 +202,11 @@ InteractiveAuth.prototype = {
      * @param {object} authData new auth dict to send to the server. Should
      *    include a `type` propterty denoting the login type, as well as any
      *    other params for that stage.
+     * @param {bool} background If true, this request failing will not result
+     *    in the attemptAuth promise being rejected. This can be set to true
+     *    for requests that just poll to see if auth has been completed elsewhere.
      */
-    submitAuthDict: function(authData) {
+    submitAuthDict: function(authData, background) {
         if (!this._completionDeferred) {
             throw new Error("submitAuthDict() called before attemptAuth()");
         }
@@ -120,7 +217,29 @@ InteractiveAuth.prototype = {
         };
         utils.extend(auth, authData);
 
-        this._doRequest(auth);
+        this._doRequest(auth, background);
+    },
+
+    /**
+     * Gets the sid for the email validation session
+     * Specific to m.login.email.identity
+     *
+     * @returns {string} The sid of the email auth session
+     */
+    getEmailSid: function() {
+        return this._emailSid;
+    },
+
+    /**
+     * Sets the sid for the email validation session
+     * This must be set in order to successfully poll for completion
+     * of the email validation.
+     * Specific to m.login.email.identity
+     *
+     * @param {string} sid The sid for the email validation session
+     */
+    setEmailSid: function(sid) {
+        this._emailSid = sid;
     },
 
     /**
@@ -129,8 +248,12 @@ InteractiveAuth.prototype = {
      *
      * @private
      * @param {object?} auth new auth dict, including session id
+     * @param {bool?} background If true, this request is a background poll, so it
+     *    failing will not result in the attemptAuth promise being rejected.
+     *    This can be set to true for requests that just poll to see if auth has
+     *    been completed elsewhere.
      */
-    _doRequest: function(auth) {
+    _doRequest: function(auth, background) {
         const self = this;
 
         // hackery to make sure that synchronous exceptions end up in the catch
@@ -138,45 +261,83 @@ InteractiveAuth.prototype = {
         // extra q().then)
         let prom;
         try {
-            prom = this._requestCallback(auth);
+            prom = this._requestCallback(auth, background);
         } catch (e) {
             prom = q.reject(e);
         }
 
-        prom.then(
+        prom = prom.then(
             function(result) {
                 console.log("result from request: ", result);
                 self._completionDeferred.resolve(result);
             }, function(error) {
-                if (error.httpStatus !== 401 || !error.data || !error.data.flows) {
+                // sometimes UI auth errors don't come with flows
+                const errorFlows = error.data ? error.data.flows : null;
+                const haveFlows = Boolean(self._data.flows) || Boolean(errorFlows);
+                if (error.httpStatus !== 401 || !error.data || !haveFlows) {
                     // doesn't look like an interactive-auth failure. fail the whole lot.
                     throw error;
                 }
+                // if the error didn't come with flows, completed flows or session ID,
+                // copy over the ones we have. Synapse sometimes sends responses without
+                // any UI auth data (eg. when polling for email validation, if the email
+                // has not yet been validated). This appears to be a Synapse bug, which
+                // we workaround here.
+                if (!error.data.flows && !error.data.completed && !error.data.session) {
+                    error.data.flows = self._data.flows;
+                    error.data.completed = self._data.completed;
+                    error.data.session = self._data.session;
+                }
                 self._data = error.data;
                 self._startNextAuthStage();
             },
-        ).catch(this._completionDeferred.reject).done();
+        );
+        if (!background) {
+            prom = prom.catch(this._completionDeferred.reject);
+        } else {
+            // We ignore all failures here (even non-UI auth related ones)
+            // since we don't want to suddenly fail if the internet connection
+            // had a blip whilst we were polling
+            prom = prom.catch((error) => {
+                console.log("Ignoring error from UI auth: " + error);
+            });
+        }
+        prom.done();
     },
 
     /**
      * Pick the next stage and call the callback
      *
      * @private
+     * @throws {NoAuthFlowFoundError} If no suitable authentication flow can be found
      */
     _startNextAuthStage: function() {
         const nextStage = this._chooseStage();
         if (!nextStage) {
             throw new Error("No incomplete flows from the server");
         }
+        this._currentStage = nextStage;
+
+        if (nextStage == 'm.login.dummy') {
+            this.submitAuthDict({
+                type: 'm.login.dummy',
+            });
+            return;
+        }
 
-        let stageError = null;
         if (this._data.errcode || this._data.error) {
-            stageError = {
+            this._stateUpdatedCallback(nextStage, {
                 errcode: this._data.errcode || "",
                 error: this._data.error || "",
-            };
+            });
+            return;
         }
-        this._startAuthStageCallback(nextStage, stageError);
+
+        const stageStatus = {};
+        if (nextStage == EMAIL_STAGE_TYPE) {
+            stageStatus.emailSid = this._emailSid;
+        }
+        this._stateUpdatedCallback(nextStage, stageStatus);
     },
 
     /**
@@ -184,6 +345,7 @@ InteractiveAuth.prototype = {
      *
      * @private
      * @return {string?} login type
+     * @throws {NoAuthFlowFoundError} If no suitable authentication flow can be found
      */
     _chooseStage: function() {
         const flow = this._chooseFlow();
@@ -195,14 +357,53 @@ InteractiveAuth.prototype = {
 
     /**
      * Pick one of the flows from the returned list
+     * If a flow using all of the inputs is found, it will
+     * be returned, otherwise, null will be returned.
+     *
+     * Only flows using all given inputs are chosen because it
+     * is likley to be surprising if the user provides a
+     * credential and it is not used. For example, for registration,
+     * this could result in the email not being used which would leave
+     * the account with no means to reset a password.
      *
      * @private
      * @return {object} flow
+     * @throws {NoAuthFlowFoundError} If no suitable authentication flow can be found
      */
     _chooseFlow: function() {
         const flows = this._data.flows || [];
-        // always use the first flow for now
-        return flows[0];
+
+        // we've been given an email or we've already done an email part
+        const haveEmail = Boolean(this._inputs.emailAddress) || Boolean(this._emailSid);
+        const haveMsisdn = (
+            Boolean(this._inputs.phoneCountry) &&
+            Boolean(this._inputs.phoneNumber)
+        );
+
+        for (const flow of flows) {
+            let flowHasEmail = false;
+            let flowHasMsisdn = false;
+            for (const stage of flow.stages) {
+                if (stage === EMAIL_STAGE_TYPE) {
+                    flowHasEmail = true;
+                } else if (stage == MSISDN_STAGE_TYPE) {
+                    flowHasMsisdn = true;
+                }
+            }
+
+            if (flowHasEmail == haveEmail && flowHasMsisdn == haveMsisdn) {
+                return flow;
+            }
+        }
+        // Throw an error with a fairly generic description, but with more
+        // information such that the app can give a better one if so desired.
+        const err = new Error("No appropriate authentication flow found");
+        err.name = 'NoAuthFlowFoundError';
+        err.required_stages = [];
+        if (haveEmail) err.required_stages.push(EMAIL_STAGE_TYPE);
+        if (haveMsisdn) err.required_stages.push(MSISDN_STAGE_TYPE);
+        err.available_flows = flows;
+        throw err;
     },
 
     /**

src/matrix.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,6 +22,12 @@ module.exports.MatrixEvent = require("./models/event").MatrixEvent;
 module.exports.EventStatus = require("./models/event").EventStatus;
 /** The {@link module:store/memory.MatrixInMemoryStore|MatrixInMemoryStore} class. */
 module.exports.MatrixInMemoryStore = require("./store/memory").MatrixInMemoryStore;
+/** The {@link module:store/indexeddb.IndexedDBStore|IndexedDBStore} class. */
+module.exports.IndexedDBStore = require("./store/indexeddb").IndexedDBStore;
+/** The {@link module:store/indexeddb.IndexedDBStoreBackend|IndexedDBStoreBackend} class. */
+module.exports.IndexedDBStoreBackend = require("./store/indexeddb").IndexedDBStoreBackend;
+/** The {@link module:sync-accumulator.SyncAccumulator|SyncAccumulator} class. */
+module.exports.SyncAccumulator = require("./sync-accumulator");
 /** The {@link module:http-api.MatrixHttpApi|MatrixHttpApi} class. */
 module.exports.MatrixHttpApi = require("./http-api").MatrixHttpApi;
 /** The {@link module:http-api.MatrixError|MatrixError} class. */
@@ -56,6 +63,11 @@ module.exports.TimelineWindow = require("./timeline-window").TimelineWindow;
 module.exports.InteractiveAuth = require("./interactive-auth");
 
 
+module.exports.MemoryCryptoStore =
+    require("./crypto/store/memory-crypto-store").default;
+module.exports.IndexedDBCryptoStore =
+    require("./crypto/store/indexeddb-crypto-store").default;
+
 /**
  * Create a new Matrix Call.
  * @function
@@ -66,6 +78,23 @@ module.exports.InteractiveAuth = require("./interactive-auth");
  */
 module.exports.createNewMatrixCall = require("./webrtc/call").createNewMatrixCall;
 
+
+/**
+ * Set an audio input device to use for MatrixCalls
+ * @function
+ * @param {string=} deviceId the identifier for the device
+ * undefined treated as unset
+ */
+module.exports.setMatrixCallAudioInput = require('./webrtc/call').setAudioInput;
+/**
+ * Set a video input device to use for MatrixCalls
+ * @function
+ * @param {string=} deviceId the identifier for the device
+ * undefined treated as unset
+ */
+module.exports.setMatrixCallVideoInput = require('./webrtc/call').setVideoInput;
+
+
 // expose the underlying request object so different environments can use
 // different request libs (e.g. request or browser-request)
 let request;
@@ -100,6 +129,19 @@ module.exports.wrapRequest = function(wrapper) {
     };
 };
 
+
+let cryptoStoreFactory = () => new module.exports.MemoryCryptoStore;
+
+/**
+ * Configure a different factory to be used for creating crypto stores
+ *
+ * @param {Function} fac  a function which will return a new
+ *    {@link module:crypto.store.base~CryptoStore}.
+ */
+module.exports.setCryptoStoreFactory = function(fac) {
+    cryptoStoreFactory = fac;
+};
+
 /**
  * Construct a Matrix Client. Similar to {@link module:client~MatrixClient}
  * except that the 'request', 'store' and 'scheduler' dependencies are satisfied.
@@ -112,6 +154,13 @@ module.exports.wrapRequest = function(wrapper) {
  * {@link module:scheduler~MatrixScheduler}.
  * @param {requestFunction} opts.request If not set, defaults to the function
  * supplied to {@link request} which defaults to the request module from NPM.
+ *
+ * @param {module:crypto.store.base~CryptoStore=} opts.cryptoStore
+ *    crypto store implementation. Calls the factory supplied to
+ *    {@link setCryptoStoreFactory} if unspecified; or if no factory has been
+ *    specified, uses a default implementation (indexeddb in the browser,
+ *    in-memory otherwise).
+ *
  * @return {MatrixClient} A new matrix client.
  * @see {@link module:client~MatrixClient} for the full list of options for
  * <code>opts</code>.
@@ -127,6 +176,7 @@ module.exports.createClient = function(opts) {
       localStorage: global.localStorage,
     });
     opts.scheduler = opts.scheduler || new module.exports.MatrixScheduler();
+    opts.cryptoStore = opts.cryptoStore || cryptoStoreFactory();
     return new module.exports.MatrixClient(opts);
 };
 

src/models/event-timeline-set.js

@@ -154,10 +154,11 @@ EventTimelineSet.prototype.replaceEventId = function(oldEventId, newEventId) {
  * @fires module:client~MatrixClient#event:"Room.timelineReset"
  */
 EventTimelineSet.prototype.resetLiveTimeline = function(backPaginationToken, flush) {
-    let newTimeline;
+    // if timeline support is disabled, forget about the old timelines
+    const resetAllTimelines = !this._timelineSupport || flush;
 
-    if (!this._timelineSupport || flush) {
-        // if timeline support is disabled, forget about the old timelines
+    let newTimeline;
+    if (resetAllTimelines) {
         newTimeline = new EventTimeline(this);
         this._timelines = [newTimeline];
         this._eventIdToTimeline = {};
@@ -187,7 +188,7 @@ EventTimelineSet.prototype.resetLiveTimeline = function(backPaginationToken, flu
     newTimeline.setPaginationToken(backPaginationToken, EventTimeline.BACKWARDS);
 
     this._liveTimeline = newTimeline;
-    this.emit("Room.timelineReset", this.room, this);
+    this.emit("Room.timelineReset", this.room, this, resetAllTimelines);
 };
 
 /**
@@ -655,4 +656,5 @@ module.exports = EventTimelineSet;
  * @event module:client~MatrixClient#"Room.timelineReset"
  * @param {Room} room The room whose live timeline was reset, if any
  * @param {EventTimelineSet} timelineSet timelineSet room whose live timeline was reset
+ * @param {boolean} resetAllTimelines True if all timelines were reset.
  */

src/models/event-timeline.js

@@ -1,3 +1,18 @@
+/*
+Copyright 2016, 2017 OpenMarket Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
 "use strict";
 
 /**

src/models/event.js

@@ -49,6 +49,8 @@ module.exports.EventStatus = {
     CANCELLED: "cancelled",
 };
 
+const interns = {};
+
 /**
  * Construct a Matrix Event object
  * @constructor
@@ -61,7 +63,9 @@ module.exports.EventStatus = {
  * from changes to event JSON between Matrix versions.
  *
  * @prop {RoomMember} sender The room member who sent this event, or null e.g.
- * this is a presence event.
+ * this is a presence event. This is only guaranteed to be set for events that
+ * appear in a timeline, ie. do not guarantee that it will be set on state
+ * events.
  * @prop {RoomMember} target The room member who is the target of this event, e.g.
  * the invitee, the person being banned, etc.
  * @prop {EventStatus} status The sending status of the event.
@@ -73,7 +77,31 @@ module.exports.EventStatus = {
 module.exports.MatrixEvent = function MatrixEvent(
     event,
 ) {
+    // intern the values of matrix events to force share strings and reduce the
+    // amount of needless string duplication. This can save moderate amounts of
+    // memory (~10% on a 350MB heap).
+    ["state_key", "type", "sender", "room_id"].forEach((prop) => {
+        if (!event[prop]) {
+            return;
+        }
+        if (!interns[event[prop]]) {
+            interns[event[prop]] = event[prop];
+        }
+        event[prop] = interns[event[prop]];
+    });
+
+    ["membership", "avatar_url", "displayname"].forEach((prop) => {
+        if (!event.content || !event.content[prop]) {
+            return;
+        }
+        if (!interns[event.content[prop]]) {
+            interns[event.content[prop]] = event.content[prop];
+        }
+        event.content[prop] = interns[event.content[prop]];
+    });
+
     this.event = event || {};
+
     this.sender = null;
     this.target = null;
     this.status = null;
@@ -404,31 +432,18 @@ utils.extend(module.exports.MatrixEvent.prototype, {
 });
 
 
-/* http://matrix.org/docs/spec/r0.0.1/client_server.html#redactions says:
+/* _REDACT_KEEP_KEY_MAP gives the keys we keep when an event is redacted
  *
- * the server should strip off any keys not in the following list:
- *    event_id
- *    type
- *    room_id
- *    user_id
- *    state_key
- *    prev_state
- *    content
- *    [we keep 'unsigned' as well, since that is created by the local server]
+ * This is specified here:
+ *  http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#redactions
  *
- * The content object should also be stripped of all keys, unless it is one of
- * one of the following event types:
- *    m.room.member allows key membership
- *    m.room.create allows key creator
- *    m.room.join_rules allows key join_rule
- *    m.room.power_levels allows keys ban, events, events_default, kick,
- *        redact, state_default, users, users_default.
- *    m.room.aliases allows key aliases
+ * Also:
+ *  - We keep 'unsigned' since that is created by the local server
+ *  - We keep user_id for backwards-compat with v1
  */
-// a map giving the keys we keep when an event is redacted
 const _REDACT_KEEP_KEY_MAP = [
-    'event_id', 'type', 'room_id', 'user_id', 'state_key', 'prev_state',
-    'content', 'unsigned',
+    'event_id', 'type', 'room_id', 'user_id', 'sender', 'state_key', 'prev_state',
+    'content', 'unsigned', 'origin_server_ts',
 ].reduce(function(ret, val) {
     ret[val] = 1; return ret;
 }, {});

src/models/room-state.js

@@ -246,6 +246,45 @@ RoomState.prototype.getUserIdsWithDisplayName = function(displayName) {
     return this._displayNameToUserIds[displayName] || [];
 };
 
+/**
+ * Returns true if userId is in room, event is not redacted and either sender of
+ * mxEvent or has power level sufficient to redact events other than their own.
+ * @param {MatrixEvent} mxEvent The event to test permission for
+ * @param {string} userId The user ID of the user to test permission for
+ * @return {boolean} true if the given used ID can redact given event
+ */
+RoomState.prototype.maySendRedactionForEvent = function(mxEvent, userId) {
+    const member = this.getMember(userId);
+    if (!member || member.membership === 'leave') return false;
+
+    if (mxEvent.status || mxEvent.isRedacted()) return false;
+    if (mxEvent.getSender() === userId) return true;
+
+    return this._hasSufficientPowerLevelFor('redact', member.powerLevel);
+};
+
+/**
+ * Returns true if the given power level is sufficient for action
+ * @param {string} action The type of power level to check
+ * @param {number} powerLevel The power level of the member
+ * @return {boolean} true if the given power level is sufficient
+ */
+RoomState.prototype._hasSufficientPowerLevelFor = function(action, powerLevel) {
+    const powerLevelsEvent = this.getStateEvents('m.room.power_levels', '');
+
+    let powerLevels = {};
+    if (powerLevelsEvent) {
+        powerLevels = powerLevelsEvent.getContent();
+    }
+
+    let requiredLevel = 50;
+    if (powerLevels[action] !== undefined) {
+        requiredLevel = powerLevels[action];
+    }
+
+    return powerLevel >= requiredLevel;
+};
+
 /**
  * Short-form for maySendEvent('m.room.message', userId)
  * @param {string} userId The user ID of the user to test permission for
@@ -269,7 +308,6 @@ RoomState.prototype.maySendEvent = function(eventType, userId) {
     return this._maySendEventOfType(eventType, userId, false);
 };
 
-
 /**
  * Returns true if the given MatrixClient has permission to send a state
  * event of type `stateEventType` into this room.

src/models/room.js

@@ -27,6 +27,7 @@ const ContentRepo = require("../content-repo");
 const EventTimeline = require("./event-timeline");
 const EventTimelineSet = require("./event-timeline-set");
 
+import reEmit from '../reemit';
 
 function synthesizeReceipt(userId, event, receiptType) {
     // console.log("synthesizing receipt for "+event.getId());
@@ -204,10 +205,12 @@ Room.prototype.getLiveTimeline = function() {
  * <p>This is used when /sync returns a 'limited' timeline.
  *
  * @param {string=} backPaginationToken   token for back-paginating the new timeline
+ * @param {boolean=} flush True to remove all events in all timelines. If false, only
+ * the live timeline is reset.
  */
-Room.prototype.resetLiveTimeline = function(backPaginationToken) {
+Room.prototype.resetLiveTimeline = function(backPaginationToken, flush) {
     for (let i = 0; i < this._timelineSets.length; i++) {
-        this._timelineSets[i].resetLiveTimeline(backPaginationToken);
+        this._timelineSets[i].resetLiveTimeline(backPaginationToken, flush);
     }
 
     this._fixUpLegacyTimelineFields();
@@ -749,6 +752,8 @@ ALLOWED_TRANSITIONS[EventStatus.CANCELLED] =
  * @fires module:client~MatrixClient#event:"Room.localEchoUpdated"
  */
 Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
+    console.log(`setting pendingEvent status to ${newStatus} in ${event.getRoomId()}`);
+
     // if the message was sent, we expect an event id
     if (newStatus == EventStatus.SENT && !newEventId) {
         throw new Error("updatePendingEvent called with status=SENT, " +
@@ -1175,7 +1180,9 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
 
     // get members that are NOT ourselves and are actually in the room.
     const otherMembers = utils.filter(room.currentState.getMembers(), function(m) {
-        return (m.userId !== userId && m.membership !== "leave");
+        return (
+            m.userId !== userId && m.membership !== "leave" && m.membership !== "ban"
+        );
     });
     const allMembers = utils.filter(room.currentState.getMembers(), function(m) {
         return (m.membership !== "leave");
@@ -1248,25 +1255,6 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
     }
 }
 
-// FIXME: copypasted from sync.js
-function reEmit(reEmitEntity, emittableEntity, eventNames) {
-    utils.forEach(eventNames, function(eventName) {
-        // setup a listener on the entity (the Room, User, etc) for this event
-        emittableEntity.on(eventName, function() {
-            // take the args from the listener and reuse them, adding the
-            // event name to the arg list so it works with .emit()
-            // Transformation Example:
-            // listener on "foo" => function(a,b) { ... }
-            // Re-emit on "thing" => thing.emit("foo", a, b)
-            const newArgs = [eventName];
-            for (let i = 0; i < arguments.length; i++) {
-                newArgs.push(arguments[i]);
-            }
-            reEmitEntity.emit(...newArgs);
-        });
-    });
-}
-
 /**
  * The Room class.
  */

src/realtime-callbacks.js

@@ -168,7 +168,7 @@ function _runCallbacks() {
     for (let i = 0; i < callbacksToRun.length; i++) {
         cb = callbacksToRun[i];
         try {
-            cb.func.apply(null, cb.params);
+            cb.func.apply(global, cb.params);
         } catch (e) {
             console.error("Uncaught exception in callback function",
                           e.stack || e);

src/reemit.js

@@ -0,0 +1,44 @@
+/*
+Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module
+ */
+
+/**
+ * re-emit events raised by one EventEmitter from another
+ *
+ * @param {external:EventEmitter} reEmitEntity
+ *     entity from which we want events to be emitted
+ * @param {external:EventEmitter} emittableEntity
+ *     entity from which events are currently emitted
+ * @param {Array<string>} eventNames
+ *     list of events to be reemitted
+ */
+export default function reEmit(reEmitEntity, emittableEntity, eventNames) {
+    for (const eventName of eventNames) {
+        // setup a listener on the entity (the Room, User, etc) for this event
+        emittableEntity.on(eventName, function(...args) {
+            // take the args from the listener and reuse them, adding the
+            // event name to the arg list so it works with .emit()
+            // Transformation Example:
+            // listener on "foo" => function(a,b) { ... }
+            // Re-emit on "thing" => thing.emit("foo", a, b)
+            reEmitEntity.emit(eventName, ...args);
+        });
+    }
+}

src/store/indexeddb-local-backend.js

@@ -0,0 +1,318 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import q from "q";
+import SyncAccumulator from "../sync-accumulator";
+import utils from "../utils";
+
+const VERSION = 1;
+
+function createDatabase(db) {
+    // Make user store, clobber based on user ID. (userId property of User objects)
+    db.createObjectStore("users", { keyPath: ["userId"] });
+
+    // Make account data store, clobber based on event type.
+    // (event.type property of MatrixEvent objects)
+    db.createObjectStore("accountData", { keyPath: ["type"] });
+
+    // Make /sync store (sync tokens, room data, etc), always clobber (const key).
+    db.createObjectStore("sync", { keyPath: ["clobber"] });
+}
+
+/**
+ * Helper method to collect results from a Cursor and promiseify it.
+ * @param {ObjectStore|Index} store The store to perform openCursor on.
+ * @param {IDBKeyRange=} keyRange Optional key range to apply on the cursor.
+ * @param {Function} resultMapper A function which is repeatedly called with a
+ * Cursor.
+ * Return the data you want to keep.
+ * @return {Promise<T[]>} Resolves to an array of whatever you returned from
+ * resultMapper.
+ */
+function selectQuery(store, keyRange, resultMapper) {
+    const query = store.openCursor(keyRange);
+    return q.Promise((resolve, reject) => { /*eslint new-cap: 0*/
+        const results = [];
+        query.onerror = (event) => {
+            reject(new Error("Query failed: " + event.target.errorCode));
+        };
+        // collect results
+        query.onsuccess = (event) => {
+            const cursor = event.target.result;
+            if (!cursor) {
+                resolve(results);
+                return; // end of results
+            }
+            results.push(resultMapper(cursor));
+            cursor.continue();
+        };
+    });
+}
+
+function promiseifyTxn(txn) {
+    return new q.Promise((resolve, reject) => {
+        txn.oncomplete = function(event) {
+            resolve(event);
+        };
+        txn.onerror = function(event) {
+            reject(event);
+        };
+    });
+}
+
+function promiseifyRequest(req) {
+    return new q.Promise((resolve, reject) => {
+        req.onsuccess = function(event) {
+            resolve(event);
+        };
+        req.onerror = function(event) {
+            reject(event);
+        };
+    });
+}
+
+/**
+ * Does the actual reading from and writing to the indexeddb
+ *
+ * Construct a new Indexed Database store backend. This requires a call to
+ * <code>connect()</code> before this store can be used.
+ * @constructor
+ * @param {Object} indexedDBInterface The Indexed DB interface e.g
+ * <code>window.indexedDB</code>
+ * @param {string=} dbName Optional database name. The same name must be used
+ * to open the same database.
+ */
+const LocalIndexedDBStoreBackend = function LocalIndexedDBStoreBackend(
+    indexedDBInterface, dbName,
+) {
+    this.indexedDB = indexedDBInterface;
+    this._dbName = "matrix-js-sdk:" + (dbName || "default");
+    this.db = null;
+    this._syncAccumulator = new SyncAccumulator();
+};
+
+
+LocalIndexedDBStoreBackend.prototype = {
+    /**
+     * Attempt to connect to the database. This can fail if the user does not
+     * grant permission.
+     * @return {Promise} Resolves if successfully connected.
+     */
+    connect: function() {
+        if (this.db) {
+            return q();
+        }
+        const req = this.indexedDB.open(this._dbName, VERSION);
+        req.onupgradeneeded = (ev) => {
+            const db = ev.target.result;
+            const oldVersion = ev.oldVersion;
+            if (oldVersion < 1) { // The database did not previously exist.
+                createDatabase(db);
+            }
+            // Expand as needed.
+        };
+
+        return promiseifyRequest(req).then((ev) => {
+            this.db = ev.target.result;
+
+            // add a poorly-named listener for when deleteDatabase is called
+            // so we can close our db connections.
+            this.db.onversionchange = () => {
+                this.db.close();
+            };
+
+            return this._init();
+        });
+    },
+
+    /**
+     * Having connected, load initial data from the database and prepare for use
+     * @return {Promise} Resolves on success
+     */
+    _init: function() {
+        return q.all([
+            this._loadAccountData(),
+            this._loadSyncData(),
+        ]).then(([accountData, syncData]) => {
+            this._syncAccumulator.accumulate({
+                next_batch: syncData.nextBatch,
+                rooms: syncData.roomsData,
+                account_data: {
+                    events: accountData,
+                },
+            });
+        });
+    },
+
+    /**
+     * Clear the entire database. This should be used when logging out of a client
+     * to prevent mixing data between accounts.
+     * @return {Promise} Resolved when the database is cleared.
+     */
+    clearDatabase: function() {
+        console.log("Removing indexeddb instance: ", this._dbName);
+        return promiseifyRequest(this.indexedDB.deleteDatabase(this._dbName));
+    },
+
+    /**
+     * @param {boolean=} copy If false, the data returned is from internal
+     * buffers and must not be muated. Otherwise, a copy is made before
+     * returning such that the data can be safely mutated. Default: true.
+     *
+     * @return {Promise} Resolves with a sync response to restore the
+     * client state to where it was at the last save, or null if there
+     * is no saved sync data.
+     */
+    getSavedSync: function(copy) {
+        if (copy === undefined) copy = true;
+
+        const data = this._syncAccumulator.getJSON();
+        if (!data.nextBatch) return q(null);
+        if (copy) {
+            // We must deep copy the stored data so that the /sync processing code doesn't
+            // corrupt the internal state of the sync accumulator (it adds non-clonable keys)
+            return q(utils.deepCopy(data));
+        } else {
+            return q(data);
+        }
+    },
+
+    setSyncData: function(syncData) {
+        return q().then(() => {
+            this._syncAccumulator.accumulate(syncData);
+        });
+    },
+
+    syncToDatabase: function(userTuples) {
+        const syncData = this._syncAccumulator.getJSON();
+
+        return q.all([
+            this._persistUserPresenceEvents(userTuples),
+            this._persistAccountData(syncData.accountData),
+            this._persistSyncData(syncData.nextBatch, syncData.roomsData),
+        ]);
+    },
+
+    /**
+     * Persist rooms /sync data along with the next batch token.
+     * @param {string} nextBatch The next_batch /sync value.
+     * @param {Object} roomsData The 'rooms' /sync data from a SyncAccumulator
+     * @return {Promise} Resolves if the data was persisted.
+     */
+    _persistSyncData: function(nextBatch, roomsData) {
+        console.log("Persisting sync data up to ", nextBatch);
+        return q.try(() => {
+            const txn = this.db.transaction(["sync"], "readwrite");
+            const store = txn.objectStore("sync");
+            store.put({
+                clobber: "-", // constant key so will always clobber
+                nextBatch: nextBatch,
+                roomsData: roomsData,
+            }); // put == UPSERT
+            return promiseifyTxn(txn);
+        });
+    },
+
+    /**
+     * Persist a list of account data events. Events with the same 'type' will
+     * be replaced.
+     * @param {Object[]} accountData An array of raw user-scoped account data events
+     * @return {Promise} Resolves if the events were persisted.
+     */
+    _persistAccountData: function(accountData) {
+        return q.try(() => {
+            const txn = this.db.transaction(["accountData"], "readwrite");
+            const store = txn.objectStore("accountData");
+            for (let i = 0; i < accountData.length; i++) {
+                store.put(accountData[i]); // put == UPSERT
+            }
+            return promiseifyTxn(txn);
+        });
+    },
+
+    /**
+     * Persist a list of [user id, presence event] they are for.
+     * Users with the same 'userId' will be replaced.
+     * Presence events should be the event in its raw form (not the Event
+     * object)
+     * @param {Object[]} tuples An array of [userid, event] tuples
+     * @return {Promise} Resolves if the users were persisted.
+     */
+    _persistUserPresenceEvents: function(tuples) {
+        return q.try(() => {
+            const txn = this.db.transaction(["users"], "readwrite");
+            const store = txn.objectStore("users");
+            for (const tuple of tuples) {
+                store.put({
+                    userId: tuple[0],
+                    event: tuple[1],
+                }); // put == UPSERT
+            }
+            return promiseifyTxn(txn);
+        });
+    },
+
+    /**
+     * Load all user presence events from the database. This is not cached.
+     * FIXME: It would probably be more sensible to store the events in the
+     * sync.
+     * @return {Promise<Object[]>} A list of presence events in their raw form.
+     */
+    getUserPresenceEvents: function() {
+        return q.try(() => {
+            const txn = this.db.transaction(["users"], "readonly");
+            const store = txn.objectStore("users");
+            return selectQuery(store, undefined, (cursor) => {
+                return [cursor.value.userId, cursor.value.event];
+            });
+        });
+    },
+
+    /**
+     * Load all the account data events from the database. This is not cached.
+     * @return {Promise<Object[]>} A list of raw global account events.
+     */
+    _loadAccountData: function() {
+        return q.try(() => {
+            const txn = this.db.transaction(["accountData"], "readonly");
+            const store = txn.objectStore("accountData");
+            return selectQuery(store, undefined, (cursor) => {
+                return cursor.value;
+            });
+        });
+    },
+
+    /**
+     * Load the sync data from the database.
+     * @return {Promise<Object>} An object with "roomsData" and "nextBatch" keys.
+     */
+    _loadSyncData: function() {
+        return q.try(() => {
+            const txn = this.db.transaction(["sync"], "readonly");
+            const store = txn.objectStore("sync");
+            return selectQuery(store, undefined, (cursor) => {
+                return cursor.value;
+            }).then((results) => {
+                if (results.length > 1) {
+                    console.warn("loadSyncData: More than 1 sync row found.");
+                }
+                return (results.length > 0 ? results[0] : {});
+            });
+        });
+    },
+};
+
+export default LocalIndexedDBStoreBackend;

src/store/indexeddb-remote-backend.js

@@ -0,0 +1,140 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import q from "q";
+
+/**
+ * An IndexedDB store backend where the actual backend sits in a web
+ * worker.
+ *
+ * Construct a new Indexed Database store backend. This requires a call to
+ * <code>connect()</code> before this store can be used.
+ * @constructor
+ * @param {string} workerScript URL to the worker script
+ * @param {string=} dbName Optional database name. The same name must be used
+ * to open the same database.
+ * @param {Object} WorkerApi The web worker compatible interface object
+ */
+const RemoteIndexedDBStoreBackend = function RemoteIndexedDBStoreBackend(
+    workerScript, dbName, WorkerApi,
+) {
+    this._dbName = dbName;
+    this._worker = new WorkerApi(workerScript);
+    this._nextSeq = 0;
+    // The currently in-flight requests to the actual backend
+    this._inFlight = {
+        // seq: promise,
+    };
+
+    this._worker.onmessage = this._onWorkerMessage.bind(this);
+};
+
+
+RemoteIndexedDBStoreBackend.prototype = {
+    /**
+     * Attempt to connect to the database. This can fail if the user does not
+     * grant permission.
+     * @return {Promise} Resolves if successfully connected.
+     */
+    connect: function() {
+        return this._doCmd('_setupWorker', [this._dbName]).then(() => {
+            console.log("IndexedDB worker is ready");
+            return this._doCmd('connect');
+        });
+    },
+
+    /**
+     * Clear the entire database. This should be used when logging out of a client
+     * to prevent mixing data between accounts.
+     * @return {Promise} Resolved when the database is cleared.
+     */
+    clearDatabase: function() {
+        return this._doCmd('clearDatabase');
+    },
+
+    /**
+     * @return {Promise} Resolves with a sync response to restore the
+     * client state to where it was at the last save, or null if there
+     * is no saved sync data.
+     */
+    getSavedSync: function() {
+        return this._doCmd('getSavedSync');
+    },
+
+    setSyncData: function(syncData) {
+        return this._doCmd('setSyncData', [syncData]);
+    },
+
+    syncToDatabase: function(users) {
+        return this._doCmd('syncToDatabase', [users]);
+    },
+
+
+    /**
+     * Load all user presence events from the database. This is not cached.
+     * @return {Promise<Object[]>} A list of presence events in their raw form.
+     */
+    getUserPresenceEvents: function() {
+        return this._doCmd('getUserPresenceEvents');
+    },
+
+    _doCmd: function(cmd, args) {
+        // wrap in a q so if the postMessage throws,
+        // the promise automatically gets rejected
+        return q().then(() => {
+            const seq = this._nextSeq++;
+            const def = q.defer();
+
+            this._inFlight[seq] = def;
+
+            this._worker.postMessage({
+                command: cmd,
+                seq: seq,
+                args: args,
+            });
+
+            return def.promise;
+        });
+    },
+
+    _onWorkerMessage: function(ev) {
+        const msg = ev.data;
+
+        if (msg.command == 'cmd_success' || msg.command == 'cmd_fail') {
+            if (msg.seq === undefined) {
+                console.error("Got reply from worker with no seq");
+                return;
+            }
+
+            const def = this._inFlight[msg.seq];
+            if (def === undefined) {
+                console.error("Got reply for unknown seq " + msg.seq);
+                return;
+            }
+            delete this._inFlight[msg.seq];
+
+            if (msg.command == 'cmd_success') {
+                def.resolve(msg.result);
+            } else {
+                def.reject(msg.error);
+            }
+        } else {
+            console.warn("Unrecognised message from worker: " + msg);
+        }
+    },
+};
+
+export default RemoteIndexedDBStoreBackend;

src/store/indexeddb-store-worker.js

@@ -0,0 +1,122 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import q from "q";
+import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";
+
+/**
+ * This class lives in the webworker and drives a LocalIndexedDBStoreBackend
+ * controlled by messages from the main process.
+ *
+ * It should be instantiated by a web worker script provided by the application
+ * in a script, for example:
+ *
+ * import {IndexedDBStoreWorker} from 'matrix-js-sdk/lib/indexeddb-worker.js';
+ * const remoteWorker = new IndexedDBStoreWorker(postMessage);
+ * onmessage = remoteWorker.onMessage;
+ *
+ * Note that it is advisable to import this class by referencing the file directly to
+ * avoid a dependency on the whole js-sdk.
+ *
+ */
+class IndexedDBStoreWorker {
+    /**
+     * @param {function} postMessage The web worker postMessage function that
+     * should be used to communicate back to the main script.
+     */
+    constructor(postMessage) {
+        this.backend = null;
+        this.postMessage = postMessage;
+
+        this.onMessage = this.onMessage.bind(this);
+    }
+
+    /**
+     * Passes a message event from the main script into the class. This method
+     * can be directly assigned to the web worker `onmessage` variable.
+     *
+     * @param {Object} ev The message event
+     */
+    onMessage(ev) {
+        const msg = ev.data;
+        let prom;
+
+        switch (msg.command) {
+            case '_setupWorker':
+                this.backend = new LocalIndexedDBStoreBackend(
+                    // this is the 'indexedDB' global (where global != window
+                    // because it's a web worker and there is no window).
+                    indexedDB, msg.args[0],
+                );
+                prom = q();
+                break;
+            case 'connect':
+                prom = this.backend.connect();
+                break;
+            case 'clearDatabase':
+                prom = this.backend.clearDatabase().then((result) => {
+                    // This returns special classes which can't be cloned
+                    // across to the main script, so don't try.
+                    return {};
+                });
+                break;
+            case 'getSavedSync':
+                prom = this.backend.getSavedSync(false);
+                break;
+            case 'setSyncData':
+                prom = this.backend.setSyncData(...msg.args);
+                break;
+            case 'syncToDatabase':
+                prom = this.backend.syncToDatabase(...msg.args).then(() => {
+                    // This also returns IndexedDB events which are not cloneable
+                    return {};
+                });
+                break;
+            case 'getUserPresenceEvents':
+                prom = this.backend.getUserPresenceEvents();
+                break;
+        }
+
+        if (prom === undefined) {
+            postMessage({
+                command: 'cmd_fail',
+                seq: msg.seq,
+                // Can't be an Error because they're not structured cloneable
+                error: "Unrecognised command",
+            });
+            return;
+        }
+
+        prom.done((ret) => {
+            this.postMessage.call(null, {
+                command: 'cmd_success',
+                seq: msg.seq,
+                result: ret,
+            });
+        }, (err) => {
+            console.error("Error running command: "+msg.command);
+            console.error(err);
+            this.postMessage.call(null, {
+                command: 'cmd_fail',
+                seq: msg.seq,
+                // Just send a string because Error objects aren't cloneable
+                error: "Error running command",
+            });
+        });
+    }
+}
+
+module.exports = IndexedDBStoreWorker;

src/store/indexeddb.js

@@ -0,0 +1,195 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import q from "q";
+import {MatrixInMemoryStore} from "./memory";
+import utils from "../utils";
+import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";
+import RemoteIndexedDBStoreBackend from "./indexeddb-remote-backend.js";
+import User from "../models/user";
+import {MatrixEvent} from "../models/event";
+
+/**
+ * This is an internal module. See {@link IndexedDBStore} for the public class.
+ * @module store/indexeddb
+ */
+
+// If this value is too small we'll be writing very often which will cause
+// noticable stop-the-world pauses. If this value is too big we'll be writing
+// so infrequently that the /sync size gets bigger on reload. Writing more
+// often does not affect the length of the pause since the entire /sync
+// response is persisted each time.
+const WRITE_DELAY_MS = 1000 * 60 * 5; // once every 5 minutes
+
+
+/**
+ * Construct a new Indexed Database store, which extends MatrixInMemoryStore.
+ *
+ * This store functions like a MatrixInMemoryStore except it periodically persists
+ * the contents of the store to an IndexedDB backend.
+ *
+ * All data is still kept in-memory but can be loaded from disk by calling
+ * <code>startup()</code>. This can make startup times quicker as a complete
+ * sync from the server is not required. This does not reduce memory usage as all
+ * the data is eagerly fetched when <code>startup()</code> is called.
+ * <pre>
+ * let opts = { localStorage: window.localStorage };
+ * let store = new IndexedDBStore();
+ * await store.startup(); // load from indexed db
+ * let client = sdk.createClient({
+ *     store: store,
+ * });
+ * client.startClient();
+ * client.on("sync", function(state, prevState, data) {
+ *     if (state === "PREPARED") {
+ *         console.log("Started up, now with go faster stripes!");
+ *     }
+ * });
+ * </pre>
+ *
+ * @constructor
+ * @extends MatrixInMemoryStore
+ * @param {Object} opts Options object.
+ * @param {Object} opts.indexedDB The Indexed DB interface e.g.
+ * <code>window.indexedDB</code>
+ * @param {string=} opts.dbName Optional database name. The same name must be used
+ * to open the same database.
+ * @param {string=} opts.workerScript Optional URL to a script to invoke a web
+ * worker with to run IndexedDB queries on the web worker. The IndexedDbStoreWorker
+ * class is provided for this purpose and requires the application to provide a
+ * trivial wrapper script around it.
+ * @param {Object=} opts.workerApi The webWorker API object. If omitted, the global Worker
+ * object will be used if it exists.
+ * @prop {IndexedDBStoreBackend} backend The backend instance. Call through to
+ * this API if you need to perform specific indexeddb actions like deleting the
+ * database.
+ */
+const IndexedDBStore = function IndexedDBStore(opts) {
+    MatrixInMemoryStore.call(this, opts);
+
+    if (!opts.indexedDB) {
+        throw new Error('Missing required option: indexedDB');
+    }
+
+    if (opts.workerScript) {
+        // try & find a webworker-compatible API
+        let workerApi = opts.workerApi;
+        if (!workerApi) {
+            // default to the global Worker object (which is where it in a browser)
+            workerApi = global.Worker;
+        }
+        this.backend = new RemoteIndexedDBStoreBackend(
+            opts.workerScript, opts.dbName, workerApi,
+        );
+    } else {
+        this.backend = new LocalIndexedDBStoreBackend(opts.indexedDB, opts.dbName);
+    }
+
+    this.startedUp = false;
+    this._syncTs = 0;
+
+    // Records the last-modified-time of each user at the last point we saved
+    // the database, such that we can derive the set if users that have been
+    // modified since we last saved.
+    this._userModifiedMap = {
+        // user_id : timestamp
+    };
+};
+utils.inherits(IndexedDBStore, MatrixInMemoryStore);
+
+/**
+ * @return {Promise} Resolved when loaded from indexed db.
+  */
+IndexedDBStore.prototype.startup = function() {
+    if (this.startedUp) {
+        return q();
+    }
+
+    return this.backend.connect().then(() => {
+        return this.backend.getUserPresenceEvents();
+    }).then((userPresenceEvents) => {
+        userPresenceEvents.forEach(([userId, rawEvent]) => {
+            const u = new User(userId);
+            if (rawEvent) {
+                u.setPresenceEvent(new MatrixEvent(rawEvent));
+            }
+            this._userModifiedMap[u.userId] = u.getLastModifiedTime();
+            this.storeUser(u);
+        });
+    });
+};
+
+/**
+ * @return {Promise} Resolves with a sync response to restore the
+ * client state to where it was at the last save, or null if there
+ * is no saved sync data.
+ */
+IndexedDBStore.prototype.getSavedSync = function() {
+    return this.backend.getSavedSync();
+};
+
+/**
+ * Delete all data from this store.
+ * @return {Promise} Resolves if the data was deleted from the database.
+ */
+IndexedDBStore.prototype.deleteAllData = function() {
+    MatrixInMemoryStore.prototype.deleteAllData.call(this);
+    return this.backend.clearDatabase().then(() => {
+        console.log("Deleted indexeddb data.");
+    }, (err) => {
+        console.error("Failed to delete indexeddb data: ", err);
+        throw err;
+    });
+};
+
+/**
+ * Possibly write data to the database.
+ * @return {Promise} Promise resolves after the write completes.
+ */
+IndexedDBStore.prototype.save = function() {
+    const now = Date.now();
+    if (now - this._syncTs > WRITE_DELAY_MS) {
+        return this._reallySave();
+    }
+    return q();
+};
+
+IndexedDBStore.prototype._reallySave = function() {
+    this._syncTs = Date.now(); // set now to guard against multi-writes
+
+    // work out changed users (this doesn't handle deletions but you
+    // can't 'delete' users as they are just presence events).
+    const userTuples = [];
+    for (const u of this.getUsers()) {
+        if (this._userModifiedMap[u.userId] === u.getLastModifiedTime()) continue;
+        if (!u.events.presence) continue;
+
+        userTuples.push([u.userId, u.events.presence.event]);
+
+        // note that we've saved this version of the user
+        this._userModifiedMap[u.userId] = u.getLastModifiedTime();
+    }
+
+    return this.backend.syncToDatabase(userTuples).catch((err) => {
+        console.error("sync fail:", err);
+    });
+};
+
+IndexedDBStore.prototype.setSyncData = function(syncData) {
+    return this.backend.setSyncData(syncData);
+};
+
+module.exports.IndexedDBStore = IndexedDBStore;

src/store/memory.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -20,6 +21,7 @@ limitations under the License.
  */
  const utils = require("../utils");
  const User = require("../models/user");
+ const q = require("q");
 
 /**
  * Construct a new in-memory data store for the Matrix Client.
@@ -275,9 +277,58 @@ module.exports.MatrixInMemoryStore.prototype = {
         return this.accountData[eventType];
     },
 
-    // TODO
-    //setMaxHistoryPerRoom: function(maxHistory) {},
+    /**
+     * setSyncData does nothing as there is no backing data store.
+     *
+     * @param {Object} syncData The sync data
+     * @return {Promise} An immediately resolved promise.
+     */
+    setSyncData: function(syncData) {
+        return q();
+    },
 
-    // TODO
-    //reapOldMessages: function() {},
+    /**
+     * Save does nothing as there is no backing data store.
+     */
+    save: function() {},
+
+    /**
+     * Startup does nothing as this store doesn't require starting up.
+     * @return {Promise} An immediately resolved promise.
+     */
+    startup: function() {
+        return q();
+    },
+
+    /**
+     * @return {Promise} Resolves with a sync response to restore the
+     * client state to where it was at the last save, or null if there
+     * is no saved sync data.
+     */
+    getSavedSync: function() {
+        return q(null);
+    },
+
+    /**
+     * Delete all data from this store.
+     * @return {Promise} An immediately resolved promise.
+     */
+    deleteAllData: function() {
+        this.rooms = {
+            // roomId: Room
+        };
+        this.users = {
+            // userId: User
+        };
+        this.syncToken = null;
+        this.filters = {
+            // userId: {
+            //    filterId: Filter
+            // }
+        };
+        this.accountData = {
+            // type : content
+        };
+        return q();
+    },
 };

src/store/session/webstorage.js

@@ -99,6 +99,14 @@ WebStorageSessionStore.prototype = {
         return getJsonItem(this.store, keyEndToEndDevicesForUser(userId));
     },
 
+    storeEndToEndDeviceTrackingStatus: function(statusMap) {
+        setJsonItem(this.store, KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS, statusMap);
+    },
+
+    getEndToEndDeviceTrackingStatus: function() {
+        return getJsonItem(this.store, KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS);
+    },
+
     /**
      * Store the sync token corresponding to the device list.
      *
@@ -202,6 +210,7 @@ WebStorageSessionStore.prototype = {
 const KEY_END_TO_END_ACCOUNT = E2E_PREFIX + "account";
 const KEY_END_TO_END_ANNOUNCED = E2E_PREFIX + "announced";
 const KEY_END_TO_END_DEVICE_SYNC_TOKEN = E2E_PREFIX + "device_sync_token";
+const KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS = E2E_PREFIX + "device_tracking";
 
 function keyEndToEndDevicesForUser(userId) {
     return E2E_PREFIX + "devices/" + userId;

src/store/stub.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,6 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 "use strict";
+import q from "q";
 /**
  * This is an internal module.
  * @module store/stub
@@ -180,11 +182,46 @@ StubStore.prototype = {
 
     },
 
-    // TODO
-    //setMaxHistoryPerRoom: function(maxHistory) {},
+    /**
+     * setSyncData does nothing as there is no backing data store.
+     *
+     * @param {Object} syncData The sync data
+     * @return {Promise} An immediately resolved promise.
+     */
+    setSyncData: function(syncData) {
+        return q();
+    },
 
-    // TODO
-    //reapOldMessages: function() {},
+    /**
+     * Save does nothing as there is no backing data store.
+     */
+    save: function() {},
+
+    /**
+     * Startup does nothing.
+     * @return {Promise} An immediately resolved promise.
+     */
+    startup: function() {
+        return q();
+    },
+
+    /**
+     * @return {Promise} Resolves with a sync response to restore the
+     * client state to where it was at the last save, or null if there
+     * is no saved sync data.
+     */
+    getSavedSync: function() {
+        return q(null);
+    },
+
+    /**
+     * Delete all data from this store. Does nothing since this store
+     * doesn't store anything.
+     * @return {Promise} An immediately resolved promise.
+     */
+    deleteAllData: function() {
+        return q();
+    },
 };
 
 /** Stub Store class. */

src/sync-accumulator.js

@@ -0,0 +1,488 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * This is an internal module. See {@link SyncAccumulator} for the public class.
+ * @module sync-accumulator
+ */
+
+import utils from "./utils";
+
+
+/**
+ * The purpose of this class is to accumulate /sync responses such that a
+ * complete "initial" JSON response can be returned which accurately represents
+ * the sum total of the /sync responses accumulated to date. It only handles
+ * room data: that is, everything under the "rooms" top-level key.
+ *
+ * This class is used when persisting room data so a complete /sync response can
+ * be loaded from disk and incremental syncs can be performed on the server,
+ * rather than asking the server to do an initial sync on startup.
+ */
+class SyncAccumulator {
+
+    /**
+     * @param {Object} opts
+     * @param {Number=} opts.maxTimelineEntries The ideal maximum number of
+     * timeline entries to keep in the sync response. This is best-effort, as
+     * clients do not always have a back-pagination token for each event, so
+     * it's possible there may be slightly *less* than this value. There will
+     * never be more. This cannot be 0 or else it makes it impossible to scroll
+     * back in a room. Default: 50.
+     */
+    constructor(opts) {
+        opts = opts || {};
+        opts.maxTimelineEntries = opts.maxTimelineEntries || 50;
+        this.opts = opts;
+        this.accountData = {
+            //$event_type: Object
+        };
+        this.inviteRooms = {
+            //$roomId: { ... sync 'invite' json data ... }
+        };
+        this.joinRooms = {
+            //$roomId: {
+            //    _currentState: { $event_type: { $state_key: json } },
+            //    _timeline: [
+            //       { event: $event, token: null|token },
+            //       { event: $event, token: null|token },
+            //       { event: $event, token: null|token },
+            //       ...
+            //    ],
+            //    _accountData: { $event_type: json },
+            //    _unreadNotifications: { ... unread_notifications JSON ... },
+            //    _readReceipts: { $user_id: { data: $json, eventId: $event_id }}
+            //}
+        };
+        // the /sync token which corresponds to the last time rooms were
+        // accumulated. We remember this so that any caller can obtain a
+        // coherent /sync response and know at what point they should be
+        // streaming from without losing events.
+        this.nextBatch = null;
+    }
+
+    accumulate(syncResponse) {
+        this._accumulateRooms(syncResponse);
+        this._accumulateAccountData(syncResponse);
+        this.nextBatch = syncResponse.next_batch;
+    }
+
+    _accumulateAccountData(syncResponse) {
+        if (!syncResponse.account_data || !syncResponse.account_data.events) {
+            return;
+        }
+        // Clobbers based on event type.
+        syncResponse.account_data.events.forEach((e) => {
+            this.accountData[e.type] = e;
+        });
+    }
+
+    /**
+     * Accumulate incremental /sync room data.
+     * @param {Object} syncResponse the complete /sync JSON
+     */
+    _accumulateRooms(syncResponse) {
+        if (!syncResponse.rooms) {
+            return;
+        }
+        if (syncResponse.rooms.invite) {
+            Object.keys(syncResponse.rooms.invite).forEach((roomId) => {
+                this._accumulateRoom(
+                    roomId, "invite", syncResponse.rooms.invite[roomId],
+                );
+            });
+        }
+        if (syncResponse.rooms.join) {
+            Object.keys(syncResponse.rooms.join).forEach((roomId) => {
+                this._accumulateRoom(
+                    roomId, "join", syncResponse.rooms.join[roomId],
+                );
+            });
+        }
+        if (syncResponse.rooms.leave) {
+            Object.keys(syncResponse.rooms.leave).forEach((roomId) => {
+                this._accumulateRoom(
+                    roomId, "leave", syncResponse.rooms.leave[roomId],
+                );
+            });
+        }
+    }
+
+    _accumulateRoom(roomId, category, data) {
+        // Valid /sync state transitions
+        //       +--------+ <======+            1: Accept an invite
+        //   +== | INVITE |        | (5)        2: Leave a room
+        //   |   +--------+ =====+ |            3: Join a public room previously
+        //   |(1)            (4) | |               left (handle as if new room)
+        //   V         (2)       V |            4: Reject an invite
+        // +------+ ========> +--------+         5: Invite to a room previously
+        // | JOIN |    (3)    | LEAVE* |            left (handle as if new room)
+        // +------+ <======== +--------+
+        //
+        // * equivalent to "no state"
+        switch (category) {
+            case "invite": // (5)
+                this._accumulateInviteState(roomId, data);
+                break;
+            case "join":
+                if (this.inviteRooms[roomId]) { // (1)
+                    // was previously invite, now join. We expect /sync to give
+                    // the entire state and timeline on 'join', so delete previous
+                    // invite state
+                    delete this.inviteRooms[roomId];
+                }
+                // (3)
+                this._accumulateJoinState(roomId, data);
+                break;
+            case "leave":
+                if (this.inviteRooms[roomId]) { // (4)
+                    delete this.inviteRooms[roomId];
+                } else { // (2)
+                    delete this.joinRooms[roomId];
+                }
+                break;
+            default:
+                console.error("Unknown cateogory: ", category);
+        }
+    }
+
+    _accumulateInviteState(roomId, data) {
+        if (!data.invite_state || !data.invite_state.events) { // no new data
+            return;
+        }
+        if (!this.inviteRooms[roomId]) {
+            this.inviteRooms[roomId] = {
+                invite_state: data.invite_state,
+            };
+            return;
+        }
+        // accumulate extra keys for invite->invite transitions
+        // clobber based on event type / state key
+        // We expect invite_state to be small, so just loop over the events
+        const currentData = this.inviteRooms[roomId];
+        data.invite_state.events.forEach((e) => {
+            let hasAdded = false;
+            for (let i = 0; i < currentData.invite_state.events.length; i++) {
+                const current = currentData.invite_state.events[i];
+                if (current.type === e.type && current.state_key == e.state_key) {
+                    currentData.invite_state.events[i] = e; // update
+                    hasAdded = true;
+                }
+            }
+            if (!hasAdded) {
+                currentData.invite_state.events.push(e);
+            }
+        });
+    }
+
+    // Accumulate timeline and state events in a room.
+    _accumulateJoinState(roomId, data) {
+        // We expect this function to be called a lot (every /sync) so we want
+        // this to be fast. /sync stores events in an array but we often want
+        // to clobber based on type/state_key. Rather than convert arrays to
+        // maps all the time, just keep private maps which contain
+        // the actual current accumulated sync state, and array-ify it when
+        // getJSON() is called.
+
+        // State resolution:
+        // The 'state' key is the delta from the previous sync (or start of time
+        // if no token was supplied), to the START of the timeline. To obtain
+        // the current state, we need to "roll forward" state by reading the
+        // timeline. We want to store the current state so we can drop events
+        // out the end of the timeline based on opts.maxTimelineEntries.
+        //
+        //      'state'     'timeline'     current state
+        // |-------x<======================>x
+        //          T   I   M   E
+        //
+        // When getJSON() is called, we 'roll back' the current state by the
+        // number of entries in the timeline to work out what 'state' should be.
+
+        // Back-pagination:
+        // On an initial /sync, the server provides a back-pagination token for
+        // the start of the timeline. When /sync deltas come down, they also
+        // include back-pagination tokens for the start of the timeline. This
+        // means not all events in the timeline have back-pagination tokens, as
+        // it is only the ones at the START of the timeline which have them.
+        // In order for us to have a valid timeline (and back-pagination token
+        // to match), we need to make sure that when we remove old timeline
+        // events, that we roll forward to an event which has a back-pagination
+        // token. This means we can't keep a strict sliding-window based on
+        // opts.maxTimelineEntries, and we may have a few less. We should never
+        // have more though, provided that the /sync limit is less than or equal
+        // to opts.maxTimelineEntries.
+
+        if (!this.joinRooms[roomId]) {
+            // Create truly empty objects so event types of 'hasOwnProperty' and co
+            // don't cause this code to break.
+            this.joinRooms[roomId] = {
+                _currentState: Object.create(null),
+                _timeline: [],
+                _accountData: Object.create(null),
+                _unreadNotifications: {},
+                _readReceipts: {},
+            };
+        }
+        const currentData = this.joinRooms[roomId];
+
+        if (data.account_data && data.account_data.events) {
+            // clobber based on type
+            data.account_data.events.forEach((e) => {
+                currentData._accountData[e.type] = e;
+            });
+        }
+
+        // these probably clobber, spec is unclear.
+        if (data.unread_notifications) {
+            currentData._unreadNotifications = data.unread_notifications;
+        }
+
+        if (data.ephemeral && data.ephemeral.events) {
+            data.ephemeral.events.forEach((e) => {
+                // We purposefully do not persist m.typing events.
+                // Technically you could refresh a browser before the timer on a
+                // typing event is up, so it'll look like you aren't typing when
+                // you really still are. However, the alternative is worse. If
+                // we do persist typing events, it will look like people are
+                // typing forever until someone really does start typing (which
+                // will prompt Synapse to send down an actual m.typing event to
+                // clobber the one we persisted).
+                if (e.type !== "m.receipt" || !e.content) {
+                    // This means we'll drop unknown ephemeral events but that
+                    // seems okay.
+                    return;
+                }
+                // Handle m.receipt events. They clobber based on:
+                //   (user_id, receipt_type)
+                // but they are keyed in the event as:
+                //   content:{ $event_id: { $receipt_type: { $user_id: {json} }}}
+                // so store them in the former so we can accumulate receipt deltas
+                // quickly and efficiently (we expect a lot of them). Fold the
+                // receipt type into the key name since we only have 1 at the
+                // moment (m.read) and nested JSON objects are slower and more
+                // of a hassle to work with. We'll inflate this back out when
+                // getJSON() is called.
+                Object.keys(e.content).forEach((eventId) => {
+                    if (!e.content[eventId]["m.read"]) {
+                        return;
+                    }
+                    Object.keys(e.content[eventId]["m.read"]).forEach((userId) => {
+                        // clobber on user ID
+                        currentData._readReceipts[userId] = {
+                            data: e.content[eventId]["m.read"][userId],
+                            eventId: eventId,
+                        };
+                    });
+                });
+            });
+        }
+
+        // if we got a limited sync, we need to remove all timeline entries or else
+        // we will have gaps in the timeline.
+        if (data.timeline && data.timeline.limited) {
+            currentData._timeline = [];
+        }
+
+        // Work out the current state. The deltas need to be applied in the order:
+        // - existing state which didn't come down /sync.
+        // - State events under the 'state' key.
+        // - State events in the 'timeline'.
+        if (data.state && data.state.events) {
+            data.state.events.forEach((e) => {
+                setState(currentData._currentState, e);
+            });
+        }
+        if (data.timeline && data.timeline.events) {
+            data.timeline.events.forEach((e, index) => {
+                // this nops if 'e' isn't a state event
+                setState(currentData._currentState, e);
+                // append the event to the timeline. The back-pagination token
+                // corresponds to the first event in the timeline
+                currentData._timeline.push({
+                    event: e,
+                    token: index === 0 ? data.timeline.prev_batch : null,
+                });
+            });
+        }
+
+        // attempt to prune the timeline by jumping between events which have
+        // pagination tokens.
+        if (currentData._timeline.length > this.opts.maxTimelineEntries) {
+            const startIndex = (
+                currentData._timeline.length - this.opts.maxTimelineEntries
+            );
+            for (let i = startIndex; i < currentData._timeline.length; i++) {
+                if (currentData._timeline[i].token) {
+                    // keep all events after this, including this one
+                    currentData._timeline = currentData._timeline.slice(
+                        i, currentData._timeline.length,
+                    );
+                    break;
+                }
+            }
+        }
+    }
+
+    /**
+     * Return everything under the 'rooms' key from a /sync response which
+     * represents all room data that should be stored. This should be paired
+     * with the sync token which represents the most recent /sync response
+     * provided to accumulate().
+     * @return {Object} An object with a "nextBatch", "roomsData" and "accountData"
+     * keys.
+     * The "nextBatch" key is a string which represents at what point in the
+     * /sync stream the accumulator reached. This token should be used when
+     * restarting a /sync stream at startup. Failure to do so can lead to missing
+     * events. The "roomsData" key is an Object which represents the entire
+     * /sync response from the 'rooms' key onwards. The "accountData" key is
+     * a list of raw events which represent global account data.
+     */
+    getJSON() {
+        const data = {
+            join: {},
+            invite: {},
+            // always empty. This is set by /sync when a room was previously
+            // in 'invite' or 'join'. On fresh startup, the client won't know
+            // about any previous room being in 'invite' or 'join' so we can
+            // just omit mentioning it at all, even if it has previously come
+            // down /sync.
+            // The notable exception is when a client is kicked or banned:
+            // we may want to hold onto that room so the client can clearly see
+            // why their room has disappeared. We don't persist it though because
+            // it is unclear *when* we can safely remove the room from the DB.
+            // Instead, we assume that if you're loading from the DB, you've
+            // refreshed the page, which means you've seen the kick/ban already.
+            leave: {},
+        };
+        Object.keys(this.inviteRooms).forEach((roomId) => {
+            data.invite[roomId] = this.inviteRooms[roomId];
+        });
+        Object.keys(this.joinRooms).forEach((roomId) => {
+            const roomData = this.joinRooms[roomId];
+            const roomJson = {
+                ephemeral: { events: [] },
+                account_data: { events: [] },
+                state: { events: [] },
+                timeline: {
+                    events: [],
+                    prev_batch: null,
+                },
+                unread_notifications: roomData._unreadNotifications,
+            };
+            // Add account data
+            Object.keys(roomData._accountData).forEach((evType) => {
+                roomJson.account_data.events.push(roomData._accountData[evType]);
+            });
+
+            // Add receipt data
+            const receiptEvent = {
+                type: "m.receipt",
+                room_id: roomId,
+                content: {
+                    // $event_id: { "m.read": { $user_id: $json } }
+                },
+            };
+            Object.keys(roomData._readReceipts).forEach((userId) => {
+                const receiptData = roomData._readReceipts[userId];
+                if (!receiptEvent.content[receiptData.eventId]) {
+                    receiptEvent.content[receiptData.eventId] = {
+                        "m.read": {},
+                    };
+                }
+                receiptEvent.content[receiptData.eventId]["m.read"][userId] = (
+                    receiptData.data
+                );
+            });
+            // add only if we have some receipt data
+            if (Object.keys(receiptEvent.content).length > 0) {
+                roomJson.ephemeral.events.push(receiptEvent);
+            }
+
+            // Add timeline data
+            roomData._timeline.forEach((msgData) => {
+                if (!roomJson.timeline.prev_batch) {
+                    // the first event we add to the timeline MUST match up to
+                    // the prev_batch token.
+                    if (!msgData.token) {
+                        return; // this shouldn't happen as we prune constantly.
+                    }
+                    roomJson.timeline.prev_batch = msgData.token;
+                }
+                roomJson.timeline.events.push(msgData.event);
+            });
+
+            // Add state data: roll back current state to the start of timeline,
+            // by "reverse clobbering" from the end of the timeline to the start.
+            // Convert maps back into arrays.
+            const rollBackState = Object.create(null);
+            for (let i = roomJson.timeline.events.length - 1; i >=0; i--) {
+                const timelineEvent = roomJson.timeline.events[i];
+                if (timelineEvent.state_key === null ||
+                        timelineEvent.state_key === undefined) {
+                    continue; // not a state event
+                }
+                // since we're going back in time, we need to use the previous
+                // state value else we'll break causality. We don't have the
+                // complete previous state event, so we need to create one.
+                const prevStateEvent = utils.deepCopy(timelineEvent);
+                if (prevStateEvent.unsigned) {
+                    if (prevStateEvent.unsigned.prev_content) {
+                        prevStateEvent.content = prevStateEvent.unsigned.prev_content;
+                    }
+                    if (prevStateEvent.unsigned.prev_sender) {
+                        prevStateEvent.sender = prevStateEvent.unsigned.prev_sender;
+                    }
+                }
+                setState(rollBackState, prevStateEvent);
+            }
+            Object.keys(roomData._currentState).forEach((evType) => {
+                Object.keys(roomData._currentState[evType]).forEach((stateKey) => {
+                    let ev = roomData._currentState[evType][stateKey];
+                    if (rollBackState[evType] && rollBackState[evType][stateKey]) {
+                        // use the reverse clobbered event instead.
+                        ev = rollBackState[evType][stateKey];
+                    }
+                    roomJson.state.events.push(ev);
+                });
+            });
+            data.join[roomId] = roomJson;
+        });
+
+        // Add account data
+        const accData = [];
+        Object.keys(this.accountData).forEach((evType) => {
+            accData.push(this.accountData[evType]);
+        });
+
+        return {
+            nextBatch: this.nextBatch,
+            roomsData: data,
+            accountData: accData,
+        };
+    }
+}
+
+function setState(eventMap, event) {
+    if (event.state_key === null || event.state_key === undefined || !event.type) {
+        return;
+    }
+    if (!eventMap[event.type]) {
+        eventMap[event.type] = Object.create(null);
+    }
+    eventMap[event.type][event.state_key] = event;
+}
+
+module.exports = SyncAccumulator;

src/sync.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -30,6 +31,8 @@ const utils = require("./utils");
 const Filter = require("./filter");
 const EventTimeline = require("./models/event-timeline");
 
+import reEmit from './reemit';
+
 const DEBUG = true;
 
 // /sync requests allow you to set a timeout= but the request may continue
@@ -38,6 +41,11 @@ const DEBUG = true;
 // to determine the max time we're willing to wait.
 const BUFFER_PERIOD_MS = 80 * 1000;
 
+// Number of consecutive failed syncs that will lead to a syncState of ERROR as opposed
+// to RECONNECTING. This is needed to inform the client of server issues when the
+// keepAlive is successful but the server /sync fails.
+const FAILED_SYNC_ERROR_THRESHOLD = 3;
+
 function getFilterName(userId, suffix) {
     // scope this on the user ID because people may login on many accounts
     // and they all need to be stored!
@@ -59,6 +67,11 @@ function debuglog() {
  * @param {MatrixClient} client The matrix client instance to use.
  * @param {Object} opts Config options
  * @param {module:crypto=} opts.crypto Crypto manager
+ * @param {Function=} opts.canResetEntireTimeline A function which is called
+ * with a room ID and returns a boolean. It should return 'true' if the SDK can
+ * SAFELY remove events from this room. It may not be safe to remove events if
+ * there are other references to the timelines for this room.
+ * Default: returns false.
  */
 function SyncApi(client, opts) {
     this.client = client;
@@ -69,14 +82,21 @@ function SyncApi(client, opts) {
     opts.resolveInvitesToProfiles = opts.resolveInvitesToProfiles || false;
     opts.pollTimeout = opts.pollTimeout || (30 * 1000);
     opts.pendingEventOrdering = opts.pendingEventOrdering || "chronological";
+    if (!opts.canResetEntireTimeline) {
+        opts.canResetEntireTimeline = function(roomId) {
+            return false;
+        };
+    }
     this.opts = opts;
     this._peekRoomId = null;
     this._currentSyncRequest = null;
     this._syncState = null;
+    this._catchingUp = false;
     this._running = false;
     this._keepAliveTimer = null;
     this._connectionReturnedDefer = null;
     this._notifEvents = []; // accumulator of sync events in the current sync response
+    this._failedSyncCount = 0; // Number of consecutive failed /sync requests
 
     if (client.getNotifTimelineSet()) {
         reEmit(client, client.getNotifTimelineSet(),
@@ -278,7 +298,7 @@ SyncApi.prototype.peek = function(roomId) {
         client.store.storeRoom(peekRoom);
         client.emit("Room", peekRoom);
 
-        self._peekPoll(roomId);
+        self._peekPoll(peekRoom);
         return peekRoom;
     });
 };
@@ -293,22 +313,26 @@ SyncApi.prototype.stopPeeking = function() {
 
 /**
  * Do a peek room poll.
- * @param {string} roomId
+ * @param {Room} peekRoom
  * @param {string} token from= token
  */
-SyncApi.prototype._peekPoll = function(roomId, token) {
-    if (this._peekRoomId !== roomId) {
-        debuglog("Stopped peeking in room %s", roomId);
+SyncApi.prototype._peekPoll = function(peekRoom, token) {
+    if (this._peekRoomId !== peekRoom.roomId) {
+        debuglog("Stopped peeking in room %s", peekRoom.roomId);
         return;
     }
 
     const self = this;
     // FIXME: gut wrenching; hard-coded timeout values
     this.client._http.authedRequest(undefined, "GET", "/events", {
-        room_id: roomId,
+        room_id: peekRoom.roomId,
         timeout: 30 * 1000,
         from: token,
     }, undefined, 50 * 1000).done(function(res) {
+        if (self._peekRoomId !== peekRoom.roomId) {
+            debuglog("Stopped peeking in room %s", peekRoom.roomId);
+            return;
+        }
         // We have a problem that we get presence both from /events and /sync
         // however, /sync only returns presence for users in rooms
         // you're actually joined to.
@@ -334,15 +358,15 @@ SyncApi.prototype._peekPoll = function(roomId, token) {
 
         // strip out events which aren't for the given room_id (e.g presence)
         const events = res.chunk.filter(function(e) {
-            return e.room_id === roomId;
+            return e.room_id === peekRoom.roomId;
         }).map(self.client.getEventMapper());
-        const room = self.client.getRoom(roomId);
-        room.addLiveEvents(events);
-        self._peekPoll(roomId, res.end);
+
+        peekRoom.addLiveEvents(events);
+        self._peekPoll(peekRoom, res.end);
     }, function(err) {
-        console.error("[%s] Peek poll failed: %s", roomId, err);
+        console.error("[%s] Peek poll failed: %s", peekRoom.roomId, err);
         setTimeout(function() {
-            self._peekPoll(roomId, token);
+            self._peekPoll(peekRoom, token);
         }, 30 * 1000);
     });
 };
@@ -360,9 +384,6 @@ SyncApi.prototype.getSyncState = function() {
  * Main entry point
  */
 SyncApi.prototype.sync = function() {
-    debuglog("SyncApi.sync: starting with sync token " +
-             this.client.store.getSyncToken());
-
     const client = this.client;
     const self = this;
 
@@ -486,9 +507,30 @@ SyncApi.prototype._sync = function(syncOptions) {
 
     const syncToken = client.store.getSyncToken();
 
+    let pollTimeout = this.opts.pollTimeout;
+
+    if (this.getSyncState() !== 'SYNCING' || this._catchingUp) {
+        // unless we are happily syncing already, we want the server to return
+        // as quickly as possible, even if there are no events queued. This
+        // serves two purposes:
+        //
+        // * When the connection dies, we want to know asap when it comes back,
+        //   so that we can hide the error from the user. (We don't want to
+        //   have to wait for an event or a timeout).
+        //
+        // * We want to know if the server has any to_device messages queued up
+        //   for us. We do that by calling it with a zero timeout until it
+        //   doesn't give us any more to_device messages.
+        this._catchingUp = true;
+        pollTimeout = 0;
+    }
+
+    // normal timeout= plus buffer time
+    const clientSideTimeoutMs = pollTimeout + BUFFER_PERIOD_MS;
+
     const qps = {
         filter: filterId,
-        timeout: this.opts.pollTimeout,
+        timeout: pollTimeout,
     };
 
     if (syncToken) {
@@ -508,19 +550,59 @@ SyncApi.prototype._sync = function(syncOptions) {
         qps.timeout = 0;
     }
 
-    // normal timeout= plus buffer time
-    const clientSideTimeoutMs = this.opts.pollTimeout + BUFFER_PERIOD_MS;
+    let isCachedResponse = false;
 
-    this._currentSyncRequest = client._http.authedRequest(
-        undefined, "GET", "/sync", qps, undefined, clientSideTimeoutMs,
-    );
+    let syncPromise;
+    if (!syncOptions.hasSyncedBefore) {
+        // Don't do an HTTP hit to /sync. Instead, load up the persisted /sync data,
+        // if there is data there.
+        syncPromise = client.store.getSavedSync();
+    } else {
+        syncPromise = q(null);
+    }
+
+    syncPromise.then((savedSync) => {
+        if (savedSync) {
+            debuglog("sync(): not doing HTTP hit, instead returning stored /sync data");
+            isCachedResponse = true;
+            return {
+                next_batch: savedSync.nextBatch,
+                rooms: savedSync.roomsData,
+                account_data: {
+                    events: savedSync.accountData,
+                },
+            };
+        } else {
+            //debuglog('Starting sync since=' + syncToken);
+            this._currentSyncRequest = client._http.authedRequest(
+                undefined, "GET", "/sync", qps, undefined, clientSideTimeoutMs,
+            );
+            return this._currentSyncRequest;
+        }
+    }).then(function(data) {
+        //debuglog('Completed sync, next_batch=' + data.next_batch);
 
-    this._currentSyncRequest.done(function(data) {
         // set the sync token NOW *before* processing the events. We do this so
         // if something barfs on an event we can skip it rather than constantly
         // polling with the same token.
         client.store.setSyncToken(data.next_batch);
 
+        // Reset after a successful sync
+        self._failedSyncCount = 0;
+
+        // We need to wait until the sync data has been sent to the backend
+        // because it appears that the sync data gets modified somewhere in
+        // processing it in such a way as to make it no longer cloneable.
+        // XXX: Find out what is modifying it!
+        if (!isCachedResponse) {
+            // Don't give the store back its own cached data
+            return client.store.setSyncData(data).then(() => {
+                return data;
+            });
+        } else {
+            return q(data);
+        }
+    }).done((data) => {
         try {
             self._processSyncResponse(syncToken, data);
         } catch (e) {
@@ -533,6 +615,7 @@ SyncApi.prototype._sync = function(syncOptions) {
         const syncEventData = {
             oldSyncToken: syncToken,
             nextSyncToken: data.next_batch,
+            catchingUp: self._catchingUp,
         };
 
         if (!syncOptions.hasSyncedBefore) {
@@ -541,8 +624,17 @@ SyncApi.prototype._sync = function(syncOptions) {
         }
 
         // keep emitting SYNCING -> SYNCING for clients who want to do bulk updates
-        self._updateSyncState("SYNCING", syncEventData);
+        if (!isCachedResponse) {
+            self._updateSyncState("SYNCING", syncEventData);
 
+            // tell databases that everything is now in a consistent state and can be
+            // saved (no point doing so if we only have the data we just got out of the
+            // store).
+            client.store.save();
+        }
+
+
+        // Begin next sync
         self._sync(syncOptions);
     }, function(err) {
         if (!self._running) {
@@ -557,6 +649,9 @@ SyncApi.prototype._sync = function(syncOptions) {
         console.error("/sync error %s", err);
         console.error(err);
 
+        self._failedSyncCount++;
+        console.log('Number of consecutive failed sync requests:', self._failedSyncCount);
+
         debuglog("Starting keep-alive");
         // Note that we do *not* mark the sync connection as
         // lost yet: we only do this if a keepalive poke
@@ -569,7 +664,11 @@ SyncApi.prototype._sync = function(syncOptions) {
             self._sync(syncOptions);
         });
         self._currentSyncRequest = null;
-        self._updateSyncState("RECONNECTING");
+        // Transition from RECONNECTING to ERROR after a given number of failed syncs
+        self._updateSyncState(
+            self._failedSyncCount >= FAILED_SYNC_ERROR_THRESHOLD ?
+                "ERROR" : "RECONNECTING",
+        );
     });
 };
 
@@ -656,7 +755,9 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
     }
 
     // handle to-device events
-    if (data.to_device && utils.isArray(data.to_device.events)) {
+    if (data.to_device && utils.isArray(data.to_device.events) &&
+        data.to_device.events.length > 0
+       ) {
         data.to_device.events
             .map(client.getEventMapper())
             .forEach(
@@ -666,8 +767,10 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
                         toDeviceEvent.getType() == "m.room.message" &&
                             content.msgtype == "m.bad.encrypted"
                     ) {
-                        console.warn(
-                            "Unable to decrypt to-device event: " + content.body,
+                        // the mapper already logged a warning.
+                        console.log(
+                            'Ignoring undecryptable to-device event from ' +
+                                toDeviceEvent.getSender(),
                         );
                         return;
                     }
@@ -675,6 +778,9 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
                     client.emit("toDeviceEvent", toDeviceEvent);
                 },
             );
+    } else {
+        // no more to-device events: we can stop polling with a short timeout.
+        this._catchingUp = false;
     }
 
     // the returned json structure is a bit crap, so make it into a
@@ -783,7 +889,10 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
                 // timeline.
                 room.currentState.paginationToken = syncToken;
                 self._deregisterStateListeners(room);
-                room.resetLiveTimeline(joinObj.timeline.prev_batch);
+                room.resetLiveTimeline(
+                    joinObj.timeline.prev_batch,
+                    self.opts.canResetEntireTimeline(room.roomId),
+                );
 
                 // We have to assume any gap in any timeline is
                 // reason to stop incrementally tracking notifications and
@@ -880,7 +989,7 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
  * @param {number} delay How long to delay until the first poll.
  *        defaults to a short, randomised interval (to prevent
  *        tightlooping if /versions succeeds but /sync etc. fail).
- * @return {promise}
+ * @return {promise} which resolves once the connection returns
  */
 SyncApi.prototype._startKeepAlives = function(delay) {
     if (delay === undefined) {
@@ -906,7 +1015,11 @@ SyncApi.prototype._startKeepAlives = function(delay) {
 };
 
 /**
+ * Make a dummy call to /_matrix/client/versions, to see if the HS is
+ * reachable.
  *
+ * On failure, schedules a call back to itself. On success, resolves
+ * this._connectionReturnedDefer.
  */
 SyncApi.prototype._pokeKeepAlive = function() {
     const self = this;
@@ -1145,23 +1258,5 @@ function createNewUser(client, userId) {
     return user;
 }
 
-function reEmit(reEmitEntity, emittableEntity, eventNames) {
-    utils.forEach(eventNames, function(eventName) {
-        // setup a listener on the entity (the Room, User, etc) for this event
-        emittableEntity.on(eventName, function() {
-            // take the args from the listener and reuse them, adding the
-            // event name to the arg list so it works with .emit()
-            // Transformation Example:
-            // listener on "foo" => function(a,b) { ... }
-            // Re-emit on "thing" => thing.emit("foo", a, b)
-            const newArgs = [eventName];
-            for (let i = 0; i < arguments.length; i++) {
-                newArgs.push(arguments[i]);
-            }
-            reEmitEntity.emit(...newArgs);
-        });
-    });
-}
-
 /** */
 module.exports = SyncApi;

src/timeline-window.js

@@ -95,10 +95,30 @@ TimelineWindow.prototype.load = function(initialEventId, initialWindowSize) {
     const self = this;
     initialWindowSize = initialWindowSize || 20;
 
-    // given an EventTimeline, and an event index within it, initialise our
+    // given an EventTimeline, find the event we were looking for, and initialise our
     // fields so that the event in question is in the middle of the window.
-    const initFields = function(timeline, eventIndex) {
-        const endIndex = Math.min(timeline.getEvents().length,
+    const initFields = function(timeline) {
+        let eventIndex;
+
+        const events = timeline.getEvents();
+
+        if (!initialEventId) {
+            // we were looking for the live timeline: initialise to the end
+            eventIndex = events.length;
+        } else {
+            for (let i = 0; i < events.length; i++) {
+                if (events[i].getId() == initialEventId) {
+                    eventIndex = i;
+                    break;
+                }
+            }
+
+            if (eventIndex === undefined) {
+                throw new Error("getEventTimeline result didn't include requested event");
+            }
+        }
+
+        const endIndex = Math.min(events.length,
                                 eventIndex + Math.ceil(initialWindowSize / 2));
         const startIndex = Math.max(0, endIndex - initialWindowSize);
         self._start = new TimelineIndex(timeline, startIndex - timeline.getBaseIndex());
@@ -110,24 +130,19 @@ TimelineWindow.prototype.load = function(initialEventId, initialWindowSize) {
     // we already have the data we need, which is important to keep room-switching
     // feeling snappy.
     //
-    // TODO: ideally we'd spot getEventTimeline returning a resolved promise and
-    // skip straight to the find-event loop.
     if (initialEventId) {
-        return this._client.getEventTimeline(this._timelineSet, initialEventId)
-            .then(function(tl) {
-                // make sure that our window includes the event
-                for (let i = 0; i < tl.getEvents().length; i++) {
-                    if (tl.getEvents()[i].getId() == initialEventId) {
-                        initFields(tl, i);
-                        return;
-                    }
-                }
-                throw new Error("getEventTimeline result didn't include requested event");
-            });
+        const prom = this._client.getEventTimeline(this._timelineSet, initialEventId);
+
+        const promState = prom.inspect();
+        if (promState.state == 'fulfilled') {
+            initFields(promState.value);
+            return q();
+        } else {
+            return prom.then(initFields);
+        }
     } else {
-        // start with the most recent events
         const tl = this._timelineSet.getLiveTimeline();
-        initFields(tl, tl.getEvents().length);
+        initFields(tl);
         return q();
     }
 };

src/webrtc/call.js

@@ -24,6 +24,47 @@ const DEBUG = true;  // set true to enable console logging.
 
 // events: hangup, error(err), replaced(call), state(state, oldState)
 
+/**
+ * Fires when the MatrixCall encounters an error when sending a Matrix event.
+ * <p>
+ * This is required to allow errors, which occur during sending of events, to bubble up.
+ * (This is because call.js does a hangup when it encounters a normal `error`, which in
+ * turn could lead to an UnknownDeviceError.)
+ * <p>
+ * To deal with an UnknownDeviceError when trying to send events, the application should let
+ * users know that there are new devices in the encrypted room (into which the event was
+ * sent) and give the user the options to resend unsent events or cancel them. Resending
+ * is done using {@link module:client~MatrixClient#resendEvent} and cancelling can be done by using
+ * {@link module:client~MatrixClient#cancelPendingEvent}.
+ * <p>
+ * MatrixCall will not do anything in response to an error that causes `send_event_error`
+ * to be emitted with the exception of sending `m.call.candidates`, which is retried upon
+ * failure when ICE candidates are being sent. This happens during call setup.
+ *
+ * @event module:webrtc/call~MatrixCall#"send_event_error"
+ * @param {Error} err The error caught from calling client.sendEvent in call.js.
+ * @example
+ * matrixCall.on("send_event_error", function(err){
+ *   console.error(err);
+ * });
+ */
+
+/**
+ * Fires whenever an error occurs when call.js encounters an issue with setting up the call.
+ * <p>
+ * The error given will have a code equal to either `MatrixCall.ERR_LOCAL_OFFER_FAILED` or
+ * `MatrixCall.ERR_NO_USER_MEDIA`. `ERR_LOCAL_OFFER_FAILED` is emitted when the local client
+ * fails to create an offer. `ERR_NO_USER_MEDIA` is emitted when the user has denied access
+ * to their audio/video hardware.
+ *
+ * @event module:webrtc/call~MatrixCall#"error"
+ * @param {Error} err The error raised by MatrixCall.
+ * @example
+ * matrixCall.on("error", function(err){
+ *   console.error(err.code, err);
+ * });
+ */
+
 /**
  * Construct a new Matrix Call.
  * @constructor
@@ -50,7 +91,7 @@ function MatrixCall(opts) {
         utils.checkObjectHasKeys(server, ["urls"]);
     });
 
-    this.callId = "c" + new Date().getTime();
+    this.callId = "c" + new Date().getTime() + Math.random();
     this.state = 'fledgling';
     this.didConnect = false;
 
@@ -114,7 +155,7 @@ MatrixCall.prototype.placeVideoCall = function(remoteVideoElement, localVideoEle
 /**
  * Place a screen-sharing call to this room. This includes audio.
  * <b>This method is EXPERIMENTAL and subject to change without warning. It
- * only works in Google Chrome.</b>
+ * only works in Google Chrome and Firefox >= 44.</b>
  * @param {Element} remoteVideoElement a <code>&lt;video&gt;</code> DOM element
  * to render video to.
  * @param {Element} localVideoElement a <code>&lt;video&gt;</code> DOM element
@@ -125,7 +166,7 @@ MatrixCall.prototype.placeScreenSharingCall =
     function(remoteVideoElement, localVideoElement) {
     debuglog("placeScreenSharingCall");
     checkForErrorListener(this);
-    const screenConstraints = _getChromeScreenSharingConstraints(this);
+    const screenConstraints = _getScreenSharingConstraints(this);
     if (!screenConstraints) {
         return;
     }
@@ -207,22 +248,23 @@ MatrixCall.prototype.pauseElement = function(element, queueId) {
  * serialising the operation into a chain of promises to avoid racing access
  * to the element
  * @param {Element} element HTMLMediaElement element to pause
- * @param {string} src the src attribute value to assign to the element
+ * @param {MediaStream} srcObject the srcObject attribute value to assign to the element
  * @param {string} queueId Arbitrary ID to track the chain of promises to be used
  */
-MatrixCall.prototype.assignElement = function(element, src, queueId) {
-    console.log("queuing assign on " + queueId + " element " + element + " for " + src);
+MatrixCall.prototype.assignElement = function(element, srcObject, queueId) {
+    console.log("queuing assign on " + queueId + " element " + element + " for " +
+        srcObject);
     if (this.mediaPromises[queueId]) {
         this.mediaPromises[queueId] =
             this.mediaPromises[queueId].then(function() {
                 console.log("previous promise completed for " + queueId);
-                element.src = src;
+                element.srcObject = srcObject;
             }, function() {
                 console.log("previous promise failed for " + queueId);
-                element.src = src;
+                element.srcObject = srcObject;
             });
     } else {
-        element.src = src;
+        element.srcObject = srcObject;
     }
 };
 
@@ -262,9 +304,7 @@ MatrixCall.prototype.setLocalVideoElement = function(element) {
 
     if (element && this.localAVStream && this.type === 'video') {
         element.autoplay = true;
-        this.assignElement(element,
-                           this.URL.createObjectURL(this.localAVStream),
-                           "localVideo");
+        this.assignElement(element, this.localAVStream, "localVideo");
         element.muted = true;
         const self = this;
         setTimeout(function() {
@@ -295,6 +335,7 @@ MatrixCall.prototype.setRemoteVideoElement = function(element) {
 MatrixCall.prototype.setRemoteAudioElement = function(element) {
     this.remoteVideoElement.muted = true;
     this.remoteAudioElement = element;
+    this.remoteAudioElement.muted = false;
     _tryPlayRemoteAudioStream(this);
 };
 
@@ -369,12 +410,12 @@ MatrixCall.prototype.answer = function() {
     if (!this.localAVStream && !this.waitForLocalAVStream) {
         this.webRtc.getUserMedia(
             _getUserMediaVideoContraints(this.type),
-            hookCallback(self, self._gotUserMediaForAnswer),
-            hookCallback(self, self._getUserMediaFailed),
+            hookCallback(self, self._maybeGotUserMediaForAnswer),
+            hookCallback(self, self._maybeGotUserMediaForAnswer),
         );
         setState(this, 'wait_local_media');
     } else if (this.localAVStream) {
-        this._gotUserMediaForAnswer(this.localAVStream);
+        this._maybeGotUserMediaForAnswer(this.localAVStream);
     } else if (this.waitForLocalAVStream) {
         setState(this, 'wait_local_media');
     }
@@ -393,11 +434,11 @@ MatrixCall.prototype._replacedBy = function(newCall) {
         newCall.waitForLocalAVStream = true;
     } else if (this.state == 'create_offer') {
         debuglog("Handing local stream to new call");
-        newCall._gotUserMediaForAnswer(this.localAVStream);
+        newCall._maybeGotUserMediaForAnswer(this.localAVStream);
         delete(this.localAVStream);
     } else if (this.state == 'invite_sent') {
         debuglog("Handing local stream to new call");
-        newCall._gotUserMediaForAnswer(this.localAVStream);
+        newCall._maybeGotUserMediaForAnswer(this.localAVStream);
         delete(this.localAVStream);
     }
     newCall.localVideoElement = this.localVideoElement;
@@ -483,52 +524,69 @@ MatrixCall.prototype.isMicrophoneMuted = function() {
  * @private
  * @param {Object} stream
  */
-MatrixCall.prototype._gotUserMediaForInvite = function(stream) {
+MatrixCall.prototype._maybeGotUserMediaForInvite = function(stream) {
     if (this.successor) {
-        this.successor._gotUserMediaForAnswer(stream);
+        this.successor._maybeGotUserMediaForAnswer(stream);
         return;
     }
     if (this.state == 'ended') {
         return;
     }
-    debuglog("_gotUserMediaForInvite -> " + this.type);
+    debuglog("_maybeGotUserMediaForInvite -> " + this.type);
     const self = this;
-    const videoEl = this.getLocalVideoElement();
 
-    if (videoEl && this.type == 'video') {
-        videoEl.autoplay = true;
-        if (this.screenSharingStream) {
-            debuglog("Setting screen sharing stream to the local video element");
-            this.assignElement(videoEl,
-                   this.URL.createObjectURL(this.screenSharingStream),
-                   "localVideo");
-        } else {
-            this.assignElement(videoEl,
-                   this.URL.createObjectURL(stream),
-                   "localVideo");
-        }
-        videoEl.muted = true;
-        setTimeout(function() {
-            const vel = self.getLocalVideoElement();
-            if (vel.play) {
-                self.playElement(vel, "localVideo");
+    const error = stream;
+    const constraints = {
+        'mandatory': {
+            'OfferToReceiveAudio': true,
+            'OfferToReceiveVideo': self.type === 'video',
+        },
+    };
+    if (stream instanceof MediaStream) {
+        const videoEl = this.getLocalVideoElement();
+
+        if (videoEl && this.type == 'video') {
+            videoEl.autoplay = true;
+            if (this.screenSharingStream) {
+                debuglog("Setting screen sharing stream to the local video" +
+                    " element");
+                this.assignElement(videoEl, this.screenSharingStream, "localVideo");
+            } else {
+                this.assignElement(videoEl, stream, "localVideo");
             }
-        }, 0);
+            videoEl.muted = true;
+            setTimeout(function() {
+                const vel = self.getLocalVideoElement();
+                if (vel.play) {
+                    self.playElement(vel, "localVideo");
+                }
+            }, 0);
+        }
+
+        if (this.screenSharingStream) {
+            this.screenSharingStream.addTrack(stream.getAudioTracks()[0]);
+            stream = this.screenSharingStream;
+        }
+
+        this.localAVStream = stream;
+        // why do we enable audio (and only audio) tracks here? -- matthew
+        setTracksEnabled(stream.getAudioTracks(), true);
+        this.peerConn = _createPeerConnection(this);
+        this.peerConn.addStream(stream);
+    } else if (error.name === 'PermissionDeniedError') {
+        debuglog('User denied access to camera/microphone.' +
+            ' Or possibly you are using an insecure domain. Receiving only.');
+        this.peerConn = _createPeerConnection(this);
+    } else {
+        debuglog('Failed to getUserMedia.');
+        this._getUserMediaFailed(error);
+        return;
     }
 
-    if (this.screenSharingStream) {
-        this.screenSharingStream.addTrack(stream.getAudioTracks()[0]);
-        stream = this.screenSharingStream;
-    }
-
-    this.localAVStream = stream;
-    // why do we enable audio (and only audio) tracks here? -- matthew
-    setTracksEnabled(stream.getAudioTracks(), true);
-    this.peerConn = _createPeerConnection(this);
-    this.peerConn.addStream(stream);
     this.peerConn.createOffer(
         hookCallback(self, self._gotLocalOffer),
         hookCallback(self, self._getLocalOfferFailed),
+        constraints,
     );
     setState(self, 'create_offer');
 };
@@ -538,35 +596,44 @@ MatrixCall.prototype._gotUserMediaForInvite = function(stream) {
  * @private
  * @param {Object} stream
  */
-MatrixCall.prototype._gotUserMediaForAnswer = function(stream) {
+MatrixCall.prototype._maybeGotUserMediaForAnswer = function(stream) {
     const self = this;
     if (self.state == 'ended') {
         return;
     }
-    const localVidEl = self.getLocalVideoElement();
 
-    if (localVidEl && self.type == 'video') {
-        localVidEl.autoplay = true;
-        this.assignElement(localVidEl,
-               this.URL.createObjectURL(stream),
-               "localVideo");
-        localVidEl.muted = true;
-        setTimeout(function() {
-            const vel = self.getLocalVideoElement();
-            if (vel.play) {
-                self.playElement(vel, "localVideo");
-            }
-        }, 0);
+    const error = stream;
+    if (stream instanceof MediaStream) {
+        const localVidEl = self.getLocalVideoElement();
+
+        if (localVidEl && self.type == 'video') {
+            localVidEl.autoplay = true;
+            this.assignElement(localVidEl, stream, "localVideo");
+            localVidEl.muted = true;
+            setTimeout(function() {
+                const vel = self.getLocalVideoElement();
+                if (vel.play) {
+                    self.playElement(vel, "localVideo");
+                }
+            }, 0);
+        }
+
+        self.localAVStream = stream;
+        setTracksEnabled(stream.getAudioTracks(), true);
+        self.peerConn.addStream(stream);
+    } else if (error.name === 'PermissionDeniedError') {
+        debuglog('User denied access to camera/microphone.' +
+            ' Or possibly you are using an insecure domain. Receiving only.');
+    } else {
+        debuglog('Failed to getUserMedia.');
+        this._getUserMediaFailed(error);
+        return;
     }
 
-    self.localAVStream = stream;
-    setTracksEnabled(stream.getAudioTracks(), true);
-    self.peerConn.addStream(stream);
-
     const constraints = {
         'mandatory': {
             'OfferToReceiveAudio': true,
-            'OfferToReceiveVideo': self.type == 'video',
+            'OfferToReceiveVideo': self.type === 'video',
         },
     };
     self.peerConn.createAnswer(function(description) {
@@ -903,7 +970,11 @@ const setState = function(self, state) {
  * @return {Promise}
  */
 const sendEvent = function(self, eventType, content) {
-    return self.client.sendEvent(self.roomId, eventType, content);
+    return self.client.sendEvent(self.roomId, eventType, content).catch(
+        (err) => {
+            self.emit('send_event_error', err);
+        },
+    );
 };
 
 const sendCandidate = function(self, content) {
@@ -922,19 +993,19 @@ const terminate = function(self, hangupParty, hangupReason, shouldEmit) {
         if (self.getRemoteVideoElement().pause) {
             self.pauseElement(self.getRemoteVideoElement(), "remoteVideo");
         }
-        self.assignElement(self.getRemoteVideoElement(), "", "remoteVideo");
+        self.assignElement(self.getRemoteVideoElement(), null, "remoteVideo");
     }
     if (self.getRemoteAudioElement()) {
         if (self.getRemoteAudioElement().pause) {
             self.pauseElement(self.getRemoteAudioElement(), "remoteAudio");
         }
-        self.assignElement(self.getRemoteAudioElement(), "", "remoteAudio");
+        self.assignElement(self.getRemoteAudioElement(), null, "remoteAudio");
     }
     if (self.getLocalVideoElement()) {
         if (self.getLocalVideoElement().pause) {
             self.pauseElement(self.getLocalVideoElement(), "localVideo");
         }
-        self.assignElement(self.getLocalVideoElement(), "", "localVideo");
+        self.assignElement(self.getLocalVideoElement(), null, "localVideo");
     }
     self.hangupParty = hangupParty;
     self.hangupReason = hangupReason;
@@ -992,9 +1063,7 @@ const _tryPlayRemoteStream = function(self) {
     if (self.getRemoteVideoElement() && self.remoteAVStream) {
         const player = self.getRemoteVideoElement();
         player.autoplay = true;
-        self.assignElement(player,
-                           self.URL.createObjectURL(self.remoteAVStream),
-                           "remoteVideo");
+        self.assignElement(player, self.remoteAVStream, "remoteVideo");
         setTimeout(function() {
             const vel = self.getRemoteVideoElement();
             if (vel.play) {
@@ -1012,9 +1081,7 @@ const _tryPlayRemoteAudioStream = function(self) {
     if (self.getRemoteAudioElement() && self.remoteAStream) {
         const player = self.getRemoteAudioElement();
         player.autoplay = true;
-        self.assignElement(player,
-                           self.URL.createObjectURL(self.remoteAStream),
-                           "remoteAudio");
+        self.assignElement(player, self.remoteAStream, "remoteAudio");
         setTimeout(function() {
             const ael = self.getRemoteAudioElement();
             if (ael.play) {
@@ -1092,8 +1159,8 @@ const _placeCallWithConstraints = function(self, constraints) {
     self.client.callList[self.callId] = self;
     self.webRtc.getUserMedia(
         constraints,
-        hookCallback(self, self._gotUserMediaForInvite),
-        hookCallback(self, self._getUserMediaFailed),
+        hookCallback(self, self._maybeGotUserMediaForInvite),
+        hookCallback(self, self._maybeGotUserMediaForInvite),
     );
     setState(self, 'wait_local_media');
     self.direction = 'outbound';
@@ -1126,7 +1193,7 @@ const _createPeerConnection = function(self) {
     return pc;
 };
 
-const _getChromeScreenSharingConstraints = function(call) {
+const _getScreenSharingConstraints = function(call) {
     const screen = global.screen;
     if (!screen) {
         call.emit("error", callError(
@@ -1138,6 +1205,7 @@ const _getChromeScreenSharingConstraints = function(call) {
 
     return {
         video: {
+            mediaSource: 'screen',
             mandatory: {
                 chromeMediaSource: "screen",
                 chromeMediaSourceId: "" + Date.now(),
@@ -1151,18 +1219,28 @@ const _getChromeScreenSharingConstraints = function(call) {
 };
 
 const _getUserMediaVideoContraints = function(callType) {
+    const isWebkit = !!global.window.navigator.webkitGetUserMedia;
+
     switch (callType) {
         case 'voice':
-            return ({audio: true, video: false});
+            return {
+                audio: {
+                    deviceId: audioInput ? {exact: audioInput} : undefined,
+                }, video: false,
+            };
         case 'video':
-            return ({audio: true, video: {
-                mandatory: {
-                    minWidth: 640,
-                    maxWidth: 640,
-                    minHeight: 360,
-                    maxHeight: 360,
+            return {
+                audio: {
+                    deviceId: audioInput ? {exact: audioInput} : undefined,
+                }, video: {
+                    deviceId: videoInput ? {exact: videoInput} : undefined,
+                    /* We want 640x360.  Chrome will give it only if we ask exactly,
+                       FF refuses entirely if we ask exactly, so have to ask for ideal
+                       instead */
+                    width: isWebkit ? { exact: 640 } : { ideal: 640 },
+                    height: isWebkit ? { exact: 360 } : { ideal: 360 },
                 },
-            }});
+            };
     }
 };
 
@@ -1194,6 +1272,22 @@ const forAllTracksOnStream = function(s, f) {
 /** The MatrixCall class. */
 module.exports.MatrixCall = MatrixCall;
 
+let audioInput;
+let videoInput;
+/**
+ * Set an audio input device to use for MatrixCalls
+ * @function
+ * @param {string=} deviceId the identifier for the device
+ * undefined treated as unset
+ */
+module.exports.setAudioInput = function(deviceId) { audioInput = deviceId; };
+/**
+ * Set a video input device to use for MatrixCalls
+ * @function
+ * @param {string=} deviceId the identifier for the device
+ * undefined treated as unset
+ */
+module.exports.setVideoInput = function(deviceId) { videoInput = deviceId; };
 
 /**
  * Create a new Matrix call for the browser.