v0.7.6

Prepare changelog for v0.7.6
v0.7.6-rc.2
2017-04-12 09:56:46 +01:00 · 2017-04-12 09:56:45 +01:00 · 2017-04-10 14:52:10 +01:00 · 2017-04-10 14:52:09 +01:00 · 2017-04-10 14:50:16 +01:00 · 2017-04-10 11:28:40 +01:00
61 changed files with 4265 additions and 1033 deletions
@@ -0,0 +1,23 @@
+#   Copyright 2017 Aviral Dasgupta
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+root = true
+
+[*]
+charset=utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = true
@@ -1,3 +1,127 @@
+Changes in [0.7.6](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.6) (2017-04-12)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.6-rc.2...v0.7.6)
+
+ * No changes
+
+Changes in [0.7.6-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.6-rc.2) (2017-04-10)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.6-rc.1...v0.7.6-rc.2)
+
+ * Add feature detection for webworkers
+   [\#416](https://github.com/matrix-org/matrix-js-sdk/pull/416)
+ * Fix release script
+   [\#415](https://github.com/matrix-org/matrix-js-sdk/pull/415)
+
+Changes in [0.7.6-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.6-rc.1) (2017-04-07)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.5...v0.7.6-rc.1)
+
+ * Make indexeddb save after the first sync
+   [\#414](https://github.com/matrix-org/matrix-js-sdk/pull/414)
+ * Make indexeddb startup faster
+   [\#413](https://github.com/matrix-org/matrix-js-sdk/pull/413)
+ * Add ability to do indexeddb sync work in webworker
+   [\#412](https://github.com/matrix-org/matrix-js-sdk/pull/412)
+ * Move more functionality to the indexeddb backend
+   [\#409](https://github.com/matrix-org/matrix-js-sdk/pull/409)
+ * Indicate syncState ERROR after many failed /syncs
+   [\#410](https://github.com/matrix-org/matrix-js-sdk/pull/410)
+ * Further reorganising of indexeddb sync code
+   [\#407](https://github.com/matrix-org/matrix-js-sdk/pull/407)
+ * Change interface of IndexedDBStore: hide internals
+   [\#406](https://github.com/matrix-org/matrix-js-sdk/pull/406)
+ * Don't be SYNCING until updating from the server
+   [\#405](https://github.com/matrix-org/matrix-js-sdk/pull/405)
+ * Don't log the entire /sync response
+   [\#403](https://github.com/matrix-org/matrix-js-sdk/pull/403)
+ * webrtc/call: Assign MediaStream to video element srcObject
+   [\#402](https://github.com/matrix-org/matrix-js-sdk/pull/402)
+ * Fix undefined reference in http-api
+   [\#400](https://github.com/matrix-org/matrix-js-sdk/pull/400)
+ * Add copyright header to event-timeline.js
+   [\#382](https://github.com/matrix-org/matrix-js-sdk/pull/382)
+ * client: fix docs for user-scoped account_data events
+   [\#397](https://github.com/matrix-org/matrix-js-sdk/pull/397)
+ * Add a CONTRIBUTING for js-sdk
+   [\#399](https://github.com/matrix-org/matrix-js-sdk/pull/399)
+ * Fix leaking room state objects on limited sync responses
+   [\#395](https://github.com/matrix-org/matrix-js-sdk/pull/395)
+ * Extend 'ignoreFailure' to be 'background'
+   [\#396](https://github.com/matrix-org/matrix-js-sdk/pull/396)
+ * Add x_show_msisdn parameter to register calls
+   [\#388](https://github.com/matrix-org/matrix-js-sdk/pull/388)
+ * Update event redaction to keep sender and origin_server_ts
+   [\#394](https://github.com/matrix-org/matrix-js-sdk/pull/394)
+ * Handle 'limited' timeline responses in the SyncAccumulator
+   [\#393](https://github.com/matrix-org/matrix-js-sdk/pull/393)
+ * Give a better error message if the HS doesn't support msisdn registeration
+   [\#391](https://github.com/matrix-org/matrix-js-sdk/pull/391)
+ * Add getEmailSid
+   [\#383](https://github.com/matrix-org/matrix-js-sdk/pull/383)
+ * Add m.login.email.identity support to UI auth
+   [\#380](https://github.com/matrix-org/matrix-js-sdk/pull/380)
+ * src/client.js: Fix incorrect roomId reference in VoIP glare code
+   [\#381](https://github.com/matrix-org/matrix-js-sdk/pull/381)
+ * add .editorconfig
+   [\#379](https://github.com/matrix-org/matrix-js-sdk/pull/379)
+ * Store account data in the same way as room data
+   [\#377](https://github.com/matrix-org/matrix-js-sdk/pull/377)
+ * Upload one-time keys on /sync rather than a timer
+   [\#376](https://github.com/matrix-org/matrix-js-sdk/pull/376)
+ * Increase the WRITE_DELAY on database syncing
+   [\#374](https://github.com/matrix-org/matrix-js-sdk/pull/374)
+ * Make deleteAllData() return a Promise
+   [\#373](https://github.com/matrix-org/matrix-js-sdk/pull/373)
+ * Don't include banned users in the room name
+   [\#372](https://github.com/matrix-org/matrix-js-sdk/pull/372)
+ * Support IndexedDB as a backing store
+   [\#363](https://github.com/matrix-org/matrix-js-sdk/pull/363)
+ * Poll /sync with a short timeout while catching up
+   [\#370](https://github.com/matrix-org/matrix-js-sdk/pull/370)
+ * Make test coverage work again
+   [\#368](https://github.com/matrix-org/matrix-js-sdk/pull/368)
+ * Add docs to event
+   [\#367](https://github.com/matrix-org/matrix-js-sdk/pull/367)
+ * Keep the device-sync token more up-to-date
+   [\#366](https://github.com/matrix-org/matrix-js-sdk/pull/366)
+ * Fix race conditions in device list download
+   [\#365](https://github.com/matrix-org/matrix-js-sdk/pull/365)
+ * Fix the unban method
+   [\#364](https://github.com/matrix-org/matrix-js-sdk/pull/364)
+ * Spread out device verification work
+   [\#362](https://github.com/matrix-org/matrix-js-sdk/pull/362)
+ * Clean up/improve e2e logging
+   [\#361](https://github.com/matrix-org/matrix-js-sdk/pull/361)
+ * Fix decryption of events whose key arrives later
+   [\#360](https://github.com/matrix-org/matrix-js-sdk/pull/360)
+ * Invalidate device lists when encryption is enabled in a room
+   [\#359](https://github.com/matrix-org/matrix-js-sdk/pull/359)
+ * Switch from jasmine to mocha + expect + lolex
+   [\#358](https://github.com/matrix-org/matrix-js-sdk/pull/358)
+ * Install source-map-support in each test
+   [\#356](https://github.com/matrix-org/matrix-js-sdk/pull/356)
+ * searchMessageText: avoid setting keys=undefined
+   [\#357](https://github.com/matrix-org/matrix-js-sdk/pull/357)
+ * realtime-callbacks: pass `global` as `this`
+   [\#355](https://github.com/matrix-org/matrix-js-sdk/pull/355)
+ * Make the tests work without olm
+   [\#354](https://github.com/matrix-org/matrix-js-sdk/pull/354)
+ * Tests: Factor out TestClient and use it in crypto tests
+   [\#353](https://github.com/matrix-org/matrix-js-sdk/pull/353)
+ * Fix some lint
+   [\#352](https://github.com/matrix-org/matrix-js-sdk/pull/352)
+ * Make a sig for source tarballs when releasing
+   [\#351](https://github.com/matrix-org/matrix-js-sdk/pull/351)
+ * When doing a pre-release, don't bother merging to master and develop.
+   [\#350](https://github.com/matrix-org/matrix-js-sdk/pull/350)
+
+Changes in [0.7.5](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.5) (2017-02-04)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.5-rc.3...v0.7.5)
+
+No changes from 0.7.5-rc.3
+
 Changes in [0.7.5-rc.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.5-rc.3) (2017-02-03)
 ==========================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.5-rc.2...v0.7.5-rc.3)
@@ -1,4 +1,115 @@
 Contributing code to matrix-js-sdk
 ==================================

-matrix-js-sdk follows the same pattern as https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst
+Everyone is welcome to contribute code to matrix-js-sdk, provided that they are
+willing to license their contributions under the same license as the project
+itself. We follow a simple 'inbound=outbound' model for contributions: the act
+of submitting an 'inbound' contribution means that the contributor agrees to
+license the code under the same terms as the project's overall 'outbound'
+license - in this case, Apache Software License v2 (see `<LICENSE>`_).
+
+How to contribute
+~~~~~~~~~~~~~~~~~
+
+The preferred and easiest way to contribute changes to the project is to fork
+it on github, and then create a pull request to ask us to pull your changes
+into our repo (https://help.github.com/articles/using-pull-requests/)
+
+**The single biggest thing you need to know is: please base your changes on
+the develop branch - /not/ master.**
+
+We use the master branch to track the most recent release, so that folks who
+blindly clone the repo and automatically check out master get something that
+works. Develop is the unstable branch where all the development actually
+happens: the workflow is that contributors should fork the develop branch to
+make a 'feature' branch for a particular contribution, and then make a pull
+request to merge this back into the matrix.org 'official' develop branch. We
+use github's pull request workflow to review the contribution, and either ask
+you to make any refinements needed or merge it and make them ourselves. The
+changes will then land on master when we next do a release.
+
+We use Travis for continuous integration, and all pull requests get
+automatically tested by Travis: if your change breaks the build, then the PR
+will show that there are failed checks, so please check back after a few
+minutes.
+
+Code style
+~~~~~~~~~~
+
+The code-style for matrix-js-sdk is not formally documented, but contributors
+are encouraged to read the code style document for matrix-react-sdk
+(`<https://github.com/matrix-org/matrix-react-sdk/blob/master/code_style.md>`_)
+and follow the principles set out there.
+
+Please ensure your changes match the cosmetic style of the existing project,
+and **never** mix cosmetic and functional changes in the same commit, as it
+makes it horribly hard to review otherwise.
+
+Attribution
+~~~~~~~~~~~
+
+Everyone who contributes anything to Matrix is welcome to be listed in the
+AUTHORS.rst file for the project in question. Please feel free to include a
+change to AUTHORS.rst in your pull request to list yourself and a short
+description of the area(s) you've worked on. Also, we sometimes have swag to
+give away to contributors - if you feel that Matrix-branded apparel is missing
+from your life, please mail us your shipping address to matrix at matrix.org
+and we'll try to fix it :)
+
+Sign off
+~~~~~~~~
+
+In order to have a concrete record that your contribution is intentional
+and you agree to license it under the same terms as the project's license, we've adopted the
+same lightweight approach that the Linux Kernel
+(https://www.kernel.org/doc/Documentation/SubmittingPatches), Docker
+(https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other
+projects use: the DCO (Developer Certificate of Origin:
+http://developercertificate.org/). This is a simple declaration that you wrote
+the contribution or otherwise have the right to contribute it to Matrix::
+
+    Developer Certificate of Origin
+    Version 1.1
+
+    Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+    660 York Street, Suite 102,
+    San Francisco, CA 94110 USA
+
+    Everyone is permitted to copy and distribute verbatim copies of this
+    license document, but changing it is not allowed.
+
+    Developer's Certificate of Origin 1.1
+
+    By making a contribution to this project, I certify that:
+
+    (a) The contribution was created in whole or in part by me and I
+        have the right to submit it under the open source license
+        indicated in the file; or
+
+    (b) The contribution is based upon previous work that, to the best
+        of my knowledge, is covered under an appropriate open source
+        license and I have the right under that license to submit that
+        work with modifications, whether created in whole or in part
+        by me, under the same open source license (unless I am
+        permitted to submit under a different license), as indicated
+        in the file; or
+
+    (c) The contribution was provided directly to me by some other
+        person who certified (a), (b) or (c) and I have not modified
+        it.
+
+    (d) I understand and agree that this project and the contribution
+        are public and that a record of the contribution (including all
+        personal information I submit with it, including my sign-off) is
+        maintained indefinitely and may be redistributed consistent with
+        this project or the open source license(s) involved.
+
+If you agree to this for your contribution, then all that's needed is to
+include the line in your commit or pull request comment::
+
+    Signed-off-by: Your Name <your@email.example.org>
+
+...using your real name; unfortunately pseudonyms and anonymous contributions
+can't be accepted. Git makes this trivial - just use the -s flag when you do
+``git commit``, having first set ``user.name`` and ``user.email`` git configs
+(which you should have done anyway :)
@@ -44,7 +44,7 @@ window.onload = function() {
    disableButtons(true, true, true);
 };

-matrixClient.on("sync", function(state, prevState, data) {
+client.on("sync", function(state, prevState, data) {
    switch (state) {
        case "PREPARED":
          syncComplete();
@@ -2,7 +2,7 @@

 set -x

-export NVM_DIR="/home/jenkins/.nvm"
+export NVM_DIR="$HOME/.nvm"
 [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"

 nvm use 6 || exit $?
@@ -15,6 +15,9 @@ function fail {
    RC=1
 }

+# don't use last time's test reports
+rm -rf reports coverage || exit $?
+
 npm test || fail "npm test finished with return code $?"

 npm run -s lint -- -f checkstyle > eslint.xml ||
@@ -1,18 +1,19 @@
 {
  "name": "matrix-js-sdk",
-  "version": "0.7.5-rc.3",
+  "version": "0.7.6",
  "description": "Matrix Client-Server SDK for Javascript",
  "main": "index.js",
  "scripts": {
-    "buildtest": "babel -s -d specbuild spec",
-    "test": "npm run buildtest && istanbul cover --report cobertura --config .istanbul.yml -i \"lib/**/*.js\" jasmine-node -- specbuild --verbose --junitreport --captureExceptions",
-    "check": "npm run buildtest && jasmine-node specbuild --verbose --junitreport --captureExceptions",
+    "test:build": "babel -s -d specbuild spec",
+    "test:run": "istanbul cover --report text --report cobertura --config .istanbul.yml -i \"lib/**/*.js\" _mocha -- --recursive specbuild --colors --reporter mocha-jenkins-reporter --reporter-options junit_report_path=reports/test-results.xml",
+    "test": "npm run test:build && npm run test:run",
+    "check": "npm run test:build && _mocha --recursive specbuild --colors",
    "gendoc": "jsdoc -r lib -P package.json -R README.md -d .jsdoc",
    "start": "babel -s -w -d lib src",
    "build": "babel -s -d lib src && rimraf dist && mkdir dist && browserify -d browser-index.js | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js && uglifyjs -c -m -o dist/browser-matrix.min.js --source-map dist/browser-matrix.min.js.map --in-source-map dist/browser-matrix.js.map dist/browser-matrix.js",
    "dist": "npm run build",
    "watch": "watchify -d browser-index.js -o 'exorcist dist/browser-matrix.js.map > dist/browser-matrix.js' -v",
-    "lint": "eslint --max-warnings 121 src spec",
+    "lint": "eslint --max-warnings 115 src spec",
    "prepublish": "npm run build && git rev-parse HEAD > git-revision.txt"
  },
  "repository": {
@@ -60,10 +61,14 @@
    "eslint": "^3.13.1",
    "eslint-config-google": "^0.7.1",
    "exorcist": "^0.4.0",
+    "expect": "^1.20.2",
    "istanbul": "^0.3.13",
-    "jasmine-node": "^1.14.5",
    "jsdoc": "^3.4.0",
+    "lolex": "^1.5.2",
+    "mocha": "^3.2.0",
+    "mocha-jenkins-reporter": "^0.3.6",
    "rimraf": "^2.5.4",
+    "source-map-support": "^0.4.11",
    "sourceify": "^0.1.0",
    "uglifyjs": "^2.4.10",
    "watchify": "^3.2.1"
@@ -186,6 +186,13 @@ if [ -n "$signing_id" ]; then
    # make a signed tag
    # gnupg seems to fail to get the right tty device unless we set it here
    GIT_COMMITTER_EMAIL="$signing_id" GPG_TTY=`tty` git tag -u "$signing_id" -F "${latest_changes}" "$tag"
+
+    # also make a signature for the source tarball.
+    project_name=`jq -r '.name' package.json`
+    source_sigfile="${tag}-src.tar.gz.asc"
+    git archive --format tgz --prefix="${project_name}-${release}/" "$tag" |
+        gpg -u "$signing_id" --armor --output "$source_sigfile" --detach-sig -
+    assets="$assets -a $source_sigfile"
 else
    git tag -a -F "${latest_changes}" "$tag"
 fi
@@ -210,6 +217,9 @@ fi
 rm "${release_text}"
 rm "${latest_changes}"

+# publish to npmjs
+npm publish
+
 if [ -z "$skip_jsdoc" ]; then
    echo "generating jsdocs"
    npm run gendoc
@@ -225,6 +235,11 @@ if [ -z "$skip_jsdoc" ]; then
    git commit --no-verify -m "Add jsdoc for $release" index.html "$release"
 fi

+# if it is a pre-release, leave it on the release branch for now.
+if [ $prerelease -eq 1 ]; then
+    exit 0
+fi
+
 # merge release branch to master
 echo "updating master branch"
 git checkout master
@@ -237,9 +252,6 @@ if [ -z "$skip_jsdoc" ]; then
    git push origin gh-pages
 fi

-# publish to npmjs
-npm publish
-
 # finally, merge master back onto develop
 git checkout develop
 git pull
@@ -1,5 +1,5 @@
 module.exports = {
    env: {
-        jasmine: true,
+        mocha: true,
    },
 }
@@ -0,0 +1,185 @@
+/*
+Copyright 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+"use strict";
+
+import sdk from '..';
+import testUtils from './test-utils';
+import MockHttpBackend from './mock-request';
+import expect from 'expect';
+import q from 'q';
+
+/**
+ * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
+ *
+ * @constructor
+ * @param {string} userId
+ * @param {string} deviceId
+ * @param {string} accessToken
+ */
+export default function TestClient(userId, deviceId, accessToken) {
+    this.userId = userId;
+    this.deviceId = deviceId;
+
+    this.storage = new sdk.WebStorageSessionStore(new testUtils.MockStorageApi());
+    this.httpBackend = new MockHttpBackend();
+    this.client = sdk.createClient({
+        baseUrl: "http://" + userId + ".test.server",
+        userId: userId,
+        accessToken: accessToken,
+        deviceId: deviceId,
+        sessionStore: this.storage,
+        request: this.httpBackend.requestFn,
+    });
+
+    this.deviceKeys = null;
+    this.oneTimeKeys = {};
+}
+
+TestClient.prototype.toString = function() {
+    return 'TestClient[' + this.userId + ']';
+};
+
+/**
+ * start the client, and wait for it to initialise.
+ *
+ * @return {Promise}
+ */
+TestClient.prototype.start = function() {
+    this.httpBackend.when("GET", "/pushrules").respond(200, {});
+    this.httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+    this.expectDeviceKeyUpload();
+
+    // we let the client do a very basic initial sync, which it needs before
+    // it will upload one-time keys.
+    this.httpBackend.when("GET", "/sync").respond(200, { next_batch: 1 });
+
+    this.client.startClient({
+        // set this so that we can get hold of failed events
+        pendingEventOrdering: 'detached',
+    });
+
+    return this.httpBackend.flush().then(() => {
+        console.log(this + ': started');
+    });
+};
+
+/**
+ * stop the client
+ */
+TestClient.prototype.stop = function() {
+    this.client.stopClient();
+};
+
+/**
+ * Set up expectations that the client will upload device keys.
+ */
+TestClient.prototype.expectDeviceKeyUpload = function() {
+    const self = this;
+    this.httpBackend.when("POST", "/keys/upload").respond(200, function(path, content) {
+        expect(content.one_time_keys).toBe(undefined);
+        expect(content.device_keys).toBeTruthy();
+
+        console.log(self + ': received device keys');
+        // we expect this to happen before any one-time keys are uploaded.
+        expect(Object.keys(self.oneTimeKeys).length).toEqual(0);
+
+        self.deviceKeys = content.device_keys;
+        return {one_time_key_counts: {signed_curve25519: 0}};
+    });
+};
+
+
+/**
+ * If one-time keys have already been uploaded, return them. Otherwise,
+ * set up an expectation that the keys will be uploaded, and wait for
+ * that to happen.
+ *
+ * @returns {Promise} for the one-time keys
+ */
+TestClient.prototype.awaitOneTimeKeyUpload = function() {
+    if (Object.keys(this.oneTimeKeys).length != 0) {
+        // already got one-time keys
+        return q(this.oneTimeKeys);
+    }
+
+    this.httpBackend.when("POST", "/keys/upload")
+        .respond(200, (path, content) => {
+            expect(content.device_keys).toBe(undefined);
+            expect(content.one_time_keys).toBe(undefined);
+            return {one_time_key_counts: {
+                signed_curve25519: Object.keys(this.oneTimeKeys).length,
+            }};
+        });
+
+    this.httpBackend.when("POST", "/keys/upload")
+          .respond(200, (path, content) => {
+              expect(content.device_keys).toBe(undefined);
+              expect(content.one_time_keys).toBeTruthy();
+              expect(content.one_time_keys).toNotEqual({});
+              console.log('%s: received %i one-time keys', this,
+                          Object.keys(content.one_time_keys).length);
+              this.oneTimeKeys = content.one_time_keys;
+              return {one_time_key_counts: {
+                  signed_curve25519: Object.keys(this.oneTimeKeys).length,
+              }};
+          });
+
+    return this.httpBackend.flush('/keys/upload', 2).then((flushed) => {
+        expect(flushed).toEqual(2);
+        return this.oneTimeKeys;
+    });
+};
+
+/**
+ * Set up expectations that the client will query device keys.
+ *
+ * We check that the query contains each of the users in `response`.
+ *
+ * @param {Object} response   response to the query.
+ */
+TestClient.prototype.expectKeyQuery = function(response) {
+    this.httpBackend.when('POST', '/keys/query').respond(
+        200, (path, content) => {
+            Object.keys(response.device_keys).forEach((userId) => {
+                expect(content.device_keys[userId]).toEqual({});
+            });
+            return response;
+        });
+};
+
+
+/**
+ * get the uploaded curve25519 device key
+ *
+ * @return {string} base64 device key
+ */
+TestClient.prototype.getDeviceKey = function() {
+    const keyId = 'curve25519:' + this.deviceId;
+    return this.deviceKeys.keys[keyId];
+};
+
+
+/**
+ * get the uploaded ed25519 device key
+ *
+ * @return {string} base64 device key
+ */
+TestClient.prototype.getSigningKey = function() {
+    const keyId = 'ed25519:' + this.deviceId;
+    return this.deviceKeys.keys[keyId];
+};
@@ -1,115 +1,59 @@
+/*
+Copyright 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/* This file consists of a set of integration tests which try to simulate
+ * communication via an Olm-encrypted room between two users, Alice and Bob.
+ *
+ * Note that megolm (group) conversation is not tested here.
+ *
+ * See also `megolm.spec.js`.
+ */
+
 "use strict";
+import 'source-map-support/register';
+import expect from 'expect';
 const sdk = require("../..");
 const q = require("q");
-const HttpBackend = require("../mock-request");
 const utils = require("../../lib/utils");
 const testUtils = require("../test-utils");
+const TestClient = require('../TestClient').default;

-let aliHttpBackend;
-let bobHttpBackend;
-let aliClient;
+let aliTestClient;
 const roomId = "!room:localhost";
 const aliUserId = "@ali:localhost";
 const aliDeviceId = "zxcvb";
 const aliAccessToken = "aseukfgwef";
-let bobClient;
+let bobTestClient;
 const bobUserId = "@bob:localhost";
 const bobDeviceId = "bvcxz";
 const bobAccessToken = "fewgfkuesa";
-let bobOneTimeKeys;
-let aliDeviceKeys;
-let bobDeviceKeys;
-let bobDeviceCurve25519Key;
-let bobDeviceEd25519Key;
-let aliStorage;
-let bobStorage;
 let aliMessages;
 let bobMessages;

-
-/**
- * Set an expectation that the client will upload device keys and a number of
- * one-time keys; then flush the http requests.
- *
- * @param {string} deviceId expected device id in upload request
- * @param {object} httpBackend
- *
- * @return {promise} completes once the http requests have completed, returning combined
- * {one_time_keys: {}, device_keys: {}}
- */
-function expectKeyUpload(deviceId, httpBackend) {
-    const uploadPath = "/keys/upload/" + deviceId;
-    const keys = {};
-
-    httpBackend.when("POST", uploadPath).respond(200, function(path, content) {
-        expect(content.one_time_keys).not.toBeDefined();
-        expect(content.device_keys).toBeDefined();
-        keys.device_keys = content.device_keys;
-        return {one_time_key_counts: {signed_curve25519: 0}};
-    });
-
-    httpBackend.when("POST", uploadPath).respond(200, function(path, content) {
-        expect(content.device_keys).not.toBeDefined();
-        expect(content.one_time_keys).toBeDefined();
-        expect(content.one_time_keys).not.toEqual({});
-        let count = 0;
-        for (const key in content.one_time_keys) {
-            if (content.one_time_keys.hasOwnProperty(key)) {
-                count++;
-            }
-        }
-        expect(count).toEqual(5);
-        keys.one_time_keys = content.one_time_keys;
-        return {one_time_key_counts: {signed_curve25519: count}};
-    });
-
-    return httpBackend.flush(uploadPath, 2).then(function() {
-        return keys;
+function bobUploadsDeviceKeys() {
+    bobTestClient.expectDeviceKeyUpload();
+    return q.all([
+        bobTestClient.client.uploadKeys(),
+        bobTestClient.httpBackend.flush(),
+    ]).then(() => {
+        expect(Object.keys(bobTestClient.deviceKeys).length).toNotEqual(0);
    });
 }

-
-/**
- * Set an expectation that ali will upload device keys and a number of one-time keys;
- * then flush the http requests.
- *
- * <p>Updates <tt>aliDeviceKeys</tt>
- *
- * @return {promise} completes once the http requests have completed.
- */
-function expectAliKeyUpload() {
-    return expectKeyUpload(aliDeviceId, aliHttpBackend).then(function(content) {
-        aliDeviceKeys = content.device_keys;
-    });
-}
-
-
-/**
- * Set an expectation that bob will upload device keys and a number of one-time keys;
- * then flush the http requests.
- *
- * <p>Updates <tt>bobDeviceKeys</tt>, <tt>bobOneTimeKeys</tt>,
- * <tt>bobDeviceCurve25519Key</tt>, <tt>bobDeviceEd25519Key</tt>
- *
- * @return {promise} completes once the http requests have completed.
- */
-function expectBobKeyUpload() {
-    return expectKeyUpload(bobDeviceId, bobHttpBackend).then(function(content) {
-        bobDeviceKeys = content.device_keys;
-        bobOneTimeKeys = content.one_time_keys;
-        expect(bobDeviceKeys).toBeDefined();
-        expect(bobOneTimeKeys).toBeDefined();
-        bobDeviceCurve25519Key = bobDeviceKeys.keys["curve25519:bvcxz"];
-        bobDeviceEd25519Key = bobDeviceKeys.keys["ed25519:bvcxz"];
-    });
-}
-
-function bobUploadsKeys() {
-    bobClient.uploadKeys(5).catch(testUtils.failTest);
-    return expectBobKeyUpload();
-}
-
-
 /**
 * Set an expectation that ali will query bobs keys; then flush the http request.
 *
@@ -117,17 +61,18 @@ function bobUploadsKeys() {
 */
 function expectAliQueryKeys() {
    // can't query keys before bob has uploaded them
-    expect(bobDeviceKeys).toBeDefined();
+    expect(bobTestClient.deviceKeys).toBeTruthy();

    const bobKeys = {};
-    bobKeys[bobDeviceId] = bobDeviceKeys;
-    aliHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
+    bobKeys[bobDeviceId] = bobTestClient.deviceKeys;
+    aliTestClient.httpBackend.when("POST", "/keys/query")
+            .respond(200, function(path, content) {
        expect(content.device_keys[bobUserId]).toEqual({});
        const result = {};
        result[bobUserId] = bobKeys;
        return {device_keys: result};
    });
-    return aliHttpBackend.flush("/keys/query", 1);
+    return aliTestClient.httpBackend.flush("/keys/query", 1);
 }

 /**
@@ -137,17 +82,21 @@ function expectAliQueryKeys() {
 */
 function expectBobQueryKeys() {
    // can't query keys before ali has uploaded them
-    expect(aliDeviceKeys).toBeDefined();
+    expect(aliTestClient.deviceKeys).toBeTruthy();

    const aliKeys = {};
-    aliKeys[aliDeviceId] = aliDeviceKeys;
-    bobHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
+    aliKeys[aliDeviceId] = aliTestClient.deviceKeys;
+    console.log("query result will be", aliKeys);
+
+    bobTestClient.httpBackend.when(
+        "POST", "/keys/query",
+    ).respond(200, function(path, content) {
        expect(content.device_keys[aliUserId]).toEqual({});
        const result = {};
        result[aliUserId] = aliKeys;
        return {device_keys: result};
    });
-    return bobHttpBackend.flush("/keys/query", 1);
+    return bobTestClient.httpBackend.flush("/keys/query", 1);
 }

 /**
@@ -156,39 +105,45 @@ function expectBobQueryKeys() {
 * @return {promise} resolves once the http request has completed.
 */
 function expectAliClaimKeys() {
-    // can't query keys before bob has uploaded them
-    expect(bobOneTimeKeys).toBeDefined();
-
-    aliHttpBackend.when("POST", "/keys/claim").respond(200, function(path, content) {
-        const claimType = content.one_time_keys[bobUserId][bobDeviceId];
-        expect(claimType).toEqual("signed_curve25519");
-        let keyId = null;
-        for (keyId in bobOneTimeKeys) {
-            if (bobOneTimeKeys.hasOwnProperty(keyId)) {
-                if (keyId.indexOf(claimType + ":") === 0) {
-                    break;
+    return bobTestClient.awaitOneTimeKeyUpload().then((keys) => {
+        aliTestClient.httpBackend.when(
+            "POST", "/keys/claim",
+        ).respond(200, function(path, content) {
+            const claimType = content.one_time_keys[bobUserId][bobDeviceId];
+            expect(claimType).toEqual("signed_curve25519");
+            let keyId = null;
+            for (keyId in keys) {
+                if (bobTestClient.oneTimeKeys.hasOwnProperty(keyId)) {
+                    if (keyId.indexOf(claimType + ":") === 0) {
+                        break;
+                    }
                }
            }
-        }
-        const result = {};
-        result[bobUserId] = {};
-        result[bobUserId][bobDeviceId] = {};
-        result[bobUserId][bobDeviceId][keyId] = bobOneTimeKeys[keyId];
-        return {one_time_keys: result};
+            const result = {};
+            result[bobUserId] = {};
+            result[bobUserId][bobDeviceId] = {};
+            result[bobUserId][bobDeviceId][keyId] = keys[keyId];
+            return {one_time_keys: result};
+        });
+    }).then(() => {
+        // it can take a while to process the key query, so give it some extra
+        // time, and make sure the claim actually happens rather than ploughing on
+        // confusingly.
+        return aliTestClient.httpBackend.flush("/keys/claim", 1, 20).then((r) => {
+            expect(r).toEqual(1);
+        });
    });
-
-    return aliHttpBackend.flush("/keys/claim", 1);
 }


 function aliDownloadsKeys() {
    // can't query keys before bob has uploaded them
-    expect(bobDeviceEd25519Key).toBeDefined();
+    expect(bobTestClient.getSigningKey()).toBeTruthy();

-    const p1 = aliClient.downloadKeys([bobUserId]).then(function() {
-        expect(aliClient.listDeviceKeys(bobUserId)).toEqual([{
+    const p1 = aliTestClient.client.downloadKeys([bobUserId]).then(function() {
+        expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([{
            id: "bvcxz",
-            key: bobDeviceEd25519Key,
+            key: bobTestClient.getSigningKey(),
            verified: false,
            blocked: false,
            display_name: null,
@@ -199,26 +154,26 @@ function aliDownloadsKeys() {
    // check that the localStorage is updated as we expect (not sure this is
    // an integration test, but meh)
    return q.all([p1, p2]).then(function() {
-        const devices = aliStorage.getEndToEndDevicesForUser(bobUserId);
-        expect(devices[bobDeviceId].keys).toEqual(bobDeviceKeys.keys);
+        const devices = aliTestClient.storage.getEndToEndDevicesForUser(bobUserId);
+        expect(devices[bobDeviceId].keys).toEqual(bobTestClient.deviceKeys.keys);
        expect(devices[bobDeviceId].verified).
            toBe(0); // DeviceVerification.UNVERIFIED
    });
 }

 function aliEnablesEncryption() {
-    return aliClient.setRoomEncryption(roomId, {
+    return aliTestClient.client.setRoomEncryption(roomId, {
        algorithm: "m.olm.v1.curve25519-aes-sha2",
    }).then(function() {
-        expect(aliClient.isRoomEncrypted(roomId)).toBeTruthy();
+        expect(aliTestClient.client.isRoomEncrypted(roomId)).toBeTruthy();
    });
 }

 function bobEnablesEncryption() {
-    return bobClient.setRoomEncryption(roomId, {
+    return bobTestClient.client.setRoomEncryption(roomId, {
        algorithm: "m.olm.v1.curve25519-aes-sha2",
    }).then(function() {
-       expect(bobClient.isRoomEncrypted(roomId)).toBeTruthy();
+       expect(bobTestClient.client.isRoomEncrypted(roomId)).toBeTruthy();
    });
 }

@@ -230,7 +185,7 @@ function bobEnablesEncryption() {
 */
 function aliSendsFirstMessage() {
    return q.all([
-        sendMessage(aliClient),
+        sendMessage(aliTestClient.client),
        expectAliQueryKeys()
            .then(expectAliClaimKeys)
            .then(expectAliSendMessageRequest),
@@ -247,7 +202,7 @@ function aliSendsFirstMessage() {
 */
 function aliSendsMessage() {
    return q.all([
-        sendMessage(aliClient),
+        sendMessage(aliTestClient.client),
        expectAliSendMessageRequest(),
    ]).spread(function(_, ciphertext) {
        return ciphertext;
@@ -262,7 +217,7 @@ function aliSendsMessage() {
 */
 function bobSendsReplyMessage() {
    return q.all([
-        sendMessage(bobClient),
+        sendMessage(bobTestClient.client),
        expectBobQueryKeys()
            .then(expectBobSendMessageRequest),
    ]).spread(function(_, ciphertext) {
@@ -276,11 +231,11 @@ function bobSendsReplyMessage() {
 * @return {promise} which resolves to the ciphertext for Bob's device.
 */
 function expectAliSendMessageRequest() {
-    return expectSendMessageRequest(aliHttpBackend).then(function(content) {
+    return expectSendMessageRequest(aliTestClient.httpBackend).then(function(content) {
        aliMessages.push(content);
-        expect(utils.keys(content.ciphertext)).toEqual([bobDeviceCurve25519Key]);
-        const ciphertext = content.ciphertext[bobDeviceCurve25519Key];
-        expect(ciphertext).toBeDefined();
+        expect(utils.keys(content.ciphertext)).toEqual([bobTestClient.getDeviceKey()]);
+        const ciphertext = content.ciphertext[bobTestClient.getDeviceKey()];
+        expect(ciphertext).toBeTruthy();
        return ciphertext;
    });
 }
@@ -291,13 +246,13 @@ function expectAliSendMessageRequest() {
 * @return {promise} which resolves to the ciphertext for Bob's device.
 */
 function expectBobSendMessageRequest() {
-    return expectSendMessageRequest(bobHttpBackend).then(function(content) {
+    return expectSendMessageRequest(bobTestClient.httpBackend).then(function(content) {
        bobMessages.push(content);
        const aliKeyId = "curve25519:" + aliDeviceId;
-        const aliDeviceCurve25519Key = aliDeviceKeys.keys[aliKeyId];
+        const aliDeviceCurve25519Key = aliTestClient.deviceKeys.keys[aliKeyId];
        expect(utils.keys(content.ciphertext)).toEqual([aliDeviceCurve25519Key]);
        const ciphertext = content.ciphertext[aliDeviceCurve25519Key];
-        expect(ciphertext).toBeDefined();
+        expect(ciphertext).toBeTruthy();
        return ciphertext;
    });
 }
@@ -310,26 +265,30 @@ function sendMessage(client) {

 function expectSendMessageRequest(httpBackend) {
    const path = "/send/m.room.encrypted/";
-    let sent;
+    const deferred = q.defer();
    httpBackend.when("PUT", path).respond(200, function(path, content) {
-        sent = content;
+        deferred.resolve(content);
        return {
            event_id: "asdfgh",
        };
    });
-    return httpBackend.flush(path, 1).then(function() {
-        return sent;
-    });
+
+    // it can take a while to process the key query, so give it 20ms
+    return httpBackend.flush(path, 1, 20).then(() => deferred.promise);
 }

 function aliRecvMessage() {
    const message = bobMessages.shift();
-    return recvMessage(aliHttpBackend, aliClient, bobUserId, message);
+    return recvMessage(
+        aliTestClient.httpBackend, aliTestClient.client, bobUserId, message,
+    );
 }

 function bobRecvMessage() {
    const message = aliMessages.shift();
-    return recvMessage(bobHttpBackend, bobClient, aliUserId, message);
+    return recvMessage(
+        bobTestClient.httpBackend, bobTestClient.client, aliUserId, message,
+    );
 }

 function recvMessage(httpBackend, client, sender, message) {
@@ -382,52 +341,14 @@ function recvMessage(httpBackend, client, sender, message) {
 }


-function aliStartClient() {
-    expectAliKeyUpload().catch(testUtils.failTest);
-
-    // ali will try to query her own keys on start
-    aliHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
-        expect(content.device_keys[aliUserId]).toEqual({});
-        const result = {};
-        result[aliUserId] = {};
-        return {device_keys: result};
-    });
-
-    startClient(aliHttpBackend, aliClient);
-    return aliHttpBackend.flush().then(function() {
-        console.log("Ali client started");
-    });
-}
-
-function bobStartClient() {
-    expectBobKeyUpload().catch(testUtils.failTest);
-
-    // bob will try to query his own keys on start
-    bobHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
-        expect(content.device_keys[bobUserId]).toEqual({});
-        const result = {};
-        result[bobUserId] = {};
-        return {device_keys: result};
-    });
-
-    startClient(bobHttpBackend, bobClient);
-    return bobHttpBackend.flush().then(function() {
-        console.log("Bob client started");
-    });
-}
-
-
 /**
- * Set http responses for the requests which are made when a client starts, and
- * start the client.
+ * Send an initial sync response to the client (which just includes the member
+ * list for our test room).
 *
- * @param {object} httpBackend
- * @param {MatrixClient} client
+ * @param {TestClient} testClient
+ * @returns {Promise} which resolves when the sync has been flushed.
 */
-function startClient(httpBackend, client) {
-    httpBackend.when("GET", "/pushrules").respond(200, {});
-    httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
-
+function firstSync(testClient) {
    // send a sync response including our test room.
    const syncData = {
        next_batch: "x",
@@ -452,9 +373,11 @@ function startClient(httpBackend, client) {
            events: [],
        },
    };
-    httpBackend.when("GET", "/sync").respond(200, syncData);

-    client.startClient();
+    testClient.httpBackend.when("GET", "/sync").respond(200, syncData);
+    return testClient.httpBackend.flush("/sync", 1).then(() => {
+        return testUtils.syncPromise(testClient.client);
+    });
 }


@@ -466,90 +389,70 @@ describe("MatrixClient crypto", function() {
    beforeEach(function() {
        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this

-        aliStorage = new sdk.WebStorageSessionStore(new testUtils.MockStorageApi());
-        aliHttpBackend = new HttpBackend();
-        aliClient = sdk.createClient({
-            baseUrl: "http://alis.server",
-            userId: aliUserId,
-            accessToken: aliAccessToken,
-            deviceId: aliDeviceId,
-            sessionStore: aliStorage,
-            request: aliHttpBackend.requestFn,
-        });
+        aliTestClient = new TestClient(aliUserId, aliDeviceId, aliAccessToken);
+        bobTestClient = new TestClient(bobUserId, bobDeviceId, bobAccessToken);

-        bobStorage = new sdk.WebStorageSessionStore(new testUtils.MockStorageApi());
-        bobHttpBackend = new HttpBackend();
-        bobClient = sdk.createClient({
-            baseUrl: "http://bobs.server",
-            userId: bobUserId,
-            accessToken: bobAccessToken,
-            deviceId: bobDeviceId,
-            sessionStore: bobStorage,
-            request: bobHttpBackend.requestFn,
-        });
-
-        bobOneTimeKeys = undefined;
-        aliDeviceKeys = undefined;
-        bobDeviceKeys = undefined;
-        bobDeviceCurve25519Key = undefined;
-        bobDeviceEd25519Key = undefined;
        aliMessages = [];
        bobMessages = [];
    });

    afterEach(function() {
-        aliClient.stopClient();
-        bobClient.stopClient();
+        aliTestClient.stop();
+        aliTestClient.httpBackend.verifyNoOutstandingExpectation();
+        bobTestClient.stop();
+        bobTestClient.httpBackend.verifyNoOutstandingExpectation();
    });

    it('Ali knows the difference between a new user and one with no devices',
        function(done) {
-            aliHttpBackend.when('POST', '/keys/query').respond(200, {
+            aliTestClient.httpBackend.when('POST', '/keys/query').respond(200, {
                device_keys: {
                    '@bob:id': {},
                },
            });

-            const p1 = aliClient.downloadKeys(['@bob:id']);
-            const p2 = aliHttpBackend.flush('/keys/query', 1);
+            const p1 = aliTestClient.client.downloadKeys(['@bob:id']);
+            const p2 = aliTestClient.httpBackend.flush('/keys/query', 1);

            q.all([p1, p2]).then(function() {
-                const devices = aliStorage.getEndToEndDevicesForUser('@bob:id');
+                const devices = aliTestClient.storage.getEndToEndDevicesForUser(
+                    '@bob:id',
+                );
                expect(utils.keys(devices).length).toEqual(0);

                // request again: should be no more requests
-                return aliClient.downloadKeys(['@bob:id']);
+                return aliTestClient.client.downloadKeys(['@bob:id']);
            }).nodeify(done);
        },
    );

-    it("Bob uploads without one-time keys and with one-time keys", function(done) {
-        q()
-            .then(bobUploadsKeys)
-            .catch(testUtils.failTest).done(done);
+    it("Bob uploads device keys", function() {
+        return q()
+            .then(bobUploadsDeviceKeys);
    });

-    it("Ali downloads Bobs keys", function(done) {
+    it("Ali downloads Bobs device keys", function(done) {
        q()
-            .then(bobUploadsKeys)
+            .then(bobUploadsDeviceKeys)
            .then(aliDownloadsKeys)
            .catch(testUtils.failTest).done(done);
    });

    it("Ali gets keys with an invalid signature", function(done) {
        q()
-            .then(bobUploadsKeys)
+            .then(bobUploadsDeviceKeys)
            .then(function() {
-                // tamper bob's keys!
-                expect(bobDeviceKeys.keys["curve25519:" + bobDeviceId]).toBeDefined();
+                // tamper bob's keys
+                const bobDeviceKeys = bobTestClient.deviceKeys;
+                expect(bobDeviceKeys.keys["curve25519:" + bobDeviceId]).toBeTruthy();
                bobDeviceKeys.keys["curve25519:" + bobDeviceId] += "abc";

-                return q.all(aliClient.downloadKeys([bobUserId]),
+                return q.all(aliTestClient.client.downloadKeys([bobUserId]),
                             expectAliQueryKeys());
            })
            .then(function() {
                // should get an empty list
-                expect(aliClient.listDeviceKeys(bobUserId)).toEqual([]);
+                expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([]);
            })
            .catch(testUtils.failTest).done(done);
    });
@@ -575,19 +478,21 @@ describe("MatrixClient crypto", function() {

        const bobKeys = {};
        bobKeys[bobDeviceId] = bobDeviceKeys;
-        aliHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
+        aliTestClient.httpBackend.when(
+            "POST", "/keys/query",
+        ).respond(200, function(path, content) {
            const result = {};
            result[bobUserId] = bobKeys;
            return {device_keys: result};
        });

        q.all(
-            aliClient.downloadKeys([bobUserId, eveUserId]),
-            aliHttpBackend.flush("/keys/query", 1),
+            aliTestClient.client.downloadKeys([bobUserId, eveUserId]),
+            aliTestClient.httpBackend.flush("/keys/query", 1),
        ).then(function() {
            // should get an empty list
-            expect(aliClient.listDeviceKeys(bobUserId)).toEqual([]);
-            expect(aliClient.listDeviceKeys(eveUserId)).toEqual([]);
+            expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([]);
+            expect(aliTestClient.client.listDeviceKeys(eveUserId)).toEqual([]);
        }).catch(testUtils.failTest).done(done);
    });

@@ -610,33 +515,39 @@ describe("MatrixClient crypto", function() {

        const bobKeys = {};
        bobKeys[bobDeviceId] = bobDeviceKeys;
-        aliHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
+        aliTestClient.httpBackend.when(
+            "POST", "/keys/query",
+        ).respond(200, function(path, content) {
            const result = {};
            result[bobUserId] = bobKeys;
            return {device_keys: result};
        });

        q.all(
-            aliClient.downloadKeys([bobUserId]),
-            aliHttpBackend.flush("/keys/query", 1),
+            aliTestClient.client.downloadKeys([bobUserId]),
+            aliTestClient.httpBackend.flush("/keys/query", 1),
        ).then(function() {
            // should get an empty list
-            expect(aliClient.listDeviceKeys(bobUserId)).toEqual([]);
+            expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([]);
        }).catch(testUtils.failTest).done(done);
    });

-    it("Ali enables encryption", function(done) {
-        q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
-            .then(aliEnablesEncryption)
-            .catch(testUtils.failTest).done(done);
+
+    it("Bob starts his client and uploads device keys and one-time keys", function() {
+        return q()
+            .then(() => bobTestClient.start())
+            .then(() => bobTestClient.awaitOneTimeKeyUpload())
+            .then((keys) => {
+                expect(Object.keys(keys).length).toEqual(5);
+                expect(Object.keys(bobTestClient.deviceKeys).length).toNotEqual(0);
+            });
    });

    it("Ali sends a message", function(done) {
        q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+            .then(() => aliTestClient.start())
+            .then(() => bobTestClient.start())
+            .then(() => firstSync(aliTestClient))
            .then(aliEnablesEncryption)
            .then(aliSendsFirstMessage)
            .catch(testUtils.failTest).nodeify(done);
@@ -644,22 +555,22 @@ describe("MatrixClient crypto", function() {

    it("Bob receives a message", function(done) {
        q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+            .then(() => aliTestClient.start())
+            .then(() => bobTestClient.start())
+            .then(() => firstSync(aliTestClient))
            .then(aliEnablesEncryption)
            .then(aliSendsFirstMessage)
-            .then(bobStartClient)
            .then(bobRecvMessage)
            .catch(testUtils.failTest).done(done);
    });

    it("Bob receives a message with a bogus sender", function(done) {
        q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+            .then(() => aliTestClient.start())
+            .then(() => bobTestClient.start())
+            .then(() => firstSync(aliTestClient))
            .then(aliEnablesEncryption)
            .then(aliSendsFirstMessage)
-            .then(bobStartClient)
            .then(function() {
                const message = aliMessages.shift();
                const syncData = {
@@ -682,11 +593,11 @@ describe("MatrixClient crypto", function() {
                        ],
                    },
                };
-                bobHttpBackend.when("GET", "/sync").respond(200, syncData);
+                bobTestClient.httpBackend.when("GET", "/sync").respond(200, syncData);

                const deferred = q.defer();
                const onEvent = function(event) {
-                    console.log(bobClient.credentials.userId + " received event",
+                    console.log(bobUserId + " received event",
                                event);

                    // ignore the m.room.member events
@@ -698,13 +609,13 @@ describe("MatrixClient crypto", function() {
                    expect(event.getContent().msgtype).toEqual("m.bad.encrypted");
                    expect(event.isEncrypted()).toBeTruthy();

-                    bobClient.removeListener("event", onEvent);
+                    bobTestClient.client.removeListener("event", onEvent);
                    deferred.resolve();
                };

-                bobClient.on("event", onEvent);
+                bobTestClient.client.on("event", onEvent);

-                bobHttpBackend.flush();
+                bobTestClient.httpBackend.flush();
                return deferred.promise;
            })
            .catch(testUtils.failTest).done(done);
@@ -712,58 +623,56 @@ describe("MatrixClient crypto", function() {

    it("Ali blocks Bob's device", function(done) {
        q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+            .then(() => aliTestClient.start())
+            .then(() => bobTestClient.start())
+            .then(() => firstSync(aliTestClient))
            .then(aliEnablesEncryption)
            .then(aliDownloadsKeys)
            .then(function() {
-                aliClient.setDeviceBlocked(bobUserId, bobDeviceId, true);
-                const p1 = sendMessage(aliClient);
-                const p2 = expectAliQueryKeys()
-                    .then(expectAliClaimKeys)
-                    .then(function() {
-                        return expectSendMessageRequest(aliHttpBackend);
-                    }).then(function(sentContent) {
-                        // no unblocked devices, so the ciphertext should be empty
-                        expect(sentContent.ciphertext).toEqual({});
-                    });
+                aliTestClient.client.setDeviceBlocked(bobUserId, bobDeviceId, true);
+                const p1 = sendMessage(aliTestClient.client);
+                const p2 = expectSendMessageRequest(aliTestClient.httpBackend)
+                      .then(function(sentContent) {
+                          // no unblocked devices, so the ciphertext should be empty
+                          expect(sentContent.ciphertext).toEqual({});
+                      });
                return q.all([p1, p2]);
-            }).catch(testUtils.failTest).nodeify(done);
+            }).nodeify(done);
    });

    it("Bob receives two pre-key messages", function(done) {
        q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+            .then(() => aliTestClient.start())
+            .then(() => bobTestClient.start())
+            .then(() => firstSync(aliTestClient))
            .then(aliEnablesEncryption)
            .then(aliSendsFirstMessage)
-            .then(bobStartClient)
            .then(bobRecvMessage)
            .then(aliSendsMessage)
            .then(bobRecvMessage)
            .catch(testUtils.failTest).done(done);
    });

-    it("Bob replies to the message", function(done) {
-        q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+    it("Bob replies to the message", function() {
+        return q()
+            .then(() => aliTestClient.start())
+            .then(() => bobTestClient.start())
+            .then(() => firstSync(aliTestClient))
+            .then(() => firstSync(bobTestClient))
            .then(aliEnablesEncryption)
            .then(aliSendsFirstMessage)
-            .then(bobStartClient)
            .then(bobRecvMessage)
            .then(bobEnablesEncryption)
            .then(bobSendsReplyMessage).then(function(ciphertext) {
                expect(ciphertext.type).toEqual(1);
-            }).then(aliRecvMessage)
-            .catch(testUtils.failTest).done(done);
+            }).then(aliRecvMessage);
    });


-    it("Ali does a key query when she gets a new_device event", function(done) {
-        q()
-            .then(bobUploadsKeys)
-            .then(aliStartClient)
+    it("Ali does a key query when she gets a new_device event", function() {
+        return q()
+            .then(() => aliTestClient.start())
+            .then(() => firstSync(aliTestClient))
            .then(function() {
                const syncData = {
                    next_batch: '2',
@@ -780,9 +689,56 @@ describe("MatrixClient crypto", function() {
                        ],
                    },
                };
-                aliHttpBackend.when('GET', '/sync').respond(200, syncData);
-                return aliHttpBackend.flush('/sync', 1);
-            }).then(expectAliQueryKeys)
-            .nodeify(done);
+                aliTestClient.httpBackend.when('GET', '/sync').respond(200, syncData);
+                return aliTestClient.httpBackend.flush('/sync', 1);
+            }).then(() => {
+                aliTestClient.expectKeyQuery({
+                    device_keys: {
+                        [bobUserId]: {},
+                    },
+                });
+                return aliTestClient.httpBackend.flush('/keys/query', 1);
+            });
+    });
+
+    it("Ali does a key query when encryption is enabled", function() {
+        // enabling encryption in the room should make alice download devices
+        // for both members.
+        return q()
+            .then(() => aliTestClient.start())
+            .then(() => firstSync(aliTestClient))
+            .then(() => {
+                const syncData = {
+                    next_batch: '2',
+                    rooms: {
+                        join: {},
+                    },
+                };
+                syncData.rooms.join[roomId] = {
+                    state: {
+                        events: [
+                            testUtils.mkEvent({
+                                type: 'm.room.encryption',
+                                skey: '',
+                                content: {
+                                    algorithm: 'm.olm.v1.curve25519-aes-sha2',
+                                },
+                            }),
+                        ],
+                    },
+                };
+
+                aliTestClient.httpBackend.when('GET', '/sync').respond(
+                    200, syncData);
+                return aliTestClient.httpBackend.flush('/sync', 1);
+            }).then(() => {
+                aliTestClient.expectKeyQuery({
+                    device_keys: {
+                        [aliUserId]: {},
+                        [bobUserId]: {},
+                    },
+                });
+                return aliTestClient.httpBackend.flush('/keys/query', 1);
+            });
    });
 });
@@ -1,8 +1,11 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const HttpBackend = require("../mock-request");
 const utils = require("../test-utils");

+import expect from 'expect';
+
 describe("MatrixClient events", function() {
    const baseUrl = "http://localhost.or.something";
    let client;
@@ -141,8 +144,8 @@ describe("MatrixClient events", function() {
            let fired = false;
            client.on("User.presence", function(event, user) {
                fired = true;
-                expect(user).toBeDefined();
-                expect(event).toBeDefined();
+                expect(user).toBeTruthy();
+                expect(event).toBeTruthy();
                if (!user || !event) {
                    return;
                }
@@ -207,7 +210,7 @@ describe("MatrixClient events", function() {
            client.on("RoomState.events", function(event, state) {
                eventsInvokeCount++;
                const index = roomStateEventTypes.indexOf(event.getType());
-                expect(index).not.toEqual(
+                expect(index).toNotEqual(
                    -1, "Unexpected room state event type: " + event.getType(),
                );
                if (index >= 0) {
@@ -1,4 +1,5 @@
 "use strict";
+import 'source-map-support/register';
 const q = require("q");
 const sdk = require("../..");
 const HttpBackend = require("../mock-request");
@@ -94,7 +95,6 @@ function startClient(httpBackend, client) {
    return deferred.promise;
 }

-
 describe("getEventTimeline support", function() {
    let httpBackend;
    let client;
@@ -142,7 +142,7 @@ describe("getEventTimeline support", function() {
            const timelineSet = room.getTimelineSets()[0];
            expect(function() {
                client.getEventTimeline(timelineSet, "event");
-            }).not.toThrow();
+            }).toNotThrow();
        }).catch(utils.failTest).done(done);

        httpBackend.flush().catch(utils.failTest);
@@ -219,6 +219,8 @@ describe("getEventTimeline support", function() {
    });
 });

+import expect from 'expect';
+
 describe("MatrixClient event timelines", function() {
    let client = null;
    let httpBackend = null;
@@ -310,7 +312,7 @@ describe("MatrixClient event timelines", function() {
            httpBackend.flush().catch(utils.failTest);
        });

-        it("should update timelines where they overlap a previous /sync", function(done) {
+        it("should update timelines where they overlap a previous /sync", function() {
            const room = client.getRoom(roomId);
            const timelineSet = room.getTimelineSets()[0];
            httpBackend.when("GET", "/sync").respond(200, {
@@ -342,6 +344,7 @@ describe("MatrixClient event timelines", function() {
                    };
                });

+            const deferred = q.defer();
            client.on("sync", function() {
                client.getEventTimeline(timelineSet, EVENTS[2].event_id,
                ).then(function(tl) {
@@ -353,10 +356,14 @@ describe("MatrixClient event timelines", function() {
                        .toEqual("start_token");
                    // expect(tl.getPaginationToken(EventTimeline.FORWARDS))
                    //    .toEqual("s_5_4");
-                }).catch(utils.failTest).done(done);
+                }).done(() => deferred.resolve(),
+                        (e) => deferred.reject(e));
            });

-            httpBackend.flush().catch(utils.failTest);
+            return q.all([
+                httpBackend.flush(),
+                deferred.promise,
+            ]);
        });

        it("should join timelines where they overlap a previous /context",
@@ -631,6 +638,8 @@ describe("MatrixClient event timelines", function() {

                // now let the sync complete, and check it again
                return httpBackend.flush("/sync", 1);
+            }).then(function() {
+                return utils.syncPromise(client);
            }).then(function() {
                return client.getEventTimeline(timelineSet, event.event_id);
            }).then(function(tl) {
@@ -658,6 +667,8 @@ describe("MatrixClient event timelines", function() {
            }).catch(utils.failTest).done(done);

            httpBackend.flush("/sync", 1).then(function() {
+                return utils.syncPromise(client);
+            }).then(function() {
                return client.getEventTimeline(timelineSet, event.event_id);
            }).then(function(tl) {
                console.log("getEventTimeline completed (1)");
@@ -1,4 +1,5 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const HttpBackend = require("../mock-request");
 const publicGlobals = require("../../lib/matrix");
@@ -8,6 +9,8 @@ const Filter = publicGlobals.Filter;
 const utils = require("../test-utils");
 const MockStorageApi = require("../MockStorageApi");

+import expect from 'expect';
+
 describe("MatrixClient", function() {
    const baseUrl = "http://localhost.or.something";
    let client = null;
@@ -60,7 +63,7 @@ describe("MatrixClient", function() {
                type: "text/plain",
            });

-            expect(prom).toBeDefined();
+            expect(prom).toBeTruthy();

            const uploads = client.getCurrentUploads();
            expect(uploads.length).toEqual(1);
@@ -208,14 +211,14 @@ describe("MatrixClient", function() {
            const httpFilterDefinition = {
                event_format: "federation",
            };
-            expect(store.getFilter(userId, filterId)).toBeNull();
+            expect(store.getFilter(userId, filterId)).toBe(null);

            httpBackend.when(
                "GET", "/user/" + encodeURIComponent(userId) + "/filter/" + filterId,
            ).respond(200, httpFilterDefinition);
            client.getFilter(userId, filterId, true).done(function(gotFilter) {
                expect(gotFilter.getDefinition()).toEqual(httpFilterDefinition);
-                expect(store.getFilter(userId, filterId)).toBeDefined();
+                expect(store.getFilter(userId, filterId)).toBeTruthy();
                done();
            });

@@ -227,7 +230,7 @@ describe("MatrixClient", function() {
        const filterId = "f1llllllerid";

        it("should do an HTTP request and then store the filter", function(done) {
-            expect(store.getFilter(userId, filterId)).toBeNull();
+            expect(store.getFilter(userId, filterId)).toBe(null);

            const filterDefinition = {
                event_format: "client",
@@ -348,7 +351,6 @@ describe("MatrixClient", function() {

            httpBackend.when("POST", "/keys/query").check(function(req) {
                expect(req.data).toEqual({device_keys: {
-                    '@alice:localhost': {},
                    'boris': {},
                    'chaz': {},
                }});
@@ -1,9 +1,12 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const MatrixClient = sdk.MatrixClient;
 const HttpBackend = require("../mock-request");
 const utils = require("../test-utils");

+import expect from 'expect';
+
 describe("MatrixClient opts", function() {
    const baseUrl = "http://localhost.or.something";
    let client = null;
@@ -96,7 +99,7 @@ describe("MatrixClient opts", function() {
                "m.room.create",
            ];
            client.on("event", function(event) {
-                expect(expectedEventTypes.indexOf(event.getType())).not.toEqual(
+                expect(expectedEventTypes.indexOf(event.getType())).toNotEqual(
                    -1, "Recv unexpected event type: " + event.getType(),
                );
                expectedEventTypes.splice(
@@ -111,6 +114,8 @@ describe("MatrixClient opts", function() {
                return httpBackend.flush("/filter", 1);
            }).then(function() {
                return httpBackend.flush("/sync", 1);
+            }).then(function() {
+                return utils.syncPromise(client);
            }).done(function() {
                expect(expectedEventTypes.length).toEqual(
                    0, "Expected to see event types: " + expectedEventTypes,
@@ -1,9 +1,12 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const HttpBackend = require("../mock-request");
 const utils = require("../test-utils");
 const EventStatus = sdk.EventStatus;

+import expect from 'expect';
+
 describe("MatrixClient retrying", function() {
    const baseUrl = "http://localhost.or.something";
    let client = null;
@@ -1,9 +1,12 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const EventStatus = sdk.EventStatus;
 const HttpBackend = require("../mock-request");
 const utils = require("../test-utils");

+import expect from 'expect';
+
 describe("MatrixClient room timelines", function() {
    const baseUrl = "http://localhost.or.something";
    let client = null;
@@ -120,7 +123,7 @@ describe("MatrixClient room timelines", function() {
        client.startClient();
        httpBackend.flush("/pushrules").then(function() {
            return httpBackend.flush("/filter");
-        }).done(done);
+        }).nodeify(done);
    });

    afterEach(function() {
@@ -247,7 +250,7 @@ describe("MatrixClient room timelines", function() {

                client.scrollback(room).done(function() {
                    expect(room.timeline.length).toEqual(1);
-                    expect(room.oldState.paginationToken).toBeNull();
+                    expect(room.oldState.paginationToken).toBe(null);
                    done();
                });

@@ -367,7 +370,7 @@ describe("MatrixClient room timelines", function() {
                    return;
                }
                const room = client.getRoom(roomId);
-                expect(room.oldState.paginationToken).toBeDefined();
+                expect(room.oldState.paginationToken).toBeTruthy();

                client.scrollback(room, 1).done(function() {
                    expect(room.oldState.paginationToken).toEqual(sbEndTok);
@@ -406,6 +409,8 @@ describe("MatrixClient room timelines", function() {

                httpBackend.flush("/messages", 1);
                httpBackend.flush("/sync", 1).then(function() {
+                    return utils.syncPromise(client);
+                }).then(function() {
                    expect(index).toEqual(2);
                    expect(room.timeline.length).toEqual(3);
                    expect(room.timeline[2].event).toEqual(
@@ -436,6 +441,8 @@ describe("MatrixClient room timelines", function() {
                }
                const room = client.getRoom(roomId);
                httpBackend.flush("/sync", 1).then(function() {
+                    return utils.syncPromise(client);
+                }).then(function() {
                    const preNameEvent = room.timeline[room.timeline.length - 3];
                    const postNameEvent = room.timeline[room.timeline.length - 1];
                    expect(preNameEvent.sender.name).toEqual(userName);
@@ -464,7 +471,9 @@ describe("MatrixClient room timelines", function() {
                    nameEmitCount += 1;
                });

-                httpBackend.flush("/sync", 1).done(function() {
+                httpBackend.flush("/sync", 1).then(() => {
+                    return utils.syncPromise(client);
+                }).done(function() {
                    expect(nameEmitCount).toEqual(1);
                    expect(room.name).toEqual("Room 2");
                    // do another round
@@ -476,7 +485,9 @@ describe("MatrixClient room timelines", function() {
                    thirdRoomNameEvent.__prev_event = secondRoomNameEvent;
                    setNextSyncData([thirdRoomNameEvent]);
                    httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
-                    httpBackend.flush("/sync", 1).done(function() {
+                    httpBackend.flush("/sync", 1).then(() => {
+                        return utils.syncPromise(client);
+                    }).done(function() {
                        expect(nameEmitCount).toEqual(2);
                        expect(room.name).toEqual("Room 3");
                        done();
@@ -507,6 +518,8 @@ describe("MatrixClient room timelines", function() {
                }
                const room = client.getRoom(roomId);
                httpBackend.flush("/sync", 1).then(function() {
+                    return utils.syncPromise(client);
+                }).then(function() {
                    expect(room.currentState.getMembers().length).toEqual(4);
                    expect(room.currentState.getMember(userC).name).toEqual("C");
                    expect(room.currentState.getMember(userC).membership).toEqual(
@@ -537,7 +550,9 @@ describe("MatrixClient room timelines", function() {
                const room = client.getRoom(roomId);

                httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1).done(function() {
+                httpBackend.flush("/sync", 1).then(function() {
+                    return utils.syncPromise(client);
+                }).done(function() {
                    expect(room.timeline.length).toEqual(1);
                    expect(room.timeline[0].event).toEqual(eventData[0]);
                    expect(room.currentState.getMembers().length).toEqual(2);
@@ -575,7 +590,9 @@ describe("MatrixClient room timelines", function() {
                });

                httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1).done(function() {
+                httpBackend.flush("/sync", 1).then(function() {
+                    return utils.syncPromise(client);
+                }).done(function() {
                    expect(emitCount).toEqual(1);
                    done();
                });
@@ -1,10 +1,13 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const HttpBackend = require("../mock-request");
 const utils = require("../test-utils");
 const MatrixEvent = sdk.MatrixEvent;
 const EventTimeline = sdk.EventTimeline;

+import expect from 'expect';
+
 describe("MatrixClient syncing", function() {
    const baseUrl = "http://localhost.or.something";
    let client = null;
@@ -134,7 +137,7 @@ describe("MatrixClient syncing", function() {
                expect(member.name).toEqual("The Boss");
                expect(
                    member.getAvatarUrl("home.server.url", null, null, null, false),
-                ).toBeDefined();
+                ).toBeTruthy();
                done();
            });
        });
@@ -211,7 +214,7 @@ describe("MatrixClient syncing", function() {
                expect(member.name).toEqual(userC);
                expect(
                    member.getAvatarUrl("home.server.url", null, null, null, false),
-                ).toBeNull();
+                ).toBe(null);
                done();
            });
        });
@@ -405,10 +408,10 @@ describe("MatrixClient syncing", function() {
            httpBackend.flush().done(function() {
                const room = client.getRoom(roomTwo);
                let member = room.getMember(otherUserId);
-                expect(member).toBeDefined();
+                expect(member).toBeTruthy();
                expect(member.typing).toEqual(true);
                member = room.getMember(selfUserId);
-                expect(member).toBeDefined();
+                expect(member).toBeTruthy();
                expect(member.typing).toEqual(false);
                done();
            });
@@ -16,6 +16,7 @@ limitations under the License.

 "use strict";

+import 'source-map-support/register';

 let Olm = null;
 try {
@@ -24,131 +25,33 @@ try {

 const anotherjson = require('another-json');
 const q = require('q');
+import expect from 'expect';

 const sdk = require('../..');
 const utils = require('../../lib/utils');
 const testUtils = require('../test-utils');
-const MockHttpBackend = require('../mock-request');
+const TestClient = require('../TestClient').default;

 const ROOM_ID = "!room:id";

-/**
- * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
- *
- * @constructor
- * @param {string} userId
- * @param {string} deviceId
- * @param {string} accessToken
- */
-function TestClient(userId, deviceId, accessToken) {
-    this.userId = userId;
-    this.deviceId = deviceId;
-
-    this.storage = new sdk.WebStorageSessionStore(new testUtils.MockStorageApi());
-    this.httpBackend = new MockHttpBackend();
-    this.client = sdk.createClient({
-        baseUrl: "http://test.server",
-        userId: userId,
-        accessToken: accessToken,
-        deviceId: deviceId,
-        sessionStore: this.storage,
-        request: this.httpBackend.requestFn,
-    });
-
-    this.deviceKeys = null;
-    this.oneTimeKeys = [];
-}
-
-/**
- * start the client, and wait for it to initialise.
- *
- * @param {object?} existingDevices  the list of our existing devices to return from
- *    the /query request. Defaults to empty device list
- * @return {Promise}
- */
-TestClient.prototype.start = function(existingDevices) {
-    const self = this;
-
-    this.httpBackend.when("GET", "/pushrules").respond(200, {});
-    this.httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
-
-    this.httpBackend.when('POST', '/keys/query').respond(200, function(path, content) {
-        expect(content.device_keys[self.userId]).toEqual({});
-        let res = existingDevices;
-        if (!res) {
-            res = { device_keys: {} };
-            res.device_keys[self.userId] = {};
-        }
-        return res;
-    });
-    this.httpBackend.when("POST", "/keys/upload").respond(200, function(path, content) {
-        expect(content.one_time_keys).not.toBeDefined();
-        expect(content.device_keys).toBeDefined();
-        self.deviceKeys = content.device_keys;
-        return {one_time_key_counts: {signed_curve25519: 0}};
-    });
-    this.httpBackend.when("POST", "/keys/upload").respond(200, function(path, content) {
-        expect(content.device_keys).not.toBeDefined();
-        expect(content.one_time_keys).toBeDefined();
-        expect(content.one_time_keys).not.toEqual({});
-        self.oneTimeKeys = content.one_time_keys;
-        return {one_time_key_counts: {
-            signed_curve25519: utils.keys(self.oneTimeKeys).length,
-        }};
-    });
-
-    this.client.startClient({
-        // set this so that we can get hold of failed events
-        pendingEventOrdering: 'detached',
-    });
-
-    return this.httpBackend.flush();
-};
-
-/**
- * stop the client
- */
-TestClient.prototype.stop = function() {
-    this.client.stopClient();
-};
-
-/**
- * get the uploaded curve25519 device key
- *
- * @return {string} base64 device key
- */
-TestClient.prototype.getDeviceKey = function() {
-    const keyId = 'curve25519:' + this.deviceId;
-    return this.deviceKeys.keys[keyId];
-};
-
-
-/**
- * get the uploaded ed25519 device key
- *
- * @return {string} base64 device key
- */
-TestClient.prototype.getSigningKey = function() {
-    const keyId = 'ed25519:' + this.deviceId;
-    return this.deviceKeys.keys[keyId];
-};
-
 /**
 * start an Olm session with a given recipient
 *
 * @param {Olm.Account} olmAccount
 * @param {TestClient} recipientTestClient
- * @return {Olm.Session}
+ * @return {Promise} promise for Olm.Session
 */
 function createOlmSession(olmAccount, recipientTestClient) {
-    const otkId = utils.keys(recipientTestClient.oneTimeKeys)[0];
-    const otk = recipientTestClient.oneTimeKeys[otkId];
+    return recipientTestClient.awaitOneTimeKeyUpload().then((keys) => {
+        const otkId = utils.keys(keys)[0];
+        const otk = keys[otkId];

-    const session = new Olm.Session();
-    session.create_outbound(
-        olmAccount, recipientTestClient.getDeviceKey(), otk.key,
-    );
-    return session;
+        const session = new Olm.Session();
+        session.create_outbound(
+            olmAccount, recipientTestClient.getDeviceKey(), otk.key,
+        );
+        return session;
+    });
 }

 /**
@@ -165,9 +68,9 @@ function createOlmSession(olmAccount, recipientTestClient) {
 * @return {object} event
 */
 function encryptOlmEvent(opts) {
-    expect(opts.senderKey).toBeDefined();
-    expect(opts.p2pSession).toBeDefined();
-    expect(opts.recipient).toBeDefined();
+    expect(opts.senderKey).toBeTruthy();
+    expect(opts.p2pSession).toBeTruthy();
+    expect(opts.recipient).toBeTruthy();

    const plaintext = {
        content: opts.plaincontent || {},
@@ -205,8 +108,8 @@ function encryptOlmEvent(opts) {
 * @return {object} event
 */
 function encryptMegolmEvent(opts) {
-    expect(opts.senderKey).toBeDefined();
-    expect(opts.groupSession).toBeDefined();
+    expect(opts.senderKey).toBeTruthy();
+    expect(opts.groupSession).toBeTruthy();

    const plaintext = opts.plaintext || {};
    if (!plaintext.content) {
@@ -219,7 +122,7 @@ function encryptMegolmEvent(opts) {
        plaintext.type = "m.room.message";
    }
    if (!plaintext.room_id) {
-        expect(opts.room_id).toBeDefined();
+        expect(opts.room_id).toBeTruthy();
        plaintext.room_id = opts.room_id;
    }

@@ -401,9 +304,9 @@ describe("megolm", function() {
    });

    it("Alice receives a megolm message", function(done) {
-        return aliceTestClient.start().then(function() {
-            const p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
-
+        return aliceTestClient.start().then(() => {
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((p2pSession) => {
            const groupSession = new Olm.OutboundGroupSession();
            groupSession.create();

@@ -441,6 +344,75 @@ describe("megolm", function() {

            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
            return aliceTestClient.httpBackend.flush("/sync", 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
+        }).then(function() {
+            const room = aliceTestClient.client.getRoom(ROOM_ID);
+            const event = room.getLiveTimeline().getEvents()[0];
+            expect(event.getContent().body).toEqual('42');
+        }).nodeify(done);
+    });
+
+    it("Alice receives a megolm message before the session keys", function(done) {
+        // https://github.com/vector-im/riot-web/issues/2273
+        let roomKeyEncrypted;
+
+        return aliceTestClient.start().then(() => {
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((p2pSession) => {
+            const groupSession = new Olm.OutboundGroupSession();
+            groupSession.create();
+
+            // make the room_key event, but don't send it yet
+            roomKeyEncrypted = encryptGroupSessionKey({
+                senderKey: testSenderKey,
+                recipient: aliceTestClient,
+                p2pSession: p2pSession,
+                groupSession: groupSession,
+                room_id: ROOM_ID,
+            });
+
+            // encrypt a message with the group session
+            const messageEncrypted = encryptMegolmEvent({
+                senderKey: testSenderKey,
+                groupSession: groupSession,
+                room_id: ROOM_ID,
+            });
+
+            // Alice just gets the message event to start with
+            const syncResponse = {
+                next_batch: 1,
+                rooms: {
+                    join: {},
+                },
+            };
+            syncResponse.rooms.join[ROOM_ID] = {
+                timeline: {
+                    events: [messageEncrypted],
+                },
+            };
+
+            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
+            return aliceTestClient.httpBackend.flush("/sync", 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
+        }).then(function() {
+            const room = aliceTestClient.client.getRoom(ROOM_ID);
+            const event = room.getLiveTimeline().getEvents()[0];
+            expect(event.getContent().msgtype).toEqual('m.bad.encrypted');
+
+            // now she gets the room_key event
+            const syncResponse = {
+                next_batch: 2,
+                to_device: {
+                    events: [roomKeyEncrypted],
+                },
+            };
+
+            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
+            return aliceTestClient.httpBackend.flush("/sync", 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
        }).then(function() {
            const room = aliceTestClient.client.getRoom(ROOM_ID);
            const event = room.getLiveTimeline().getEvents()[0];
@@ -449,9 +421,9 @@ describe("megolm", function() {
    });

    it("Alice gets a second room_key message", function(done) {
-        return aliceTestClient.start().then(function() {
-            const p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
-
+        return aliceTestClient.start().then(() => {
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((p2pSession) => {
            const groupSession = new Olm.OutboundGroupSession();
            groupSession.create();

@@ -509,6 +481,8 @@ describe("megolm", function() {
            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse2);

            return aliceTestClient.httpBackend.flush("/sync", 2);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
        }).then(function() {
            const room = aliceTestClient.client.getRoom(ROOM_ID);
            const event = room.getLiveTimeline().getEvents()[0];
@@ -519,11 +493,13 @@ describe("megolm", function() {
    it('Alice sends a megolm message', function(done) {
        let p2pSession;

-        return aliceTestClient.start().then(function() {
-            const syncResponse = getSyncResponse(['@bob:xyz']);
-
+        return aliceTestClient.start().then(() => {
            // establish an olm session with alice
-            p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((_p2pSession) => {
+            p2pSession = _p2pSession;
+
+            const syncResponse = getSyncResponse(['@bob:xyz']);

            const olmEvent = encryptOlmEvent({
                senderKey: testSenderKey,
@@ -535,6 +511,8 @@ describe("megolm", function() {

            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
            return aliceTestClient.httpBackend.flush('/sync', 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
        }).then(function() {
            // start out with the device unknown - the send should be rejected.
            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
@@ -600,6 +578,8 @@ describe("megolm", function() {
            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);

            return aliceTestClient.httpBackend.flush('/sync', 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
        }).then(function() {
            console.log("Forcing alice to download our device keys");

@@ -631,11 +611,11 @@ describe("megolm", function() {


    it("We shouldn't attempt to send to blocked devices", function(done) {
-        return aliceTestClient.start().then(function() {
-            const syncResponse = getSyncResponse(['@bob:xyz']);
-
+        return aliceTestClient.start().then(() => {
            // establish an olm session with alice
-            const p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((p2pSession) => {
+            const syncResponse = getSyncResponse(['@bob:xyz']);

            const olmEvent = encryptOlmEvent({
                senderKey: testSenderKey,
@@ -647,6 +627,8 @@ describe("megolm", function() {
            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);

            return aliceTestClient.httpBackend.flush('/sync', 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
        }).then(function() {
            console.log('Forcing alice to download our device keys');

@@ -680,11 +662,13 @@ describe("megolm", function() {
        let p2pSession;
        let megolmSessionId;

-        return aliceTestClient.start().then(function() {
-            const syncResponse = getSyncResponse(['@bob:xyz']);
-
+        return aliceTestClient.start().then(() => {
            // establish an olm session with alice
-            p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((_p2pSession) => {
+            p2pSession = _p2pSession;
+
+            const syncResponse = getSyncResponse(['@bob:xyz']);

            const olmEvent = encryptOlmEvent({
                senderKey: testSenderKey,
@@ -696,6 +680,8 @@ describe("megolm", function() {
            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);

            return aliceTestClient.httpBackend.flush('/sync', 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
        }).then(function() {
            console.log("Fetching bob's devices and marking known");

@@ -749,7 +735,7 @@ describe("megolm", function() {
                'PUT', '/send/',
            ).respond(200, function(path, content) {
                console.log('/send:', content);
-                expect(content.session_id).not.toEqual(megolmSessionId);
+                expect(content.session_id).toNotEqual(megolmSessionId);
                return {
                    event_id: '$event_id',
                };
@@ -771,9 +757,7 @@ describe("megolm", function() {
        let inboundGroupSession;
        let decrypted;

-        return aliceTestClient.start(
-            getTestKeysQueryResponse(aliceTestClient.userId),
-        ).then(function() {
+        return aliceTestClient.start().then(function() {
            // an encrypted room with just alice
            const syncResponse = {
                next_batch: 1,
@@ -800,8 +784,29 @@ describe("megolm", function() {
            };
            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);

+            // the completion of the first initialsync hould make Alice
+            // invalidate the device cache for all members in e2e rooms (ie,
+            // herself), and do a key query.
+            aliceTestClient.expectKeyQuery(
+                getTestKeysQueryResponse(aliceTestClient.userId),
+            );
+
            return aliceTestClient.httpBackend.flush();
        }).then(function() {
+            // start out with the device unknown - the send should be rejected.
+            return q.all([
+                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test').then(() => {
+                    throw new Error("sendTextMessage failed on an unknown device");
+                }, (e) => {
+                    expect(e.name).toEqual("UnknownDeviceError");
+                    expect(Object.keys(e.devices)).toEqual([aliceTestClient.userId]);
+                    expect(Object.keys(e.devices[aliceTestClient.userId])).
+                        toEqual(['DEVICE_ID']);
+                }),
+                aliceTestClient.httpBackend.flush(),
+            ]);
+        }).then(function() {
+            // mark the device as known, and resend.
            aliceTestClient.client.setDeviceKnown(aliceTestClient.userId, 'DEVICE_ID');
            aliceTestClient.httpBackend.when('POST', '/keys/claim').respond(
                200, function(path, content) {
@@ -890,11 +895,11 @@ describe("megolm", function() {
            };
        });

-        return aliceTestClient.start().then(function() {
-            const syncResponse = getSyncResponse(['@bob:xyz']);
-
+        return aliceTestClient.start().then(() => {
            // establish an olm session with alice
-            p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((p2pSession) => {
+            const syncResponse = getSyncResponse(['@bob:xyz']);

            const olmEvent = encryptOlmEvent({
                senderKey: testSenderKey,
@@ -906,6 +911,8 @@ describe("megolm", function() {

            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
            return aliceTestClient.httpBackend.flush('/sync', 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
        }).then(function() {
            console.log('Forcing alice to download our device keys');

@@ -931,14 +938,155 @@ describe("megolm", function() {
    });


+    it("We should not get confused by out-of-order device query responses",
+       () => {
+           // https://github.com/vector-im/riot-web/issues/3126
+           return aliceTestClient.start().then(() => {
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                   200, getSyncResponse(['@bob:xyz', '@chris:abc']));
+               return aliceTestClient.httpBackend.flush('/sync', 1);
+           }).then(() => {
+               return testUtils.syncPromise(aliceTestClient.client);
+           }).then(() => {
+               // to make sure the initial device queries are flushed out, we
+               // attempt to send a message.
+
+               aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
+                   200, {
+                       device_keys: {
+                           '@bob:xyz': {},
+                           '@chris:abc': {},
+                       },
+                   },
+               );
+
+               aliceTestClient.httpBackend.when('PUT', '/send/').respond(
+                   200, {event_id: '$event1'});
+
+               return q.all([
+                   aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+                   aliceTestClient.httpBackend.flush('/keys/query', 1).then(
+                       () => aliceTestClient.httpBackend.flush('/send/', 1, 20),
+                   ),
+               ]);
+           }).then(() => {
+               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(1);
+
+               // invalidate bob's and chris's device lists in separate syncs
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                   next_batch: '2',
+                   device_lists: {
+                       changed: ['@bob:xyz'],
+                   },
+               });
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                   next_batch: '3',
+                   device_lists: {
+                       changed: ['@chris:abc'],
+                   },
+               });
+               return aliceTestClient.httpBackend.flush('/sync', 2);
+           }).then(() => {
+               return testUtils.syncPromise(aliceTestClient.client);
+           }).then(() => {
+               // check that we don't yet have a request for chris's devices.
+               aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                   device_keys: {
+                       '@chris:abc': {},
+                   },
+                   token: '3',
+               }).respond(200, {
+                   device_keys: {'@chris:abc': {}},
+               });
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(0);
+               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(1);
+
+               // now add an expectation for a query for bob's devices, and let
+               // it complete.
+               aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                   device_keys: {
+                       '@bob:xyz': {},
+                   },
+                   token: '2',
+               }).respond(200, {
+                   device_keys: {'@bob:xyz': {}},
+               });
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(1);
+
+               // wait for the client to stop processing the response
+               return aliceTestClient.client.downloadKeys(['@bob:xyz']);
+           }).then(() => {
+               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(2);
+
+               // now let the query for chris's devices complete.
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(1);
+
+               // wait for the client to stop processing the response
+               return aliceTestClient.client.downloadKeys(['@chris:abc']);
+           }).then(() => {
+               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(3);
+           });
+       });
+
+    it("Device list downloads before /changes shouldn't affect sync token",
+        () => {
+            // https://github.com/vector-im/riot-web/issues/3126#issuecomment-279374939
+            aliceTestClient.storage.storeEndToEndDeviceSyncToken(0);
+            aliceTestClient.storage.storeEndToEndRoom(ROOM_ID, {
+                algorithm: 'm.megolm.v1.aes-sha2',
+            });
+
+            return aliceTestClient.start().then(() => {
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                   200, getSyncResponse([aliceTestClient.userId, '@bob:xyz']));
+               return aliceTestClient.httpBackend.flush('/sync', 1);
+            }).then(() => {
+                aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
+                    200, {device_keys: {'@bob:xyz': {}}},
+                );
+                return q.all([
+                    aliceTestClient.client.downloadKeys(['@bob:xyz']),
+                    aliceTestClient.httpBackend.flush('/keys/query', 1),
+                ]);
+           }).then(() => {
+               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(0);
+
+               aliceTestClient.httpBackend.when(
+                   'GET', '/keys/changes',
+               ).check((req) => {
+                   expect(req.queryParams.from).toEqual(0);
+                   expect(req.queryParams.to).toEqual(1);
+               }).respond(200, {changed: ['@bob:xyz']});
+
+               return aliceTestClient.httpBackend.flush('/keys/changes');
+           }).then((flushed) => {
+               aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
+                   200, {device_keys: {'@bob:xyz': {}}},
+               );
+               return aliceTestClient.httpBackend.flush('/keys/query');
+           }).then((flushed) => {
+               expect(flushed).toEqual(1);
+
+               // let the client finish processing the keys
+               return q.delay(10);
+           }).then(() => {
+               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(1);
+           });
+        });
+
    it("Alice exports megolm keys and imports them to a new device", function(done) {
        let messageEncrypted;

        return aliceTestClient.start().then(() => {
-            const p2pSession = createOlmSession(
-                testOlmAccount, aliceTestClient,
-            );
-
+            // establish an olm session with alice
+            return createOlmSession(testOlmAccount, aliceTestClient);
+        }).then((p2pSession) => {
            const groupSession = new Olm.OutboundGroupSession();
            groupSession.create();

@@ -976,6 +1124,8 @@ describe("megolm", function() {

            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
            return aliceTestClient.httpBackend.flush("/sync", 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
        }).then(function() {
            const room = aliceTestClient.client.getRoom(ROOM_ID);
            const event = room.getLiveTimeline().getEvents()[0];
@@ -1008,6 +1158,8 @@ describe("megolm", function() {

            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
            return aliceTestClient.httpBackend.flush("/sync", 1);
+        }).then(function() {
+            return testUtils.syncPromise(aliceTestClient.client);
        }).then(function() {
            const room = aliceTestClient.client.getRoom(ROOM_ID);
            const event = room.getLiveTimeline().getEvents()[0];
@@ -1,5 +1,6 @@
 "use strict";
 const q = require("q");
+import expect from 'expect';

 /**
 * Construct a mock HTTP backend, heavily inspired by Angular.js.
@@ -35,40 +36,48 @@ HttpBackend.prototype = {
     * Respond to all of the requests (flush the queue).
     * @param {string} path The path to flush (optional) default: all.
     * @param {integer} numToFlush The number of things to flush (optional), default: all.
-     * @return {Promise} resolved when there is nothing left to flush.
+     * @param {integer=} waitTime  The time (in ms) to wait for a request to happen.
+     *    default: 5
+     *
+     * @return {Promise} resolves when there is nothing left to flush, with the
+     *    number of requests flushed
     */
-    flush: function(path, numToFlush) {
+    flush: function(path, numToFlush, waitTime) {
        const defer = q.defer();
        const self = this;
        let flushed = 0;
        let triedWaiting = false;
+        if (waitTime === undefined) {
+            waitTime = 5;
+        }
        console.log(
-            "HTTP backend flushing... (path=%s  numToFlush=%s)", path, numToFlush,
+            "HTTP backend flushing... (path=%s numToFlush=%s waitTime=%s)",
+            path, numToFlush, waitTime,
        );
        const tryFlush = function() {
            // if there's more real requests and more expected requests, flush 'em.
            console.log(
-                "  trying to flush queue => reqs=%s expected=%s [%s]",
-                self.requests.length, self.expectedRequests.length, path,
+                "  trying to flush queue => reqs=[%s] expected=[%s]",
+                self.requests, self.expectedRequests.map((er) => er.path),
            );
            if (self._takeFromQueue(path)) {
                // try again on the next tick.
-                console.log("  flushed. Trying for more. [%s]", path);
                flushed += 1;
                if (numToFlush && flushed === numToFlush) {
-                    console.log("  [%s] Flushed assigned amount: %s", path, numToFlush);
-                    defer.resolve();
+                    console.log("  Flushed assigned amount: %s", numToFlush);
+                    defer.resolve(flushed);
                } else {
+                    console.log("  flushed. Trying for more.");
                    setTimeout(tryFlush, 0);
                }
            } else if (flushed === 0 && !triedWaiting) {
                // we may not have made the request yet, wait a generous amount of
                // time before giving up.
-                setTimeout(tryFlush, 5);
+                setTimeout(tryFlush, waitTime);
                triedWaiting = true;
            } else {
-                console.log("  no more flushes. [%s]", path);
-                defer.resolve();
+                console.log("  no more flushes.");
+                defer.resolve(flushed);
            }
        };

@@ -1,14 +1,37 @@
 "use strict";
+import expect from 'expect';
+import q from 'q';
+
 const sdk = require("..");
 const MatrixEvent = sdk.MatrixEvent;

+/**
+ * Return a promise that is resolved when the client next emits a
+ * SYNCING event.
+ * @param {Object} client The client
+ * @return {Promise} Resolves once the client has emitted a SYNCING event
+ */
+module.exports.syncPromise = function(client) {
+    const def = q.defer();
+    const cb = (state) => {
+        if (state == 'SYNCING') {
+            def.resolve();
+        } else {
+            client.once('sync', cb);
+        }
+    };
+    client.once('sync', cb);
+    return def.promise;
+};
+
 /**
 * Perform common actions before each test case, e.g. printing the test case
 * name to stdout.
- * @param {TestCase} testCase The test case that is about to be run.
+ * @param {Mocha.Context} context  The test context
 */
-module.exports.beforeEach = function(testCase) {
-    const desc = testCase.suite.description + " : " + testCase.description;
+module.exports.beforeEach = function(context) {
+    const desc = context.currentTest.fullTitle();
+
    console.log(desc);
    console.log(new Array(1 + desc.length).join("="));
 };
@@ -20,18 +43,18 @@ module.exports.beforeEach = function(testCase) {
 * @return {Object} An instantiated object with spied methods/properties.
 */
 module.exports.mock = function(constr, name) {
-    // By Tim Buschtöns
+    // Based on
    // http://eclipsesource.com/blogs/2014/03/27/mocks-in-jasmine-tests/
    const HelperConstr = new Function(); // jshint ignore:line
    HelperConstr.prototype = constr.prototype;
    const result = new HelperConstr();
-    result.jasmineToString = function() {
+    result.toString = function() {
        return "mock" + (name ? " of " + name : "");
    };
    for (const key in constr.prototype) { // eslint-disable-line guard-for-in
        try {
            if (constr.prototype[key] instanceof Function) {
-                result[key] = jasmine.createSpy((name || "mock") + '.' + key);
+                result[key] = expect.createSpy();
            }
        } catch (ex) {
            // Direct access to some non-function fields of DOM prototypes may
@@ -1,7 +1,10 @@
 "use strict";
+import 'source-map-support/register';
 const ContentRepo = require("../../lib/content-repo");
 const testUtils = require("../test-utils");

+import expect from 'expect';
+
 describe("ContentRepo", function() {
    const baseUrl = "https://my.home.server";

@@ -1,13 +1,19 @@

 "use strict";
-const Crypto = require("../../lib/crypto");
+import 'source-map-support/register';
+
 const sdk = require("../..");
+let Crypto;
+if (sdk.CRYPTO_ENABLED) {
+    Crypto = require("../../lib/crypto");
+}
+
+import expect from 'expect';

 describe("Crypto", function() {
    if (!sdk.CRYPTO_ENABLED) {
        return;
    }
-
    it("Crypto exposes the correct olm library version", function() {
        expect(Crypto.getOlmVersion()[0]).toEqual(2);
    });
@@ -1,4 +1,5 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const EventTimeline = sdk.EventTimeline;
 const utils = require("../test-utils");
@@ -8,6 +9,8 @@ function mockRoomStates(timeline) {
    timeline._endState = utils.mock(sdk.RoomState, "endState");
 }

+import expect from 'expect';
+
 describe("EventTimeline", function() {
    const roomId = "!foo:bar";
    const userA = "@alice:bar";
@@ -75,7 +78,7 @@ describe("EventTimeline", function() {

            expect(function() {
                timeline.initialiseState(state);
-            }).not.toThrow();
+            }).toNotThrow();
            timeline.addEvent(event, false);
            expect(function() {
                timeline.initialiseState(state);
@@ -118,7 +121,7 @@ describe("EventTimeline", function() {
            const next = {b: "b"};
            expect(function() {
                timeline.setNeighbouringTimeline(prev, EventTimeline.BACKWARDS);
-            }).not.toThrow();
+            }).toNotThrow();
            expect(timeline.getNeighbouringTimeline(EventTimeline.BACKWARDS))
                .toBe(prev);
            expect(function() {
@@ -127,7 +130,7 @@ describe("EventTimeline", function() {

            expect(function() {
                timeline.setNeighbouringTimeline(next, EventTimeline.FORWARDS);
-            }).not.toThrow();
+            }).toNotThrow();
            expect(timeline.getNeighbouringTimeline(EventTimeline.FORWARDS))
                .toBe(next);
            expect(function() {
@@ -184,14 +187,14 @@ describe("EventTimeline", function() {
                name: "Old Alice",
            };
            timeline.getState(EventTimeline.FORWARDS).getSentinelMember
-                .andCallFake(function(uid) {
+                .andCall(function(uid) {
                    if (uid === userA) {
                        return sentinel;
                    }
                    return null;
                });
            timeline.getState(EventTimeline.BACKWARDS).getSentinelMember
-                .andCallFake(function(uid) {
+                .andCall(function(uid) {
                    if (uid === userA) {
                        return oldSentinel;
                    }
@@ -226,14 +229,14 @@ describe("EventTimeline", function() {
                name: "Old Alice",
            };
            timeline.getState(EventTimeline.FORWARDS).getSentinelMember
-                .andCallFake(function(uid) {
+                .andCall(function(uid) {
                    if (uid === userA) {
                        return sentinel;
                    }
                    return null;
                });
            timeline.getState(EventTimeline.BACKWARDS).getSentinelMember
-                .andCallFake(function(uid) {
+                .andCall(function(uid) {
                    if (uid === userA) {
                        return oldSentinel;
                    }
@@ -278,7 +281,7 @@ describe("EventTimeline", function() {
            expect(events[1].forwardLooking).toBe(true);

            expect(timeline.getState(EventTimeline.BACKWARDS).setStateEvents).
-                not.toHaveBeenCalled();
+                toNotHaveBeenCalled();
        });


@@ -308,7 +311,7 @@ describe("EventTimeline", function() {
            expect(events[1].forwardLooking).toBe(false);

            expect(timeline.getState(EventTimeline.FORWARDS).setStateEvents).
-                not.toHaveBeenCalled();
+                toNotHaveBeenCalled();
        });
    });

@@ -373,4 +376,3 @@ describe("EventTimeline", function() {
           });
    });
 });
-
@@ -1,8 +1,11 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const Filter = sdk.Filter;
 const utils = require("../test-utils");

+import expect from 'expect';
+
 describe("Filter", function() {
    const filterId = "f1lt3ring15g00d4ursoul";
    const userId = "@sir_arthur_david:humming.tiger";
@@ -15,6 +15,7 @@ limitations under the License.
 */
 "use strict";

+import 'source-map-support/register';
 const q = require("q");
 const sdk = require("../..");
 const utils = require("../test-utils");
@@ -22,18 +23,29 @@ const utils = require("../test-utils");
 const InteractiveAuth = sdk.InteractiveAuth;
 const MatrixError = sdk.MatrixError;

+import expect from 'expect';
+
+// Trivial client object to test interactive auth
+// (we do not need TestClient here)
+class FakeClient {
+    generateClientSecret() {
+        return "testcl1Ent5EcreT";
+    }
+}
+
 describe("InteractiveAuth", function() {
    beforeEach(function() {
        utils.beforeEach(this); // eslint-disable-line no-invalid-this
    });

    it("should start an auth stage and complete it", function(done) {
-        const doRequest = jasmine.createSpy('doRequest');
-        const startAuthStage = jasmine.createSpy('startAuthStage');
+        const doRequest = expect.createSpy();
+        const stateUpdated = expect.createSpy();

        const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
            doRequest: doRequest,
-            startAuthStage: startAuthStage,
+            stateUpdated: stateUpdated,
            authData: {
                session: "sessionId",
                flows: [
@@ -51,7 +63,8 @@ describe("InteractiveAuth", function() {
        });

        // first we expect a call here
-        startAuthStage.andCallFake(function(stage) {
+        stateUpdated.andCall(function(stage) {
+            console.log('aaaa');
            expect(stage).toEqual("logintype");
            ia.submitAuthDict({
                type: "logintype",
@@ -61,7 +74,8 @@ describe("InteractiveAuth", function() {

        // .. which should trigger a call here
        const requestRes = {"a": "b"};
-        doRequest.andCallFake(function(authData) {
+        doRequest.andCall(function(authData) {
+            console.log('cccc');
            expect(authData).toEqual({
                session: "sessionId",
                type: "logintype",
@@ -73,26 +87,27 @@ describe("InteractiveAuth", function() {
        ia.attemptAuth().then(function(res) {
            expect(res).toBe(requestRes);
            expect(doRequest.calls.length).toEqual(1);
-            expect(startAuthStage.calls.length).toEqual(1);
+            expect(stateUpdated.calls.length).toEqual(1);
        }).catch(utils.failTest).done(done);
    });

    it("should make a request if no authdata is provided", function(done) {
-        const doRequest = jasmine.createSpy('doRequest');
-        const startAuthStage = jasmine.createSpy('startAuthStage');
+        const doRequest = expect.createSpy();
+        const stateUpdated = expect.createSpy();

        const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
+            stateUpdated: stateUpdated,
            doRequest: doRequest,
-            startAuthStage: startAuthStage,
        });

        expect(ia.getSessionId()).toBe(undefined);
        expect(ia.getStageParams("logintype")).toBe(undefined);

        // first we expect a call to doRequest
-        doRequest.andCallFake(function(authData) {
+        doRequest.andCall(function(authData) {
            console.log("request1", authData);
-            expect(authData).toBe(null);
+            expect(authData).toEqual({});
            const err = new MatrixError({
                session: "sessionId",
                flows: [
@@ -106,9 +121,9 @@ describe("InteractiveAuth", function() {
            throw err;
        });

-        // .. which should be followed by a call to startAuthStage
+        // .. which should be followed by a call to stateUpdated
        const requestRes = {"a": "b"};
-        startAuthStage.andCallFake(function(stage) {
+        stateUpdated.andCall(function(stage) {
            expect(stage).toEqual("logintype");
            expect(ia.getSessionId()).toEqual("sessionId");
            expect(ia.getStageParams("logintype")).toEqual({
@@ -116,7 +131,7 @@ describe("InteractiveAuth", function() {
            });

            // submitAuthDict should trigger another call to doRequest
-            doRequest.andCallFake(function(authData) {
+            doRequest.andCall(function(authData) {
                console.log("request2", authData);
                expect(authData).toEqual({
                    session: "sessionId",
@@ -135,7 +150,7 @@ describe("InteractiveAuth", function() {
        ia.attemptAuth().then(function(res) {
            expect(res).toBe(requestRes);
            expect(doRequest.calls.length).toEqual(2);
-            expect(startAuthStage.calls.length).toEqual(1);
+            expect(stateUpdated.calls.length).toEqual(1);
        }).catch(utils.failTest).done(done);
    });
 });
@@ -1,9 +1,13 @@
 "use strict";
+import 'source-map-support/register';
 const q = require("q");
 const sdk = require("../..");
 const MatrixClient = sdk.MatrixClient;
 const utils = require("../test-utils");

+import expect from 'expect';
+import lolex from 'lolex';
+
 describe("MatrixClient", function() {
    const userId = "@alice:bar";
    const identityServerUrl = "https://identity.server";
@@ -11,6 +15,7 @@ describe("MatrixClient", function() {
    let client;
    let store;
    let scheduler;
+    let clock;

    const KEEP_ALIVE_PATH = "/_matrix/client/versions";

@@ -120,16 +125,19 @@ describe("MatrixClient", function() {

    beforeEach(function() {
        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        jasmine.Clock.useMock();
-        scheduler = jasmine.createSpyObj("scheduler", [
+        clock = lolex.install();
+        scheduler = [
            "getQueueForEvent", "queueEvent", "removeEventFromQueue",
            "setProcessFunction",
-        ]);
-        store = jasmine.createSpyObj("store", [
+        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
+        store = [
            "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
-            "setSyncToken", "storeEvents", "storeRoom", "storeUser",
+            "save", "setSyncToken", "storeEvents", "storeRoom", "storeUser",
            "getFilterIdByName", "setFilterIdByName", "getFilter", "storeFilter",
-        ]);
+            "getSyncAccumulator", "startup", "deleteAllData",
+        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
+        store.getSavedSync = expect.createSpy().andReturn(q(null));
+        store.setSyncData = expect.createSpy().andReturn(q(null));
        client = new MatrixClient({
            baseUrl: "https://my.home.server",
            idBaseUrl: identityServerUrl,
@@ -140,14 +148,14 @@ describe("MatrixClient", function() {
            userId: userId,
        });
        // FIXME: We shouldn't be yanking _http like this.
-        client._http = jasmine.createSpyObj("httpApi", [
+        client._http = [
            "authedRequest", "authedRequestWithPrefix", "getContentUri",
            "request", "requestWithPrefix", "uploadContent",
-        ]);
-        client._http.authedRequest.andCallFake(httpReq);
-        client._http.authedRequestWithPrefix.andCallFake(httpReq);
-        client._http.requestWithPrefix.andCallFake(httpReq);
-        client._http.request.andCallFake(httpReq);
+        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
+        client._http.authedRequest.andCall(httpReq);
+        client._http.authedRequestWithPrefix.andCall(httpReq);
+        client._http.requestWithPrefix.andCall(httpReq);
+        client._http.request.andCall(httpReq);

        // set reasonable working defaults
        acceptKeepalives = true;
@@ -159,15 +167,16 @@ describe("MatrixClient", function() {
    });

    afterEach(function() {
+        clock.uninstall();
        // need to re-stub the requests with NOPs because there are no guarantees
        // clients from previous tests will be GC'd before the next test. This
        // means they may call /events and then fail an expect() which will fail
        // a DIFFERENT test (pollution between tests!) - we return unresolved
        // promises to stop the client from continuing to run.
-        client._http.authedRequest.andCallFake(function() {
+        client._http.authedRequest.andCall(function() {
            return q.defer().promise;
        });
-        client._http.authedRequestWithPrefix.andCallFake(function() {
+        client._http.authedRequestWithPrefix.andCall(function() {
            return q.defer().promise;
        });
    });
@@ -194,7 +203,7 @@ describe("MatrixClient", function() {

    describe("getSyncState", function() {
        it("should return null if the client isn't started", function() {
-            expect(client.getSyncState()).toBeNull();
+            expect(client.getSyncState()).toBe(null);
        });

        it("should return the same sync state as emitted sync events", function(done) {
@@ -266,7 +275,7 @@ describe("MatrixClient", function() {
                if (state === "ERROR" && httpLookups.length > 0) {
                    expect(httpLookups.length).toEqual(2);
                    expect(client.retryImmediately()).toBe(true);
-                    jasmine.Clock.tick(1);
+                    clock.tick(1);
                } else if (state === "PREPARED" && httpLookups.length === 0) {
                    client.removeListener("sync", syncListener);
                    done();
@@ -292,9 +301,9 @@ describe("MatrixClient", function() {
                    expect(client.retryImmediately()).toBe(
                        true, "retryImmediately returned false",
                    );
-                    jasmine.Clock.tick(1);
+                    clock.tick(1);
                } else if (state === "RECONNECTING" && httpLookups.length > 0) {
-                    jasmine.Clock.tick(10000);
+                    clock.tick(10000);
                } else if (state === "SYNCING" && httpLookups.length === 0) {
                    client.removeListener("sync", syncListener);
                    done();
@@ -316,7 +325,7 @@ describe("MatrixClient", function() {
                if (state === "ERROR" && httpLookups.length > 0) {
                    expect(httpLookups.length).toEqual(3);
                    expect(client.retryImmediately()).toBe(true);
-                    jasmine.Clock.tick(1);
+                    clock.tick(1);
                } else if (state === "PREPARED" && httpLookups.length === 0) {
                    client.removeListener("sync", syncListener);
                    done();
@@ -347,7 +356,7 @@ describe("MatrixClient", function() {
                    done();
                }
                // standard retry time is 5 to 10 seconds
-                jasmine.Clock.tick(10000);
+                clock.tick(10000);
            };
        }

@@ -1,7 +1,10 @@
 "use strict";
+import 'source-map-support/register';
 const PushProcessor = require("../../lib/pushprocessor");
 const utils = require("../test-utils");

+import expect from 'expect';
+
 describe('NotificationService', function() {
    const testUserId = "@ali:matrix.org";
    const testDisplayName = "Alice M";
@@ -1,74 +1,75 @@
 "use strict";

+import 'source-map-support/register';
 const callbacks = require("../../lib/realtime-callbacks");
 const testUtils = require("../test-utils.js");

+import expect from 'expect';
+import lolex from 'lolex';
+
 describe("realtime-callbacks", function() {
-    const clock = jasmine.Clock;
-    let fakeDate;
+    let clock;

    function tick(millis) {
-        // make sure we tick the fakedate first, otherwise nothing will happen!
-        fakeDate += millis;
        clock.tick(millis);
    }

    beforeEach(function() {
        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
-        clock.useMock();
-        fakeDate = Date.now();
-        callbacks.setNow(function() {
-            return fakeDate;
-        });
+        clock = lolex.install();
+        const fakeDate = clock.Date;
+        callbacks.setNow(fakeDate.now.bind(fakeDate));
    });

    afterEach(function() {
        callbacks.setNow();
+        clock.uninstall();
    });

    describe("setTimeout", function() {
        it("should call the callback after the timeout", function() {
-            const callback = jasmine.createSpy();
+            const callback = expect.createSpy();
            callbacks.setTimeout(callback, 100);

-            expect(callback).not.toHaveBeenCalled();
+            expect(callback).toNotHaveBeenCalled();
            tick(100);
            expect(callback).toHaveBeenCalled();
        });


        it("should default to a zero timeout", function() {
-            const callback = jasmine.createSpy();
+            const callback = expect.createSpy();
            callbacks.setTimeout(callback);

-            expect(callback).not.toHaveBeenCalled();
+            expect(callback).toNotHaveBeenCalled();
            tick(0);
            expect(callback).toHaveBeenCalled();
        });

        it("should pass any parameters to the callback", function() {
-            const callback = jasmine.createSpy();
+            const callback = expect.createSpy();
            callbacks.setTimeout(callback, 0, "a", "b", "c");
            tick(0);
            expect(callback).toHaveBeenCalledWith("a", "b", "c");
        });

        it("should set 'this' to the global object", function() {
-            const callback = jasmine.createSpy();
-            callback.andCallFake(function() {
+            let passed = false;
+            const callback = function() {
                expect(this).toBe(global); // eslint-disable-line no-invalid-this
-                expect(this.console).toBeDefined(); // eslint-disable-line no-invalid-this
-            });
+                expect(this.console).toBeTruthy(); // eslint-disable-line no-invalid-this
+                passed = true;
+            };
            callbacks.setTimeout(callback);
            tick(0);
-            expect(callback).toHaveBeenCalled();
+            expect(passed).toBe(true);
        });

        it("should handle timeouts of several seconds", function() {
-            const callback = jasmine.createSpy();
+            const callback = expect.createSpy();
            callbacks.setTimeout(callback, 2000);

-            expect(callback).not.toHaveBeenCalled();
+            expect(callback).toNotHaveBeenCalled();
            for (let i = 0; i < 4; i++) {
                tick(500);
            }
@@ -76,24 +77,24 @@ describe("realtime-callbacks", function() {
        });

        it("should call multiple callbacks in the right order", function() {
-            const callback1 = jasmine.createSpy("callback1");
-            const callback2 = jasmine.createSpy("callback2");
-            const callback3 = jasmine.createSpy("callback3");
+            const callback1 = expect.createSpy();
+            const callback2 = expect.createSpy();
+            const callback3 = expect.createSpy();
            callbacks.setTimeout(callback2, 200);
            callbacks.setTimeout(callback1, 100);
            callbacks.setTimeout(callback3, 300);

-            expect(callback1).not.toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
-            expect(callback3).not.toHaveBeenCalled();
+            expect(callback1).toNotHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
+            expect(callback3).toNotHaveBeenCalled();
            tick(100);
            expect(callback1).toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
-            expect(callback3).not.toHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
+            expect(callback3).toNotHaveBeenCalled();
            tick(100);
            expect(callback1).toHaveBeenCalled();
            expect(callback2).toHaveBeenCalled();
-            expect(callback3).not.toHaveBeenCalled();
+            expect(callback3).toNotHaveBeenCalled();
            tick(100);
            expect(callback1).toHaveBeenCalled();
            expect(callback2).toHaveBeenCalled();
@@ -101,51 +102,54 @@ describe("realtime-callbacks", function() {
        });

        it("should treat -ve timeouts the same as a zero timeout", function() {
-            const callback1 = jasmine.createSpy("callback1");
-            const callback2 = jasmine.createSpy("callback2");
+            const callback1 = expect.createSpy();
+            const callback2 = expect.createSpy();

            // check that cb1 is called before cb2
-            callback1.andCallFake(function() {
-                expect(callback2).not.toHaveBeenCalled();
+            callback1.andCall(function() {
+                expect(callback2).toNotHaveBeenCalled();
            });

            callbacks.setTimeout(callback1);
            callbacks.setTimeout(callback2, -100);

-            expect(callback1).not.toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback1).toNotHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
            tick(0);
            expect(callback1).toHaveBeenCalled();
            expect(callback2).toHaveBeenCalled();
        });

        it("should not get confused by chained calls", function() {
-            const callback2 = jasmine.createSpy("callback2");
-            const callback1 = jasmine.createSpy("callback1");
-            callback1.andCallFake(function() {
+            const callback2 = expect.createSpy();
+            const callback1 = expect.createSpy();
+            callback1.andCall(function() {
                callbacks.setTimeout(callback2, 0);
-                expect(callback2).not.toHaveBeenCalled();
+                expect(callback2).toNotHaveBeenCalled();
            });

            callbacks.setTimeout(callback1);
-            expect(callback1).not.toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback1).toNotHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
            tick(0);
            expect(callback1).toHaveBeenCalled();
+            // the fake timer won't actually run callbacks registered during
+            // one tick until the next tick.
+            tick(1);
            expect(callback2).toHaveBeenCalled();
        });

        it("should be immune to exceptions", function() {
-            const callback1 = jasmine.createSpy("callback1");
-            callback1.andCallFake(function() {
+            const callback1 = expect.createSpy();
+            callback1.andCall(function() {
                throw new Error("prepare to die");
            });
-            const callback2 = jasmine.createSpy("callback2");
+            const callback2 = expect.createSpy();
            callbacks.setTimeout(callback1, 0);
            callbacks.setTimeout(callback2, 0);

-            expect(callback1).not.toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback1).toNotHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
            tick(0);
            expect(callback1).toHaveBeenCalled();
            expect(callback2).toHaveBeenCalled();
@@ -154,16 +158,16 @@ describe("realtime-callbacks", function() {

    describe("cancelTimeout", function() {
        it("should cancel a pending timeout", function() {
-            const callback = jasmine.createSpy();
+            const callback = expect.createSpy();
            const k = callbacks.setTimeout(callback);
            callbacks.clearTimeout(k);
            tick(0);
-            expect(callback).not.toHaveBeenCalled();
+            expect(callback).toNotHaveBeenCalled();
        });

        it("should not affect sooner timeouts", function() {
-            const callback1 = jasmine.createSpy("callback1");
-            const callback2 = jasmine.createSpy("callback2");
+            const callback1 = expect.createSpy();
+            const callback2 = expect.createSpy();

            callbacks.setTimeout(callback1, 100);
            const k = callbacks.setTimeout(callback2, 200);
@@ -171,10 +175,10 @@ describe("realtime-callbacks", function() {

            tick(100);
            expect(callback1).toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();

            tick(150);
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
        });
    });
 });
@@ -1,8 +1,11 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const RoomMember = sdk.RoomMember;
 const utils = require("../test-utils");

+import expect from 'expect';
+
 describe("RoomMember", function() {
    const roomId = "!foo:bar";
    const userA = "@alice:bar";
@@ -33,7 +36,7 @@ describe("RoomMember", function() {
            const url = member.getAvatarUrl(hsUrl);
            // we don't care about how the mxc->http conversion is done, other
            // than it contains the mxc body.
-            expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
+            expect(url.indexOf("flibble/wibble")).toNotEqual(-1);
        });

        it("should return an identicon HTTP URL if allowDefault was set and there " +
@@ -243,9 +246,9 @@ describe("RoomMember", function() {
            member.setMembershipEvent(joinEvent);
            expect(member.name).toEqual("Alice"); // prefer displayname
            member.setMembershipEvent(joinEvent, roomState);
-            expect(member.name).not.toEqual("Alice"); // it should disambig.
+            expect(member.name).toNotEqual("Alice"); // it should disambig.
            // user_id should be there somewhere
-            expect(member.name.indexOf(userA)).not.toEqual(-1);
+            expect(member.name.indexOf(userA)).toNotEqual(-1);
        });

        it("should emit 'RoomMember.membership' if the membership changes", function() {
@@ -1,9 +1,12 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const RoomState = sdk.RoomState;
 const RoomMember = sdk.RoomMember;
 const utils = require("../test-utils");

+import expect from 'expect';
+
 describe("RoomState", function() {
    const roomId = "!foo:bar";
    const userA = "@alice:bar";
@@ -43,8 +46,8 @@ describe("RoomState", function() {
            const members = state.getMembers();
            expect(members.length).toEqual(2);
            // ordering unimportant
-            expect([userA, userB].indexOf(members[0].userId)).not.toEqual(-1);
-            expect([userA, userB].indexOf(members[1].userId)).not.toEqual(-1);
+            expect([userA, userB].indexOf(members[0].userId)).toNotEqual(-1);
+            expect([userA, userB].indexOf(members[1].userId)).toNotEqual(-1);
        });
    });

@@ -54,7 +57,7 @@ describe("RoomState", function() {
        });

        it("should return a member if they exist", function() {
-            expect(state.getMember(userB)).toBeDefined();
+            expect(state.getMember(userB)).toBeTruthy();
        });

        it("should return a member which changes as state changes", function() {
@@ -114,8 +117,8 @@ describe("RoomState", function() {
            const events = state.getStateEvents("m.room.member");
            expect(events.length).toEqual(2);
            // ordering unimportant
-            expect([userA, userB].indexOf(events[0].getStateKey())).not.toEqual(-1);
-            expect([userA, userB].indexOf(events[1].getStateKey())).not.toEqual(-1);
+            expect([userA, userB].indexOf(events[0].getStateKey())).toNotEqual(-1);
+            expect([userA, userB].indexOf(events[1].getStateKey())).toNotEqual(-1);
        });

        it("should return a single MatrixEvent if a state_key was specified",
@@ -238,7 +241,7 @@ describe("RoomState", function() {

            // TODO: We do this because we don't DI the RoomMember constructor
            // so we can't inject a mock :/ so we have to infer.
-            expect(state.members[userC]).toBeDefined();
+            expect(state.members[userC]).toBeTruthy();
            expect(state.members[userC].powerLevel).toEqual(10);
        });

@@ -252,7 +255,7 @@ describe("RoomState", function() {
            });
            state.setStateEvents([memberEvent]);

-            expect(state.members[userA].setMembershipEvent).not.toHaveBeenCalled();
+            expect(state.members[userA].setMembershipEvent).toNotHaveBeenCalled();
            expect(state.members[userB].setMembershipEvent).toHaveBeenCalledWith(
                memberEvent, state,
            );
@@ -1,4 +1,5 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const Room = sdk.Room;
 const RoomState = sdk.RoomState;
@@ -7,6 +8,8 @@ const EventStatus = sdk.EventStatus;
 const EventTimeline = sdk.EventTimeline;
 const utils = require("../test-utils");

+import expect from 'expect';
+
 describe("Room", function() {
    const roomId = "!foo:bar";
    const userA = "@alice:bar";
@@ -29,7 +32,7 @@ describe("Room", function() {
        const hsUrl = "https://my.home.server";

        it("should return the URL from m.room.avatar preferentially", function() {
-            room.currentState.getStateEvents.andCallFake(function(type, key) {
+            room.currentState.getStateEvents.andCall(function(type, key) {
                if (type === "m.room.avatar" && key === "") {
                    return utils.mkEvent({
                        event: true,
@@ -46,7 +49,7 @@ describe("Room", function() {
            const url = room.getAvatarUrl(hsUrl);
            // we don't care about how the mxc->http conversion is done, other
            // than it contains the mxc body.
-            expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
+            expect(url.indexOf("flibble/wibble")).toNotEqual(-1);
        });

        it("should return an identicon HTTP URL if allowDefault was set and there " +
@@ -78,7 +81,7 @@ describe("Room", function() {
        });

        it("should return the member from current state", function() {
-            expect(room.getMember(userA)).not.toEqual(null);
+            expect(room.getMember(userA)).toNotEqual(null);
        });
    });

@@ -170,7 +173,7 @@ describe("Room", function() {
            );
            expect(events[0].forwardLooking).toBe(true);
            expect(events[1].forwardLooking).toBe(true);
-            expect(room.oldState.setStateEvents).not.toHaveBeenCalled();
+            expect(room.oldState.setStateEvents).toNotHaveBeenCalled();
        });

        it("should synthesize read receipts for the senders of events", function() {
@@ -179,7 +182,7 @@ describe("Room", function() {
                membership: "join",
                name: "Alice",
            };
-            room.currentState.getSentinelMember.andCallFake(function(uid) {
+            room.currentState.getSentinelMember.andCall(function(uid) {
                if (uid === userA) {
                    return sentinel;
                }
@@ -288,13 +291,13 @@ describe("Room", function() {
                membership: "join",
                name: "Old Alice",
            };
-            room.currentState.getSentinelMember.andCallFake(function(uid) {
+            room.currentState.getSentinelMember.andCall(function(uid) {
                if (uid === userA) {
                    return sentinel;
                }
                return null;
            });
-            room.oldState.getSentinelMember.andCallFake(function(uid) {
+            room.oldState.getSentinelMember.andCall(function(uid) {
                if (uid === userA) {
                    return oldSentinel;
                }
@@ -327,13 +330,13 @@ describe("Room", function() {
                membership: "join",
                name: "Old Alice",
            };
-            room.currentState.getSentinelMember.andCallFake(function(uid) {
+            room.currentState.getSentinelMember.andCall(function(uid) {
                if (uid === userA) {
                    return sentinel;
                }
                return null;
            });
-            room.oldState.getSentinelMember.andCallFake(function(uid) {
+            room.oldState.getSentinelMember.andCall(function(uid) {
                if (uid === userA) {
                    return oldSentinel;
                }
@@ -375,7 +378,7 @@ describe("Room", function() {
            );
            expect(events[0].forwardLooking).toBe(false);
            expect(events[1].forwardLooking).toBe(false);
-            expect(room.currentState.setStateEvents).not.toHaveBeenCalled();
+            expect(room.currentState.setStateEvents).toNotHaveBeenCalled();
        });
    });

@@ -538,7 +541,7 @@ describe("Room", function() {

    describe("getJoinedMembers", function() {
        it("should return members whose membership is 'join'", function() {
-            room.currentState.getMembers.andCallFake(function() {
+            room.currentState.getMembers.andCall(function() {
                return [
                    { userId: "@alice:bar", membership: "join" },
                    { userId: "@bob:bar", membership: "invite" },
@@ -551,7 +554,7 @@ describe("Room", function() {
        });

        it("should return an empty list if no membership is 'join'", function() {
-            room.currentState.getMembers.andCallFake(function() {
+            room.currentState.getMembers.andCall(function() {
                return [
                    { userId: "@bob:bar", membership: "invite" },
                ];
@@ -646,7 +649,7 @@ describe("Room", function() {

        beforeEach(function() {
            stateLookup = {};
-            room.currentState.getStateEvents.andCallFake(function(type, key) {
+            room.currentState.getStateEvents.andCall(function(type, key) {
                if (key === undefined) {
                    const prefix = type + "$";
                    const list = [];
@@ -663,7 +666,7 @@ describe("Room", function() {
                    return stateLookup[type + "$" + key];
                }
            });
-            room.currentState.getMembers.andCallFake(function() {
+            room.currentState.getMembers.andCall(function() {
                const memberEvents = room.currentState.getStateEvents("m.room.member");
                const members = [];
                for (let i = 0; i < memberEvents.length; i++) {
@@ -678,7 +681,7 @@ describe("Room", function() {
                }
                return members;
            });
-            room.currentState.getMember.andCallFake(function(userId) {
+            room.currentState.getMember.andCall(function(userId) {
                const memberEvent = room.currentState.getStateEvents(
                    "m.room.member", userId,
                );
@@ -712,7 +715,8 @@ describe("Room", function() {
                room.recalculate(userA);
                expect(room.currentState.setStateEvents).toHaveBeenCalled();
                // first call, first arg (which is an array), first element in array
-                const fakeEvent = room.currentState.setStateEvents.calls[0].args[0][0];
+                const fakeEvent = room.currentState.setStateEvents.calls[0].
+                      arguments[0][0];
                expect(fakeEvent.getContent()).toEqual({
                    name: roomName,
                });
@@ -731,7 +735,7 @@ describe("Room", function() {
                ];

                room.recalculate(userA);
-                expect(room.currentState.setStateEvents).not.toHaveBeenCalled();
+                expect(room.currentState.setStateEvents).toNotHaveBeenCalled();
            });
        });

@@ -767,8 +771,8 @@ describe("Room", function() {
                addMember(userC);
                room.recalculate(userA);
                const name = room.name;
-                expect(name.indexOf(userB)).not.toEqual(-1, name);
-                expect(name.indexOf(userC)).not.toEqual(-1, name);
+                expect(name.indexOf(userB)).toNotEqual(-1, name);
+                expect(name.indexOf(userC)).toNotEqual(-1, name);
            });

            it("should return the names of members in a public (public join_rules)" +
@@ -780,8 +784,8 @@ describe("Room", function() {
                addMember(userC);
                room.recalculate(userA);
                const name = room.name;
-                expect(name.indexOf(userB)).not.toEqual(-1, name);
-                expect(name.indexOf(userC)).not.toEqual(-1, name);
+                expect(name.indexOf(userB)).toNotEqual(-1, name);
+                expect(name.indexOf(userC)).toNotEqual(-1, name);
            });

            it("should show the other user's name for public (public join_rules)" +
@@ -792,7 +796,7 @@ describe("Room", function() {
                addMember(userB);
                room.recalculate(userA);
                const name = room.name;
-                expect(name.indexOf(userB)).not.toEqual(-1, name);
+                expect(name.indexOf(userB)).toNotEqual(-1, name);
            });

            it("should show the other user's name for private " +
@@ -803,7 +807,7 @@ describe("Room", function() {
                addMember(userB);
                room.recalculate(userA);
                const name = room.name;
-                expect(name.indexOf(userB)).not.toEqual(-1, name);
+                expect(name.indexOf(userB)).toNotEqual(-1, name);
            });

            it("should show the other user's name for private" +
@@ -813,7 +817,7 @@ describe("Room", function() {
                addMember(userB);
                room.recalculate(userA);
                const name = room.name;
-                expect(name.indexOf(userB)).not.toEqual(-1, name);
+                expect(name.indexOf(userB)).toNotEqual(-1, name);
            });

            it("should show the room alias if one exists for private " +
@@ -959,7 +963,7 @@ describe("Room", function() {

            it("should emit an event when a receipt is added",
            function() {
-                const listener = jasmine.createSpy('spy');
+                const listener = expect.createSpy();
                room.on("Room.receipt", listener);

                const ts = 13787898424;
@@ -1130,7 +1134,7 @@ describe("Room", function() {
            it("should emit Room.tags event when new tags are " +
               "received on the event stream",
            function() {
-                const listener = jasmine.createSpy('spy');
+                const listener = expect.createSpy();
                room.on("Room.tags", listener);

                const tags = { "m.foo": { "order": 0.5 } };
@@ -1,13 +1,18 @@
 // This file had a function whose name is all caps, which displeases eslint
 /* eslint new-cap: "off" */

+import 'source-map-support/register';
 const q = require("q");
 const sdk = require("../..");
 const MatrixScheduler = sdk.MatrixScheduler;
 const MatrixError = sdk.MatrixError;
 const utils = require("../test-utils");

+import expect from 'expect';
+import lolex from 'lolex';
+
 describe("MatrixScheduler", function() {
+    let clock;
    let scheduler;
    let retryFn;
    let queueFn;
@@ -22,7 +27,7 @@ describe("MatrixScheduler", function() {

    beforeEach(function() {
        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        jasmine.Clock.useMock();
+        clock = lolex.install();
        scheduler = new MatrixScheduler(function(ev, attempts, err) {
            if (retryFn) {
                return retryFn(ev, attempts, err);
@@ -39,6 +44,10 @@ describe("MatrixScheduler", function() {
        defer = q.defer();
    });

+    afterEach(function() {
+        clock.uninstall();
+    });
+
    it("should process events in a queue in a FIFO manner", function(done) {
        retryFn = function() {
            return 0;
@@ -98,7 +107,7 @@ describe("MatrixScheduler", function() {
        defer.reject({});
        retryDefer.promise.done(function() {
            expect(procCount).toEqual(1);
-            jasmine.Clock.tick(waitTimeMs);
+            clock.tick(waitTimeMs);
            expect(procCount).toEqual(2);
            done();
        });
@@ -186,7 +195,7 @@ describe("MatrixScheduler", function() {
        setTimeout(function() {
            deferA.resolve({});
        }, 1000);
-        jasmine.Clock.tick(1000);
+        clock.tick(1000);
    });

    describe("queueEvent", function() {
@@ -202,8 +211,8 @@ describe("MatrixScheduler", function() {
                return "yep";
            };
            const prom = scheduler.queueEvent(eventA);
-            expect(prom).toBeDefined();
-            expect(prom.then).toBeDefined();
+            expect(prom).toBeTruthy();
+            expect(prom.then).toBeTruthy();
        });
    });

@@ -212,14 +221,14 @@ describe("MatrixScheduler", function() {
            queueFn = function() {
                return null;
            };
-            expect(scheduler.getQueueForEvent(eventA)).toBeNull();
+            expect(scheduler.getQueueForEvent(eventA)).toBe(null);
        });

        it("should return null if the mapped queue doesn't exist", function() {
            queueFn = function() {
                return "yep";
            };
-            expect(scheduler.getQueueForEvent(eventA)).toBeNull();
+            expect(scheduler.getQueueForEvent(eventA)).toBe(null);
        });

        it("should return a list of events in the queue and modifications to" +
@@ -0,0 +1,356 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+"use strict";
+import 'source-map-support/register';
+import utils from "../test-utils";
+import sdk from "../..";
+import expect from 'expect';
+
+const SyncAccumulator = sdk.SyncAccumulator;
+
+describe("SyncAccumulator", function() {
+    let sa;
+
+    beforeEach(function() {
+        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        sa = new SyncAccumulator({
+            maxTimelineEntries: 10,
+        });
+    });
+
+    it("should return the same /sync response if accumulated exactly once", () => {
+        // technically cheating since we also cheekily pre-populate keys we
+        // know that the sync accumulator will pre-populate.
+        // It isn't 100% transitive.
+        const res = {
+            next_batch: "abc",
+            rooms: {
+                invite: {},
+                leave: {},
+                join: {
+                    "!foo:bar": {
+                        account_data: { events: [] },
+                        ephemeral: { events: [] },
+                        unread_notifications: {},
+                        state: {
+                            events: [
+                                member("alice", "join"),
+                                member("bob", "join"),
+                            ],
+                        },
+                        timeline: {
+                            events: [msg("alice", "hi")],
+                            prev_batch: "something",
+                        },
+                    },
+                },
+            },
+        };
+        sa.accumulate(res);
+        const output = sa.getJSON();
+        expect(output.nextBatch).toEqual(res.next_batch);
+        expect(output.roomsData).toEqual(res.rooms);
+    });
+
+    it("should prune the timeline to the oldest prev_batch within the limit", () => {
+        // maxTimelineEntries is 10 so we should get back all
+        // 10 timeline messages with a prev_batch of "pinned_to_1"
+        sa.accumulate(syncSkeleton({
+            state: { events: [member("alice", "join")] },
+            timeline: {
+                events: [
+                    msg("alice", "1"),
+                    msg("alice", "2"),
+                    msg("alice", "3"),
+                    msg("alice", "4"),
+                    msg("alice", "5"),
+                    msg("alice", "6"),
+                    msg("alice", "7"),
+                ],
+                prev_batch: "pinned_to_1",
+            },
+        }));
+        sa.accumulate(syncSkeleton({
+            state: { events: [] },
+            timeline: {
+                events: [
+                    msg("alice", "8"),
+                ],
+                prev_batch: "pinned_to_8",
+            },
+        }));
+        sa.accumulate(syncSkeleton({
+            state: { events: [] },
+            timeline: {
+                events: [
+                    msg("alice", "9"),
+                    msg("alice", "10"),
+                ],
+                prev_batch: "pinned_to_10",
+            },
+        }));
+
+        let output = sa.getJSON().roomsData.join["!foo:bar"];
+
+        expect(output.timeline.events.length).toEqual(10);
+        output.timeline.events.forEach((e, i) => {
+            expect(e.content.body).toEqual(""+(i+1));
+        });
+        expect(output.timeline.prev_batch).toEqual("pinned_to_1");
+
+        // accumulate more messages. Now it can't have a prev_batch of "pinned to 1"
+        // AND give us <= 10 messages without losing messages in-between.
+        // It should try to find the oldest prev_batch which still fits into 10
+        // messages, which is "pinned to 8".
+        sa.accumulate(syncSkeleton({
+            state: { events: [] },
+            timeline: {
+                events: [
+                    msg("alice", "11"),
+                    msg("alice", "12"),
+                    msg("alice", "13"),
+                    msg("alice", "14"),
+                    msg("alice", "15"),
+                    msg("alice", "16"),
+                    msg("alice", "17"),
+                ],
+                prev_batch: "pinned_to_11",
+            },
+        }));
+
+        output = sa.getJSON().roomsData.join["!foo:bar"];
+
+        expect(output.timeline.events.length).toEqual(10);
+        output.timeline.events.forEach((e, i) => {
+            expect(e.content.body).toEqual(""+(i+8));
+        });
+        expect(output.timeline.prev_batch).toEqual("pinned_to_8");
+    });
+
+    it("should remove the stored timeline on limited syncs", () => {
+        sa.accumulate(syncSkeleton({
+            state: { events: [member("alice", "join")] },
+            timeline: {
+                events: [
+                    msg("alice", "1"),
+                    msg("alice", "2"),
+                    msg("alice", "3"),
+                ],
+                prev_batch: "pinned_to_1",
+            },
+        }));
+        // some time passes and now we get a limited sync
+        sa.accumulate(syncSkeleton({
+            state: { events: [] },
+            timeline: {
+                limited: true,
+                events: [
+                    msg("alice", "51"),
+                    msg("alice", "52"),
+                    msg("alice", "53"),
+                ],
+                prev_batch: "pinned_to_51",
+            },
+        }));
+
+        const output = sa.getJSON().roomsData.join["!foo:bar"];
+
+        expect(output.timeline.events.length).toEqual(3);
+        output.timeline.events.forEach((e, i) => {
+            expect(e.content.body).toEqual(""+(i+51));
+        });
+        expect(output.timeline.prev_batch).toEqual("pinned_to_51");
+    });
+
+    it("should drop typing notifications", () => {
+        const res = syncSkeleton({
+            ephemeral: {
+                events: [{
+                    type: "m.typing",
+                    content: {
+                        user_ids: ["@alice:localhost"],
+                    },
+                    room_id: "!foo:bar",
+                }],
+            },
+        });
+        sa.accumulate(res);
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].ephemeral.events.length,
+        ).toEqual(0);
+    });
+
+    it("should clobber account data based on event type", () => {
+        const acc1 = {
+            type: "favourite.food",
+            content: {
+                food: "banana",
+            },
+        };
+        const acc2 = {
+            type: "favourite.food",
+            content: {
+                food: "apple",
+            },
+        };
+        sa.accumulate(syncSkeleton({
+            account_data: {
+                events: [acc1],
+            },
+        }));
+        sa.accumulate(syncSkeleton({
+            account_data: {
+                events: [acc2],
+            },
+        }));
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].account_data.events.length,
+        ).toEqual(1);
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].account_data.events[0],
+        ).toEqual(acc2);
+    });
+
+    it("should clobber global account data based on event type", () => {
+        const acc1 = {
+            type: "favourite.food",
+            content: {
+                food: "banana",
+            },
+        };
+        const acc2 = {
+            type: "favourite.food",
+            content: {
+                food: "apple",
+            },
+        };
+        sa.accumulate({
+            account_data: {
+                events: [acc1],
+            },
+        });
+        sa.accumulate({
+            account_data: {
+                events: [acc2],
+            },
+        });
+        expect(
+            sa.getJSON().accountData.length,
+        ).toEqual(1);
+        expect(
+            sa.getJSON().accountData[0],
+        ).toEqual(acc2);
+    });
+
+    it("should accumulate read receipts", () => {
+        const receipt1 = {
+            type: "m.receipt",
+            room_id: "!foo:bar",
+            content: {
+                "$event1:localhost": {
+                    "m.read": {
+                        "@alice:localhost": { ts: 1 },
+                        "@bob:localhost": { ts: 2 },
+                    },
+                    "some.other.receipt.type": {
+                        "@should_be_ignored:localhost": { key: "val" },
+                    },
+                },
+            },
+        };
+        const receipt2 = {
+            type: "m.receipt",
+            room_id: "!foo:bar",
+            content: {
+                "$event2:localhost": {
+                    "m.read": {
+                        "@bob:localhost": { ts: 2 }, // clobbers event1 receipt
+                        "@charlie:localhost": { ts: 3 },
+                    },
+                },
+            },
+        };
+        sa.accumulate(syncSkeleton({
+            ephemeral: {
+                events: [receipt1],
+            },
+        }));
+        sa.accumulate(syncSkeleton({
+            ephemeral: {
+                events: [receipt2],
+            },
+        }));
+
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].ephemeral.events.length,
+        ).toEqual(1);
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].ephemeral.events[0],
+        ).toEqual({
+            type: "m.receipt",
+            room_id: "!foo:bar",
+            content: {
+                "$event1:localhost": {
+                    "m.read": {
+                        "@alice:localhost": { ts: 1 },
+                    },
+                },
+                "$event2:localhost": {
+                    "m.read": {
+                        "@bob:localhost": { ts: 2 },
+                        "@charlie:localhost": { ts: 3 },
+                    },
+                },
+            },
+        });
+    });
+});
+
+function syncSkeleton(joinObj) {
+    joinObj = joinObj || {};
+    return {
+        next_batch: "abc",
+        rooms: {
+            join: {
+                "!foo:bar": joinObj,
+            },
+        },
+    };
+}
+
+function msg(localpart, text) {
+    return {
+        content: {
+            body: text,
+        },
+        origin_server_ts: 123456789,
+        sender: "@" + localpart + ":localhost",
+        type: "m.room.message",
+    };
+}
+
+function member(localpart, membership) {
+    return {
+        content: {
+            membership: membership,
+        },
+        origin_server_ts: 123456789,
+        state_key: "@" + localpart + ":localhost",
+        sender: "@" + localpart + ":localhost",
+        type: "m.room.member",
+    };
+}
@@ -1,4 +1,5 @@
 "use strict";
+import 'source-map-support/register';
 const q = require("q");
 const sdk = require("../..");
 const EventTimeline = sdk.EventTimeline;
@@ -6,6 +7,7 @@ const TimelineWindow = sdk.TimelineWindow;
 const TimelineIndex = require("../../lib/timeline-window").TimelineIndex;

 const utils = require("../test-utils");
+import expect from 'expect';

 const ROOM_ID = "roomId";
 const USER_ID = "userId";
@@ -1,8 +1,11 @@
 "use strict";
+import 'source-map-support/register';
 const sdk = require("../..");
 const User = sdk.User;
 const utils = require("../test-utils");

+import expect from 'expect';
+
 describe("User", function() {
    const userId = "@alice:bar";
    let user;
@@ -1,7 +1,10 @@
 "use strict";
+import 'source-map-support/register';
 const utils = require("../../lib/utils");
 const testUtils = require("../test-utils");

+import expect from 'expect';
+
 describe("utils", function() {
    beforeEach(function() {
        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
@@ -132,7 +135,7 @@ describe("utils", function() {
                utils.checkObjectHasKeys({
                    foo: "bar",
                }, ["foo"]);
-            }).not.toThrow();
+            }).toNotThrow();
        });
    });

@@ -149,7 +152,7 @@ describe("utils", function() {
                utils.checkObjectHasNoAdditionalKeys({
                            foo: "bar",
                        }, ["foo"]);
-            }).not.toThrow();
+            }).toNotThrow();
        });
    });

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -124,7 +125,9 @@ MatrixBaseApis.prototype.makeTxnId = function() {
 * @param {string} password
 * @param {string} sessionId
 * @param {Object} auth
- * @param {boolean} bindEmail
+ * @param {Object} bindThreepids Set key 'email' to true to bind any email
+ *     threepid uses during registration in the ID server. Set 'msisdn' to
+ *     true to bind msisdn.
 * @param {string} guestAccessToken
 * @param {module:client.callback} callback Optional.
 * @return {module:client.Promise} Resolves: TODO
@@ -132,10 +135,17 @@ MatrixBaseApis.prototype.makeTxnId = function() {
 */
 MatrixBaseApis.prototype.register = function(
    username, password,
-    sessionId, auth, bindEmail, guestAccessToken,
+    sessionId, auth, bindThreepids, guestAccessToken,
    callback,
 ) {
-    if (auth === undefined) {
+    // backwards compat
+    if (bindThreepids === true) {
+        bindThreepids = {email: true};
+    } else if (bindThreepids === null || bindThreepids === undefined) {
+        bindThreepids = {};
+    }
+
+    if (auth === undefined || auth === null) {
        auth = {};
    }
    if (sessionId) {
@@ -151,12 +161,24 @@ MatrixBaseApis.prototype.register = function(
    if (password !== undefined && password !== null) {
        params.password = password;
    }
-    if (bindEmail !== undefined && bindEmail !== null) {
-        params.bind_email = bindEmail;
+    if (bindThreepids.email) {
+        params.bind_email = true;
+    }
+    if (bindThreepids.msisdn) {
+        params.bind_msisdn = true;
    }
    if (guestAccessToken !== undefined && guestAccessToken !== null) {
        params.guest_access_token = guestAccessToken;
    }
+    // Temporary parameter added to make the register endpoint advertise
+    // msisdn flows. This exists because there are clients that break
+    // when given stages they don't recognise. This parameter will cease
+    // to be necessary once these old clients are gone.
+    // Only send it if we send any params at all (the password param is
+    // mandatory, so if we send any params, we'll send the password param)
+    if (password !== undefined && password !== null) {
+        params.x_show_msisdn = true;
+    }

    return this.registerRequest(params, undefined, callback);
 };
@@ -1142,6 +1164,35 @@ MatrixBaseApis.prototype.requestEmailToken = function(email, clientSecret,
    );
 };

+/**
+ * Submits an MSISDN token to the identity server
+ *
+ * This is used when submitting the code sent by SMS to a phone number.
+ * The ID server has an equivalent API for email but the js-sdk does
+ * not expose this, since email is normally validated by the user clicking
+ * a link rather than entering a code.
+ *
+ * @param {string} sid The sid given in the response to requestToken
+ * @param {string} clientSecret A secret binary string generated by the client.
+ *                 This must be the same value submitted in the requestToken call.
+ * @param {string} token The token, as enetered by the user.
+ *
+ * @return {module:client.Promise} Resolves: Object, currently with no parameters.
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ * @throws Error if No ID server is set
+ */
+MatrixBaseApis.prototype.submitMsisdnToken = function(sid, clientSecret, token) {
+    const params = {
+        sid: sid,
+        client_secret: clientSecret,
+        token: token,
+    };
+    return this._http.idServerRequest(
+        undefined, "POST", "/validate/msisdn/submitToken",
+        params, httpApi.PREFIX_IDENTITY_V1,
+    );
+};
+
 /**
 * Looks up the public Matrix ID mapping for a given 3rd party
 * identifier from the Identity Server
@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -300,17 +301,15 @@ MatrixClient.prototype.getDeviceEd25519Key = function() {
 };

 /**
- * Upload the device keys to the homeserver and ensure that the
- * homeserver has enough one-time keys.
- * @param {number} maxKeys The maximum number of keys to generate
+ * Upload the device keys to the homeserver.
 * @return {object} A promise that will resolve when the keys are uploaded.
 */
-MatrixClient.prototype.uploadKeys = function(maxKeys) {
+MatrixClient.prototype.uploadKeys = function() {
    if (this._crypto === null) {
        throw new Error("End-to-end encryption disabled");
    }

-    return this._crypto.uploadKeys(maxKeys);
+    return this._crypto.uploadDeviceKeys();
 };

 /**
@@ -550,6 +549,9 @@ function _decryptEvent(client, event) {
    try {
        client._crypto.decryptEvent(event);
    } catch (e) {
+        console.warn(
+            `Error decrypting event (id=${event.getId()}): ${e}`,
+        );
        if (!(e instanceof Crypto.DecryptionError)) {
            throw e;
        }
@@ -1377,13 +1379,23 @@ MatrixClient.prototype.forget = function(roomId, deleteRoom, callback) {
 * @param {string} roomId
 * @param {string} userId
 * @param {module:client.callback} callback Optional.
- * @return {module:client.Promise} Resolves: TODO
+ * @return {module:client.Promise} Resolves: Object (currently empty)
 * @return {module:http-api.MatrixError} Rejects: with an error response.
 */
 MatrixClient.prototype.unban = function(roomId, userId, callback) {
-    // unbanning = set their state to leave
-    return _setMembershipState(
-        this, roomId, userId, "leave", undefined, callback,
+    // unbanning != set their state to leave: this used to be
+    // the case, but was then changed so that leaving was always
+    // a revoking of priviledge, otherwise two people racing to
+    // kick / ban someone could end up banning and then un-banning
+    // them.
+    const path = utils.encodeUri("/rooms/$roomId/unban", {
+        $roomId: roomId,
+    });
+    const data = {
+        user_id: userId,
+    };
+    return this._http.authedRequest(
+        callback, "POST", path, undefined, data,
    );
 };

@@ -2070,14 +2082,45 @@ MatrixClient.prototype.setGuestAccess = function(roomId, opts) {
 * @param {string} clientSecret As requestEmailToken
 * @param {number} sendAttempt As requestEmailToken
 * @param {string} nextLink As requestEmailToken
- * @param {module:client.callback} callback Optional. As requestEmailToken
 * @return {module:client.Promise} Resolves: As requestEmailToken
 */
 MatrixClient.prototype.requestRegisterEmailToken = function(email, clientSecret,
-                                                    sendAttempt, nextLink, callback) {
+                                                    sendAttempt, nextLink) {
    return this._requestTokenFromEndpoint(
        "/register/email/requestToken",
-        email, clientSecret, sendAttempt, nextLink, callback,
+        {
+            email: email,
+            client_secret: clientSecret,
+            send_attempt: sendAttempt,
+            next_link: nextLink,
+        },
+    );
+};
+
+/**
+ * Requests a text message verification token for the purposes of registration.
+ * This API proxies the Identity Server /validate/msisdn/requestToken API,
+ * adding registration-specific behaviour, as with requestRegisterEmailToken.
+ *
+ * @param {string} phoneCountry The ISO 3166-1 alpha-2 code for the country in which
+ *    phoneNumber should be parsed relative to.
+ * @param {string} phoneNumber The phone number, in national or international format
+ * @param {string} clientSecret As requestEmailToken
+ * @param {number} sendAttempt As requestEmailToken
+ * @param {string} nextLink As requestEmailToken
+ * @return {module:client.Promise} Resolves: As requestEmailToken
+ */
+MatrixClient.prototype.requestRegisterMsisdnToken = function(phoneCountry, phoneNumber,
+                                                    clientSecret, sendAttempt, nextLink) {
+    return this._requestTokenFromEndpoint(
+        "/register/msisdn/requestToken",
+        {
+            country: phoneCountry,
+            phone_number: phoneNumber,
+            client_secret: clientSecret,
+            send_attempt: sendAttempt,
+            next_link: nextLink,
+        },
    );
 };

@@ -2098,14 +2141,46 @@ MatrixClient.prototype.requestRegisterEmailToken = function(email, clientSecret,
 * @param {string} clientSecret As requestEmailToken
 * @param {number} sendAttempt As requestEmailToken
 * @param {string} nextLink As requestEmailToken
- * @param {module:client.callback} callback Optional. As requestEmailToken
 * @return {module:client.Promise} Resolves: As requestEmailToken
 */
 MatrixClient.prototype.requestAdd3pidEmailToken = function(email, clientSecret,
-                                                    sendAttempt, nextLink, callback) {
+                                                    sendAttempt, nextLink) {
    return this._requestTokenFromEndpoint(
        "/account/3pid/email/requestToken",
-        email, clientSecret, sendAttempt, nextLink, callback,
+        {
+            email: email,
+            client_secret: clientSecret,
+            send_attempt: sendAttempt,
+            next_link: nextLink,
+        },
+    );
+};
+
+/**
+ * Requests a text message verification token for the purposes of adding a
+ * third party identifier to an account.
+ * This API proxies the Identity Server /validate/email/requestToken API,
+ * adding specific behaviour for the addition of phone numbers to an
+ * account, as requestAdd3pidEmailToken.
+ *
+ * @param {string} phoneCountry As requestRegisterMsisdnToken
+ * @param {string} phoneNumber As requestRegisterMsisdnToken
+ * @param {string} clientSecret As requestEmailToken
+ * @param {number} sendAttempt As requestEmailToken
+ * @param {string} nextLink As requestEmailToken
+ * @return {module:client.Promise} Resolves: As requestEmailToken
+ */
+MatrixClient.prototype.requestAdd3pidMsisdnToken = function(phoneCountry, phoneNumber,
+                                                    clientSecret, sendAttempt, nextLink) {
+    return this._requestTokenFromEndpoint(
+        "/account/3pid/msisdn/requestToken",
+        {
+            country: phoneCountry,
+            phone_number: phoneNumber,
+            client_secret: clientSecret,
+            send_attempt: sendAttempt,
+            next_link: nextLink,
+        },
    );
 };

@@ -2129,10 +2204,42 @@ MatrixClient.prototype.requestAdd3pidEmailToken = function(email, clientSecret,
 * @return {module:client.Promise} Resolves: As requestEmailToken
 */
 MatrixClient.prototype.requestPasswordEmailToken = function(email, clientSecret,
-                                                    sendAttempt, nextLink, callback) {
+                                                    sendAttempt, nextLink) {
    return this._requestTokenFromEndpoint(
        "/account/password/email/requestToken",
-        email, clientSecret, sendAttempt, nextLink, callback,
+        {
+            email: email,
+            client_secret: clientSecret,
+            send_attempt: sendAttempt,
+            next_link: nextLink,
+        },
+    );
+};
+
+/**
+ * Requests a text message verification token for the purposes of resetting
+ * the password on an account.
+ * This API proxies the Identity Server /validate/email/requestToken API,
+ * adding specific behaviour for the password resetting, as requestPasswordEmailToken.
+ *
+ * @param {string} phoneCountry As requestRegisterMsisdnToken
+ * @param {string} phoneNumber As requestRegisterMsisdnToken
+ * @param {string} clientSecret As requestEmailToken
+ * @param {number} sendAttempt As requestEmailToken
+ * @param {string} nextLink As requestEmailToken
+ * @return {module:client.Promise} Resolves: As requestEmailToken
+ */
+MatrixClient.prototype.requestPasswordMsisdnToken = function(phoneCountry, phoneNumber,
+                                                    clientSecret, sendAttempt, nextLink) {
+    return this._requestTokenFromEndpoint(
+        "/account/password/msisdn/requestToken",
+        {
+            country: phoneCountry,
+            phone_number: phoneNumber,
+            client_secret: clientSecret,
+            send_attempt: sendAttempt,
+            next_link: nextLink,
+        },
    );
 };

@@ -2141,31 +2248,21 @@ MatrixClient.prototype.requestPasswordEmailToken = function(email, clientSecret,
 * requestToken endpoints.
 *
 * @param {string} endpoint The endpoint to send the request to
- * @param {string} email As requestEmailToken
- * @param {string} clientSecret As requestEmailToken
- * @param {number} sendAttempt As requestEmailToken
- * @param {string} nextLink As requestEmailToken
- * @param {module:client.callback} callback Optional. As requestEmailToken
+ * @param {object} params Parameters for the POST request
 * @return {module:client.Promise} Resolves: As requestEmailToken
 */
-MatrixClient.prototype._requestTokenFromEndpoint = function(endpoint,
-                                                    email, clientSecret,
-                                                    sendAttempt, nextLink, callback) {
+MatrixClient.prototype._requestTokenFromEndpoint = function(endpoint, params) {
    const id_server_url = url.parse(this.idBaseUrl);
    if (id_server_url.host === null) {
        throw new Error("Invalid ID server URL: " + this.idBaseUrl);
    }

-    const params = {
-        client_secret: clientSecret,
-        email: email,
-        send_attempt: sendAttempt,
-        next_link: nextLink,
+    const postParams = Object.assign({}, params, {
        id_server: id_server_url.host,
-    };
+    });
    return this._http.request(
-        callback, "POST", endpoint, undefined,
-        params,
+        undefined, "POST", endpoint, undefined,
+        postParams,
    );
 };

@@ -2286,13 +2383,18 @@ MatrixClient.prototype.setRoomMutePushRule = function(scope, roomId, mute) {
 * @return {module:http-api.MatrixError} Rejects: with an error response.
 */
 MatrixClient.prototype.searchMessageText = function(opts, callback) {
+    const roomEvents = {
+        search_term: opts.query,
+    };
+
+    if ('keys' in opts) {
+        roomEvents.keys = opts.keys;
+    }
+
    return this.search({
        body: {
            search_categories: {
-                room_events: {
-                    keys: opts.keys,
-                    search_term: opts.query,
-                },
+                room_events: roomEvents,
            },
        },
    }, callback);
@@ -2629,10 +2731,10 @@ MatrixClient.prototype.getTurnServers = function() {


 /**
- * High level helper method to call initialSync, emit the resulting events,
- * and then start polling the eventStream for new events. To listen for these
+ * High level helper method to begin syncing and poll for new events. To listen for these
 * events, add a listener for {@link module:client~MatrixClient#event:"event"}
- * via {@link module:client~MatrixClient#on}.
+ * via {@link module:client~MatrixClient#on}. Alternatively, listen for specific
+ * state change events.
 * @param {Object=} opts Options to apply when syncing.
 * @param {Number=} opts.initialSyncLimit The event <code>limit=</code> to apply
 * to initial sync. Default: 8.
@@ -2649,7 +2751,7 @@ MatrixClient.prototype.getTurnServers = function() {
 * accessbile via {@link module:models/room#getPendingEvents}. Default:
 * "chronological".
 *
- * @param {Number=} opts.pollTimeout The number of milliseconds to wait on /events.
+ * @param {Number=} opts.pollTimeout The number of milliseconds to wait on /sync.
 * Default: 30000 (30 seconds).
 *
 * @param {Filter=} opts.filter The filter to apply to /sync calls. This will override
@@ -2669,12 +2771,7 @@ MatrixClient.prototype.startClient = function(opts) {
    }

    if (this._crypto) {
-        this._crypto.uploadKeys(5).done();
-        const tenMinutes = 1000 * 60 * 10;
-        const self = this;
-        this._uploadIntervalID = global.setInterval(function() {
-            self._crypto.uploadKeys(5).done();
-        }, tenMinutes);
+        this._crypto.uploadDeviceKeys().done();
    }

    // periodically poll for turn servers if we support voip
@@ -2690,6 +2787,12 @@ MatrixClient.prototype.startClient = function(opts) {
    opts = Object.assign({}, opts);

    opts.crypto = this._crypto;
+    opts.canResetEntireTimeline = (roomId) => {
+        if (!this._canResetTimelineCallback) {
+            return false;
+        }
+        return this._canResetTimelineCallback(roomId);
+    };
    this._clientOpts = opts;

    this._syncApi = new SyncApi(this, opts);
@@ -2707,12 +2810,30 @@ MatrixClient.prototype.stopClient = function() {
        this._syncApi.stop();
        this._syncApi = null;
    }
-    if (this._crypto) {
-        global.clearInterval(this._uploadIntervalID);
-    }
    global.clearTimeout(this._checkTurnServersTimeoutID);
 };

+/*
+ * Set a function which is called when /sync returns a 'limited' response.
+ * It is called with a room ID and returns a boolean. It should return 'true' if the SDK
+ * can SAFELY remove events from this room. It may not be safe to remove events if there
+ * are other references to the timelines for this room, e.g because the client is
+ * actively viewing events in this room.
+ * Default: returns false.
+ * @param {Function} cb The callback which will be invoked.
+ */
+MatrixClient.prototype.setCanResetTimelineCallback = function(cb) {
+    this._canResetTimelineCallback = cb;
+};
+
+/**
+ * Get the callback set via `setCanResetTimelineCallback`.
+ * @return {?Function} The callback or null
+ */
+MatrixClient.prototype.getCanResetTimelineCallback = function() {
+    return this._canResetTimelineCallback;
+};
+
 function setupCallEventHandler(client) {
    const candidatesByCall = {
        // callId: [Candidate]
@@ -2817,7 +2938,7 @@ function setupCallEventHandler(client) {
            const existingCalls = utils.values(client.callList);
            for (i = 0; i < existingCalls.length; ++i) {
                const thisCall = existingCalls[i];
-                if (call.room_id === thisCall.room_id &&
+                if (call.roomId === thisCall.roomId &&
                        thisCall.direction === 'outbound' &&
                        (["wait_local_media", "create_offer", "invite_sent"].indexOf(
                            thisCall.state) !== -1)) {
@@ -3020,21 +3141,26 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
 /**
 * Fires whenever the SDK's syncing state is updated. The state can be one of:
 * <ul>
- * <li>PREPARED : The client has synced with the server at least once and is
+ *
+ * <li>PREPARED: The client has synced with the server at least once and is
 * ready for methods to be called on it. This will be immediately followed by
 * a state of SYNCING. <i>This is the equivalent of "syncComplete" in the
 * previous API.</i></li>
+ *
 * <li>SYNCING : The client is currently polling for new events from the server.
 * This will be called <i>after</i> processing latest events from a sync.</li>
+ *
 * <li>ERROR : The client has had a problem syncing with the server. If this is
 * called <i>before</i> PREPARED then there was a problem performing the initial
 * sync. If this is called <i>after</i> PREPARED then there was a problem polling
 * the server for updates. This may be called multiple times even if the state is
 * already ERROR. <i>This is the equivalent of "syncError" in the previous
 * API.</i></li>
- * <li>RECONNECTING: The sync connedtion has dropped, but not in a way that should
- * be considered erroneous.
+ *
+ * <li>RECONNECTING: The sync connection has dropped, but not (yet) in a way that
+ * should be considered erroneous.
 * </li>
+ *
 * <li>STOPPED: The client has stopped syncing with server due to stopClient
 * being called.
 * </li>
@@ -3044,32 +3170,46 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
 *                                          +---->STOPPED
 *                                          |
 *              +----->PREPARED -------> SYNCING <--+
- *              |        ^                  ^       |
- *              |        |                  |       |
- *              |        |                  V       |
- *   null ------+        |  +-RECONNECTING<-+       |
+ *              |        ^                |  ^      |
+ *              |        |                |  |      |
+ *              |        |                V  |      |
+ *   null ------+        |  +--------RECONNECTING   |
 *              |        |  V                       |
 *              +------->ERROR ---------------------+
 *
 * NB: 'null' will never be emitted by this event.
+ *
 * </pre>
 * Transitions:
 * <ul>
+ *
 * <li><code>null -> PREPARED</code> : Occurs when the initial sync is completed
 * first time. This involves setting up filters and obtaining push rules.
+ *
 * <li><code>null -> ERROR</code> : Occurs when the initial sync failed first time.
+ *
 * <li><code>ERROR -> PREPARED</code> : Occurs when the initial sync succeeds
 * after previously failing.
+ *
 * <li><code>PREPARED -> SYNCING</code> : Occurs immediately after transitioning
 * to PREPARED. Starts listening for live updates rather than catching up.
- * <li><code>SYNCING -> ERROR</code> : Occurs the first time a client cannot perform a
- * live update.
+ *
+ * <li><code>SYNCING -> RECONNECTING</code> : Occurs when the live update fails.
+ *
+ * <li><code>RECONNECTING -> RECONNECTING</code> : Can occur if the update calls
+ * continue to fail, but the keepalive calls (to /versions) succeed.
+ *
+ * <li><code>RECONNECTING -> ERROR</code> : Occurs when the keepalive call also fails
+ *
 * <li><code>ERROR -> SYNCING</code> : Occurs when the client has performed a
 * live update after having previously failed.
- * <li><code>ERROR -> ERROR</code> : Occurs when the client has failed to sync
+ *
+ * <li><code>ERROR -> ERROR</code> : Occurs when the client has failed to keepalive
 * for a second time or more.</li>
+ *
 * <li><code>SYNCING -> SYNCING</code> : Occurs when the client has performed a live
 * update. This is called <i>after</i> processing.</li>
+ *
 * <li><code>* -> STOPPED</code> : Occurs once the client has stopped syncing or
 * trying to sync after stopClient has been called.</li>
 * </ul>
@@ -3095,6 +3235,9 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
 *    will become the 'since' token for the next call to /sync. Only present if
 *    <code>state=PREPARED</code> or <code>state=SYNCING</code>.
 *
+ * @param {boolean} data.catchingUp True if we are working our way through a
+ *    backlog of events after connecting. Only present if <code>state=SYNCING</code>.
+ *
 * @example
 * matrixClient.on("sync", function(state, prevState, data) {
 *   switch (state) {
@@ -3170,7 +3313,7 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;

 /**
 * Fires whenever new user-scoped account_data is added.
- * @event module:client~MatrixClient#"Room"
+ * @event module:client~MatrixClient#"accountData"
 * @param {MatrixEvent} event The event describing the account_data just added
 * @example
 * matrixClient.on("accountData", function(event){
@@ -39,9 +39,22 @@ export default class DeviceList {
        // userId -> true
        this._pendingUsersWithNewDevices = {};

-        // userId -> promise
+        // userId -> true
        this._keyDownloadsInProgressByUser = {};

+        // deferred which is resolved when the current device query resolves.
+        // (null if there is no current request).
+        this._currentQueryDeferred = null;
+
+        // deferred which is resolved when the *next* device query resolves.
+        //
+        // Normally it is meaningless for this to be non-null when
+        // _currentQueryDeferred is null, but it can happen if the previous
+        // query has finished but the next one has not yet started (because the
+        // previous query failed, in which case we deliberately delay starting
+        // the next query to avoid tight-looping).
+        this._queuedQueryDeferred = null;
+
        this.lastKnownSyncToken = null;
    }

@@ -55,29 +68,43 @@ export default class DeviceList {
     * module:crypto/deviceinfo|DeviceInfo}.
     */
    downloadKeys(userIds, forceDownload) {
-        // promises we need to wait for while the download happens
-        const promises = [];
-
        let needsRefresh = false;
+        let waitForCurrentQuery = false;
+
        userIds.forEach((u) => {
-            if (this._keyDownloadsInProgressByUser[u]) {
-                // just wait for the existing download to complete
-                promises.push(this._keyDownloadsInProgressByUser[u]);
-            } else {
-                if (forceDownload || !this.getStoredDevicesForUser(u)) {
-                    this.invalidateUserDeviceList(u);
-                }
-                if (this._pendingUsersWithNewDevices[u]) {
-                    needsRefresh = true;
-                }
+            if (this._pendingUsersWithNewDevices[u]) {
+                // we already know this user's devices are outdated
+                needsRefresh = true;
+            } else if (this._keyDownloadsInProgressByUser[u]) {
+                // already a download in progress - just wait for it.
+                // (even if forceDownload is true)
+                waitForCurrentQuery = true;
+            } else if (forceDownload) {
+                console.log("Invalidating device list for " + u +
+                            " for forceDownload");
+                this.invalidateUserDeviceList(u);
+                needsRefresh = true;
+            } else if (!this.getStoredDevicesForUser(u)) {
+                console.log("Invalidating device list for " + u +
+                            " due to empty cache");
+                this.invalidateUserDeviceList(u);
+                needsRefresh = true;
            }
        });

+        let promise;
        if (needsRefresh) {
-            promises.push(this.refreshOutdatedDeviceLists(true));
+            console.log("downloadKeys: waiting for next key query");
+            promise = this._startOrQueueDeviceQuery();
+        } else if(waitForCurrentQuery) {
+            console.log("downloadKeys: waiting for in-flight query to complete");
+            promise = this._currentQueryDeferred.promise;
+        } else {
+            // we're all up-to-date.
+            promise = q();
        }

-        return q.all(promises).then(() => {
+        return promise.then(() => {
            return this._getDevicesFromStore(userIds);
        });
    }
@@ -146,11 +173,11 @@ export default class DeviceList {
     *
     * @param {string} userId     owner of the device
     * @param {string} algorithm  encryption algorithm
-     * @param {string} sender_key curve25519 key to match
+     * @param {string} senderKey  curve25519 key to match
     *
     * @return {module:crypto/deviceinfo?}
     */
-    getDeviceByIdentityKey(userId, algorithm, sender_key) {
+    getDeviceByIdentityKey(userId, algorithm, senderKey) {
        if (
            algorithm !== olmlib.OLM_ALGORITHM &&
            algorithm !== olmlib.MEGOLM_ALGORITHM
@@ -178,7 +205,7 @@ export default class DeviceList {
                    continue;
                }
                const deviceKey = device.keys[keyId];
-                if (deviceKey == sender_key) {
+                if (deviceKey == senderKey) {
                    return DeviceInfo.fromStorage(device, deviceId);
                }
            }
@@ -211,75 +238,104 @@ export default class DeviceList {
    }

    /**
-     * Start device queries for any users with outdated device lists
-     *
-     * We tolerate multiple concurrent device queries, but only one query per
-     * user.
-     *
-     * If any users already have downloads in progress, they are ignored - they
-     * will be refreshed when the current download completes anyway, so
-     * each user with outdated device lists will be updated eventually.
-     *
-     * The returned promise resolves immediately if there are no users with
-     * outdated device lists, or if all users with outdated device lists already
-     * have a query in progress.
-     *
-     * Otherwise, a new query request is made, and the promise resolves
-     * once that query completes. If the query fails, the promise will reject
-     * if rejectOnFailure was truthy, otherwise it will still resolve.
-     *
-     * @param {Boolean?} rejectOnFailure  true to make the returned promise
-     *   reject if the device list query fails.
+     * If there is not already a device list query in progress, and we have
+     * users who have outdated device lists, start a query now.
+     */
+    refreshOutdatedDeviceLists() {
+        if (this._currentQueryDeferred) {
+            // request already in progress - do nothing. (We will automatically
+            // make another request if there are more users with outdated
+            // device lists when the current request completes).
+            return;
+        }
+
+        this._startDeviceQuery();
+    }
+
+    /**
+     * If there is currently a device list query in progress, returns a promise
+     * which will resolve when the *next* query completes. Otherwise, starts
+     * a new query, and returns a promise which resolves when it completes.
     *
     * @return {Promise}
     */
-    refreshOutdatedDeviceLists(rejectOnFailure) {
-        const users = Object.keys(this._pendingUsersWithNewDevices).filter(
-            (u) => !this._keyDownloadsInProgressByUser[u],
-        );
+    _startOrQueueDeviceQuery() {
+        if (!this._currentQueryDeferred) {
+            this._startDeviceQuery();
+            if (!this._currentQueryDeferred) {
+                return q();
+            }

-        if (users.length === 0) {
-            return q();
+            return this._currentQueryDeferred.promise;
        }

-        let prom = this._doKeyDownloadForUsers(users).then(() => {
+        if (!this._queuedQueryDeferred) {
+            this._queuedQueryDeferred = q.defer();
+        }
+
+        return this._queuedQueryDeferred.promise;
+    }
+
+    /**
+     * kick off a new device query
+     *
+     * Throws if there is already a query in progress.
+     */
+    _startDeviceQuery() {
+        if (this._currentQueryDeferred) {
+            throw new Error("DeviceList._startDeviceQuery called with request active");
+        }
+
+        this._currentQueryDeferred = this._queuedQueryDeferred || q.defer();
+        this._queuedQueryDeferred = null;
+
+        const users = Object.keys(this._pendingUsersWithNewDevices);
+        if (users.length === 0) {
+            // nothing to do
+            this._currentQueryDeferred.resolve();
+            this._currentQueryDeferred = null;
+
+            // that means we're up-to-date with the lastKnownSyncToken.
+            const token = this.lastKnownSyncToken;
+            if (token !== null) {
+                this._sessionStore.storeEndToEndDeviceSyncToken(token);
+            }
+
+            return;
+        }
+
+        this._doKeyDownloadForUsers(users).done(() => {
            users.forEach((u) => {
                delete this._keyDownloadsInProgressByUser[u];
            });

+            this._currentQueryDeferred.resolve();
+            this._currentQueryDeferred = null;
+
            // flush out any more requests that were blocked up while that
-            // was going on, but let the initial promise complete now.
-            //
-            this.refreshOutdatedDeviceLists().done();
+            // was going on.
+            this._startDeviceQuery();
        }, (e) => {
            console.error(
                'Error updating device key cache for ' + users + ":", e,
            );

            // reinstate the pending flags on any users which failed; this will
-            // mean that we will do another download in the future, but won't
-            // tight-loop.
-            //
+            // mean that we will do another download in the future (actually on
+            // the next /sync).
            users.forEach((u) => {
                delete this._keyDownloadsInProgressByUser[u];
                this._pendingUsersWithNewDevices[u] = true;
            });

-            // TODO: schedule a retry.
-            throw e;
+            this._currentQueryDeferred.reject(e);
+            this._currentQueryDeferred = null;
        });

        users.forEach((u) => {
            delete this._pendingUsersWithNewDevices[u];
-            this._keyDownloadsInProgressByUser[u] = prom;
+            this._keyDownloadsInProgressByUser[u] = true;
        });
-
-        if (!rejectOnFailure) {
-            // normally we just want to swallow the exception - we've already
-            // logged it futher up.
-            prom = prom.catch((e) => {});
-        }
-        return prom;
    }

    /**
@@ -300,47 +356,63 @@ export default class DeviceList {
        ).then((res) => {
            const dk = res.device_keys || {};

+            // do each user in a separate promise, to avoid wedging the CPU
+            // (https://github.com/vector-im/riot-web/issues/3158)
+            //
+            // of course we ought to do this in a web worker or similar, but
+            // this serves as an easy solution for now.
+            let prom = q();
            for (const userId of downloadUsers) {
-                console.log('got keys for ' + userId + ':', dk[userId]);
-
-                // map from deviceid -> deviceinfo for this user
-                const userStore = {};
-                const devs = this._sessionStore.getEndToEndDevicesForUser(userId);
-                if (devs) {
-                    Object.keys(devs).forEach((deviceId) => {
-                        const d = DeviceInfo.fromStorage(devs[deviceId], deviceId);
-                        userStore[deviceId] = d;
-                    });
-                }
-
-                _updateStoredDeviceKeysForUser(
-                    this._olmDevice, userId, userStore, dk[userId] || {},
-                );
-
-                // update the session store
-                const storage = {};
-                Object.keys(userStore).forEach((deviceId) => {
-                    storage[deviceId] = userStore[deviceId].toStorage();
+                prom = prom.delay(5).then(() => {
+                    this._processQueryResponseForUser(userId, dk[userId]);
                });
-
-                this._sessionStore.storeEndToEndDevicesForUser(
-                    userId, storage,
-                );
-
-                if (token) {
-                    this._sessionStore.storeEndToEndDeviceSyncToken(token);
-                }
            }
+
+            return prom;
+        }).then(() => {
+            if (token !== null) {
+                this._sessionStore.storeEndToEndDeviceSyncToken(token);
+            }
+            console.log('Completed key download for ' + downloadUsers);
        });
    }
+
+    _processQueryResponseForUser(userId, response) {
+        console.log('got keys for ' + userId + ':', response);
+
+        // map from deviceid -> deviceinfo for this user
+        const userStore = {};
+        const devs = this._sessionStore.getEndToEndDevicesForUser(userId);
+        if (devs) {
+            Object.keys(devs).forEach((deviceId) => {
+                const d = DeviceInfo.fromStorage(devs[deviceId], deviceId);
+                userStore[deviceId] = d;
+            });
+        }
+
+        _updateStoredDeviceKeysForUser(
+            this._olmDevice, userId, userStore, response || {},
+        );
+
+        // update the session store
+        const storage = {};
+        Object.keys(userStore).forEach((deviceId) => {
+            storage[deviceId] = userStore[deviceId].toStorage();
+        });
+
+        this._sessionStore.storeEndToEndDevicesForUser(
+            userId, storage,
+        );
+    }
 }

+
 function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
        userResult) {
    let updated = false;

    // remove any devices in the store which aren't in the response
-    for (var deviceId in userStore) {
+    for (const deviceId in userStore) {
        if (!userStore.hasOwnProperty(deviceId)) {
            continue;
        }
@@ -353,7 +425,7 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
        }
    }

-    for (deviceId in userResult) {
+    for (const deviceId in userResult) {
        if (!userResult.hasOwnProperty(deviceId)) {
            continue;
        }
@@ -398,7 +470,7 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
    const signKeyId = "ed25519:" + deviceId;
    const signKey = deviceResult.keys[signKeyId];
    if (!signKey) {
-        console.log("Device " + userId + ":" + deviceId +
+        console.warn("Device " + userId + ":" + deviceId +
            " has no ed25519 key");
        return false;
    }
@@ -408,8 +480,8 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
    try {
        olmlib.verifySignature(_olmDevice, deviceResult, userId, deviceId, signKey);
    } catch (e) {
-        console.log("Unable to verify signature on device " +
-            userId + ":" + deviceId + ":", e);
+        console.warn("Unable to verify signature on device " +
+            userId + ":" + deviceId + ":" + e);
        return false;
    }

@@ -425,7 +497,7 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
            // best off sticking with the original keys.
            //
            // Should we warn the user about it somehow?
-            console.warn("Ed25519 key for device" + userId + ": " +
+            console.warn("Ed25519 key for device " + userId + ":" +
               deviceId + " has changed");
            return false;
        }
@@ -148,14 +148,42 @@ DecryptionAlgorithm.prototype.importRoomKey = function(session) {
 /**
 * Exception thrown when decryption fails
 *
+ * @alias module:crypto/algorithms/base.DecryptionError
 * @constructor
- * @param {string} msg message describing the problem
+ * @param {string} msg user-visible message describing the problem
+ *
+ * @param {Object=} details key/value pairs reported in the logs but not shown
+ *   to the user.
+ *
 * @extends Error
 */
-module.exports.DecryptionError = function(msg) {
+const DecryptionError = function(msg, details) {
+    this.name = 'DecryptionError';
    this.message = msg;
+    this.details = details;
 };
-utils.inherits(module.exports.DecryptionError, Error);
+utils.inherits(DecryptionError, Error);
+
+/** override the string used when logging
+ *
+ * @returns {String}
+ */
+DecryptionError.prototype.toString = function() {
+    let result = this.name + '[msg: ' + this.message;
+
+    if (this.details) {
+        result += ', ' +
+            Object.keys(this.details).map(
+                (k) => k + ': ' + this.details[k],
+            ).join(', ');
+    }
+
+    result += ']';
+
+    return result;
+};
+
+module.exports.DecryptionError = DecryptionError;

 /**
 * Exception thrown specifically when we want to warn the user to consider
@@ -544,7 +544,11 @@ MegolmDecryption.prototype.decryptEvent = function(event) {
        if (e.message === 'OLM.UNKNOWN_MESSAGE_INDEX') {
            this._addEventToPendingList(event);
        }
-        throw new base.DecryptionError(e);
+        throw new base.DecryptionError(
+            e.toString(), {
+                session: content.sender_key + '|' + content.session_id,
+            },
+        );
    }

    if (res === null) {
@@ -552,6 +556,9 @@ MegolmDecryption.prototype.decryptEvent = function(event) {
        this._addEventToPendingList(event);
        throw new base.DecryptionError(
            "The sender's device has not sent us the keys for this message.",
+            {
+                session: content.sender_key + '|' + content.session_id,
+            },
        );
    }

@@ -593,24 +600,30 @@ MegolmDecryption.prototype._addEventToPendingList = function(event) {
 * @param {module:models/event.MatrixEvent} event key event
 */
 MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
-    console.log("Adding key from ", event);
    const content = event.getContent();
+    const senderKey = event.getSenderKey();
+    const sessionId = content.session_id;

    if (!content.room_id ||
-        !content.session_id ||
+        !sessionId ||
        !content.session_key
       ) {
        console.error("key event is missing fields");
        return;
    }
+    if (!senderKey) {
+        console.error("key event has no sender key (not encrypted?)");
+        return;
+    }

+    console.log(`Adding key for megolm session ${senderKey}|${sessionId}`);
    this._olmDevice.addInboundGroupSession(
-        content.room_id, event.getSenderKey(), content.session_id,
+        content.room_id, senderKey, sessionId,
        content.session_key, event.getKeysClaimed(),
    );

    // have another go at decrypting events sent with this session.
-    this._retryDecryption(event.getSenderKey, content.session_id);
+    this._retryDecryption(senderKey, sessionId);
 };


@@ -64,7 +64,7 @@ OlmEncryption.prototype._ensureSession = function(roomMembers) {
    }

    const self = this;
-    this._prepPromise = self._crypto.downloadKeys(roomMembers, true).then(function(res) {
+    this._prepPromise = self._crypto.downloadKeys(roomMembers).then(function(res) {
        return self._crypto.ensureOlmSessionsForUsers(roomMembers);
    }).then(function() {
        self._sessionPrepared = true;
@@ -174,12 +174,12 @@ OlmDecryption.prototype.decryptEvent = function(event) {
    try {
        payloadString = this._decryptMessage(deviceKey, message);
    } catch (e) {
-        console.warn(
-            "Failed to decrypt Olm event (id=" +
-                event.getId() + ") from " + deviceKey +
-                ": " + e.message,
+        throw new base.DecryptionError(
+            "Bad Encrypted Message", {
+                sender: deviceKey,
+                err: e,
+            },
        );
-        throw new base.DecryptionError("Bad Encrypted Message");
    }

    const payload = JSON.parse(payloadString);
@@ -187,22 +187,18 @@ OlmDecryption.prototype.decryptEvent = function(event) {
    // check that we were the intended recipient, to avoid unknown-key attack
    // https://github.com/vector-im/vector-web/issues/2483
    if (payload.recipient != this._userId) {
-        console.warn(
-            "Event " + event.getId() + ": Intended recipient " +
-            payload.recipient + " does not match our id " + this._userId,
-        );
        throw new base.DecryptionError(
            "Message was intented for " + payload.recipient,
        );
    }

-    if (payload.recipient_keys.ed25519 !=
-               this._olmDevice.deviceEd25519Key) {
-        console.warn(
-            "Event " + event.getId() + ": Intended recipient ed25519 key " +
-            payload.recipient_keys.ed25519 + " did not match ours",
+    if (payload.recipient_keys.ed25519 != this._olmDevice.deviceEd25519Key) {
+        throw new base.DecryptionError(
+            "Message not intended for this device", {
+                intended: payload.recipient_keys.ed25519,
+                our_key: this._olmDevice.deviceEd25519Key,
+            },
        );
-        throw new base.DecryptionError("Message not intended for this device");
    }

    // check that the original sender matches what the homeserver told us, to
@@ -210,23 +206,19 @@ OlmDecryption.prototype.decryptEvent = function(event) {
    // (this check is also provided via the sender's embedded ed25519 key,
    // which is checked elsewhere).
    if (payload.sender != event.getSender()) {
-        console.warn(
-            "Event " + event.getId() + ": original sender " + payload.sender +
-            " does not match reported sender " + event.getSender(),
-        );
        throw new base.DecryptionError(
-            "Message forwarded from " + payload.sender,
+            "Message forwarded from " + payload.sender, {
+                reported_sender: event.getSender(),
+            },
        );
    }

    // Olm events intended for a room have a room_id.
    if (payload.room_id !== event.getRoomId()) {
-        console.warn(
-            "Event " + event.getId() + ": original room " + payload.room_id +
-            " does not match reported room " + event.room_id,
-        );
        throw new base.DecryptionError(
-            "Message intended for room " + payload.room_id,
+            "Message intended for room " + payload.room_id, {
+                reported_room: event.room_id,
+            },
        );
    }

@@ -61,6 +61,14 @@ function Crypto(baseApis, eventEmitter, sessionStore, userId, deviceId,

    this._olmDevice = new OlmDevice(sessionStore);
    this._deviceList = new DeviceList(baseApis, sessionStore, this._olmDevice);
+    this._initialDeviceListInvalidationDone = false;
+
+    this._clientRunning = false;
+
+    // the last time we did a check for the number of one-time-keys on the
+    // server.
+    this._lastOneTimeKeyCheck = null;
+    this._oneTimeKeyCheckInProgress = false;

    // EncryptionAlgorithm instance for each room
    this._roomEncryptors = {};
@@ -86,9 +94,6 @@ function Crypto(baseApis, eventEmitter, sessionStore, userId, deviceId,
    );

    if (!myDevices) {
-        // we don't yet have a list of our own devices; make sure we
-        // get one when we flush the pendingUsersWithNewDevices.
-        this._deviceList.invalidateUserDeviceList(this._userId);
        myDevices = {};
    }

@@ -113,6 +118,11 @@ function Crypto(baseApis, eventEmitter, sessionStore, userId, deviceId,
 function _registerEventHandlers(crypto, eventEmitter) {
    eventEmitter.on("sync", function(syncState, oldState, data) {
        try {
+            if (syncState === "STOPPED") {
+                crypto._clientRunning = false;
+            } else if (syncState === "PREPARED") {
+                crypto._clientRunning = true;
+            }
            if (syncState === "SYNCING") {
                crypto._onSyncCompleted(data);
            }
@@ -189,61 +199,11 @@ Crypto.prototype.getGlobalBlacklistUnverifiedDevices = function() {
 };

 /**
- * Upload the device keys to the homeserver and ensure that the
- * homeserver has enough one-time keys.
- * @param {number} maxKeys The maximum number of keys to generate
+ * Upload the device keys to the homeserver.
 * @return {object} A promise that will resolve when the keys are uploaded.
 */
-Crypto.prototype.uploadKeys = function(maxKeys) {
-    const self = this;
-    return _uploadDeviceKeys(this).then(function(res) {
-        // We need to keep a pool of one time public keys on the server so that
-        // other devices can start conversations with us. But we can only store
-        // a finite number of private keys in the olm Account object.
-        // To complicate things further then can be a delay between a device
-        // claiming a public one time key from the server and it sending us a
-        // message. We need to keep the corresponding private key locally until
-        // we receive the message.
-        // But that message might never arrive leaving us stuck with duff
-        // private keys clogging up our local storage.
-        // So we need some kind of enginering compromise to balance all of
-        // these factors.
-
-        // We first find how many keys the server has for us.
-        const keyCount = res.one_time_key_counts.signed_curve25519 || 0;
-        // We then check how many keys we can store in the Account object.
-        const maxOneTimeKeys = self._olmDevice.maxNumberOfOneTimeKeys();
-        // Try to keep at most half that number on the server. This leaves the
-        // rest of the slots free to hold keys that have been claimed from the
-        // server but we haven't recevied a message for.
-        // If we run out of slots when generating new keys then olm will
-        // discard the oldest private keys first. This will eventually clean
-        // out stale private keys that won't receive a message.
-        const keyLimit = Math.floor(maxOneTimeKeys / 2);
-        // We work out how many new keys we need to create to top up the server
-        // If there are too many keys on the server then we don't need to
-        // create any more keys.
-        let numberToGenerate = Math.max(keyLimit - keyCount, 0);
-        if (maxKeys !== undefined) {
-            // Creating keys can be an expensive operation so we limit the
-            // number we generate in one go to avoid blocking the application
-            // for too long.
-            numberToGenerate = Math.min(numberToGenerate, maxKeys);
-        }
-
-        if (numberToGenerate <= 0) {
-            // If we don't need to generate any keys then we are done.
-            return;
-        }
-
-        // Ask olm to generate new one time keys, then upload them to synapse.
-        self._olmDevice.generateOneTimeKeys(numberToGenerate);
-        return _uploadOneTimeKeys(self);
-    });
-};
-
-// returns a promise which resolves to the response
-function _uploadDeviceKeys(crypto) {
+Crypto.prototype.uploadDeviceKeys = function() {
+    const crypto = this;
    const userId = crypto._userId;
    const deviceId = crypto._deviceId;

@@ -262,6 +222,90 @@ function _uploadDeviceKeys(crypto) {
        // same one as used in login.
        device_id: deviceId,
    });
+};
+
+// check if it's time to upload one-time keys, and do so if so.
+function _maybeUploadOneTimeKeys(crypto) {
+    // frequency with which to check & upload one-time keys
+    const uploadPeriod = 1000 * 60; // one minute
+
+    // max number of keys to upload at once
+    // Creating keys can be an expensive operation so we limit the
+    // number we generate in one go to avoid blocking the application
+    // for too long.
+    const maxKeysPerCycle = 5;
+
+    if (crypto._oneTimeKeyCheckInProgress) {
+        return;
+    }
+
+    const now = Date.now();
+    if (crypto._lastOneTimeKeyCheck !== null &&
+        now - crypto._lastOneTimeKeyCheck < uploadPeriod
+       ) {
+        // we've done a key upload recently.
+        return;
+    }
+
+    crypto._lastOneTimeKeyCheck = now;
+
+    function uploadLoop(numberToGenerate) {
+        if (numberToGenerate <= 0) {
+            // If we don't need to generate any more keys then we are done.
+            return;
+        }
+
+        const keysThisLoop = Math.min(numberToGenerate, maxKeysPerCycle);
+
+        // Ask olm to generate new one time keys, then upload them to synapse.
+        crypto._olmDevice.generateOneTimeKeys(keysThisLoop);
+        return _uploadOneTimeKeys(crypto).then(() => {
+            return uploadLoop(numberToGenerate - keysThisLoop);
+        });
+    }
+
+    crypto._oneTimeKeyCheckInProgress = true;
+    q().then(() => {
+        // ask the server how many keys we have
+        return crypto._baseApis.uploadKeysRequest({}, {
+            device_id: crypto._deviceId,
+        });
+    }).then((res) => {
+        // We need to keep a pool of one time public keys on the server so that
+        // other devices can start conversations with us. But we can only store
+        // a finite number of private keys in the olm Account object.
+        // To complicate things further then can be a delay between a device
+        // claiming a public one time key from the server and it sending us a
+        // message. We need to keep the corresponding private key locally until
+        // we receive the message.
+        // But that message might never arrive leaving us stuck with duff
+        // private keys clogging up our local storage.
+        // So we need some kind of enginering compromise to balance all of
+        // these factors.
+
+        // We first find how many keys the server has for us.
+        const keyCount = res.one_time_key_counts.signed_curve25519 || 0;
+        // We then check how many keys we can store in the Account object.
+        const maxOneTimeKeys = crypto._olmDevice.maxNumberOfOneTimeKeys();
+        // Try to keep at most half that number on the server. This leaves the
+        // rest of the slots free to hold keys that have been claimed from the
+        // server but we haven't recevied a message for.
+        // If we run out of slots when generating new keys then olm will
+        // discard the oldest private keys first. This will eventually clean
+        // out stale private keys that won't receive a message.
+        const keyLimit = Math.floor(maxOneTimeKeys / 2);
+
+        // We work out how many new keys we need to create to top up the server
+        // If there are too many keys on the server then we don't need to
+        // create any more keys.
+        const numberToGenerate = Math.max(keyLimit - keyCount, 0);
+
+        return uploadLoop(numberToGenerate);
+    }).catch((e) => {
+        console.error("Error uploading one-time keys", e.stack || e);
+    }).finally(() => {
+        crypto._oneTimeKeyCheckInProgress = false;
+    }).done();
 }

 // returns a promise which resolves to the response
@@ -464,19 +508,19 @@ Crypto.prototype.getOlmSessionsForUser = function(userId) {
 * @return {module:crypto/deviceinfo?}
 */
 Crypto.prototype.getEventSenderDeviceInfo = function(event) {
-    const sender_key = event.getSenderKey();
+    const senderKey = event.getSenderKey();
    const algorithm = event.getWireContent().algorithm;

-    if (!sender_key || !algorithm) {
+    if (!senderKey || !algorithm) {
        return null;
    }

-    // sender_key is the Curve25519 identity key of the device which the event
+    // senderKey is the Curve25519 identity key of the device which the event
    // was sent from. In the case of Megolm, it's actually the Curve25519
    // identity key of the device which set up the Megolm session.

    const device = this._deviceList.getDeviceByIdentityKey(
-        event.getSender(), algorithm, sender_key,
+        event.getSender(), algorithm, senderKey,
    );

    if (device === null) {
@@ -545,6 +589,23 @@ Crypto.prototype.setRoomEncryption = function(roomId, config) {
        config: config,
    });
    this._roomEncryptors[roomId] = alg;
+
+    // if encryption was not previously enabled in this room, we will have been
+    // ignoring new device events for these users so far. We may well have
+    // up-to-date lists for some users, for instance if we were sharing other
+    // e2e rooms with them, so there is room for optimisation here, but for now
+    // we just invalidate everyone in the room.
+    if (!existingConfig) {
+        console.log("Enabling encryption in " + roomId + " for the first time; " +
+                    "invalidating device lists for all users therein");
+        const room = this._clientStore.getRoom(roomId);
+        const members = room.getJoinedMembers();
+        members.forEach((m) => {
+            this._deviceList.invalidateUserDeviceList(m.userId);
+        });
+        // the actual refresh happens once we've finished processing the sync,
+        // in _onSyncCompleted.
+    }
 };


@@ -750,7 +811,7 @@ Crypto.prototype._onCryptoEvent = function(event) {
 * @param {Object} syncData  the data from the 'MatrixClient.sync' event
 */
 Crypto.prototype._onSyncCompleted = function(syncData) {
-    this._deviceList.lastKnownSyncToken = syncData.nextSyncToken;
+    const nextSyncToken = syncData.nextSyncToken;

    if (!syncData.oldSyncToken) {
        // an initialsync.
@@ -759,22 +820,45 @@ Crypto.prototype._onSyncCompleted = function(syncData) {
        // if we have a deviceSyncToken, we can tell the deviceList to
        // invalidate devices which have changed since then.
        const oldSyncToken = this._sessionStore.getEndToEndDeviceSyncToken();
-        if (oldSyncToken) {
-            this._invalidateDeviceListsSince(oldSyncToken).catch((e) => {
+        if (oldSyncToken !== null) {
+            this._invalidateDeviceListsSince(
+                oldSyncToken, nextSyncToken,
+            ).catch((e) => {
                // if that failed, we fall back to invalidating everyone.
                console.warn("Error fetching changed device list", e);
                this._invalidateDeviceListForAllActiveUsers();
-                return this._deviceList.refreshOutdatedDeviceLists();
-            }).done();
+            }).done(() => {
+                this._initialDeviceListInvalidationDone = true;
+                this._deviceList.lastKnownSyncToken = nextSyncToken;
+                this._deviceList.refreshOutdatedDeviceLists();
+            });
        } else {
            // otherwise, we have to invalidate all devices for all users we
            // share a room with.
+            console.log("Completed first initialsync; invalidating all " +
+                        "device list caches");
            this._invalidateDeviceListForAllActiveUsers();
+            this._initialDeviceListInvalidationDone = true;
        }
    }

-    // catch up on any new devices we got told about during the sync.
-    this._deviceList.refreshOutdatedDeviceLists().done();
+    if (this._initialDeviceListInvalidationDone) {
+        // if we've got an up-to-date list of users with outdated device lists,
+        // tell the device list about the new sync token (but not otherwise, because
+        // otherwise we'll start thinking we're more in sync than we are.)
+        this._deviceList.lastKnownSyncToken = nextSyncToken;
+
+        // catch up on any new devices we got told about during the sync.
+        this._deviceList.refreshOutdatedDeviceLists();
+    }
+
+    // we don't start uploading one-time keys until we've caught up with
+    // to-device messages, to help us avoid throwing away one-time-keys that we
+    // are about to receive messages for
+    // (https://github.com/vector-im/riot-web/issues/2782).
+    if (!syncData.catchingUp) {
+        _maybeUploadOneTimeKeys(this);
+    }
 };

 /**
@@ -830,16 +914,19 @@ Crypto.prototype._sendNewDeviceEvents = function() {

 /**
 * Ask the server which users have new devices since a given token,
- * invalidate them, and start an update query.
+ * and invalidate them
 *
 * @param {String} oldSyncToken
+ * @param {String} lastKnownSyncToken
 *
 * @returns {Promise} resolves once the query is complete. Rejects if the
 *   keyChange query fails.
 */
-Crypto.prototype._invalidateDeviceListsSince = function(oldSyncToken) {
+Crypto.prototype._invalidateDeviceListsSince = function(
+    oldSyncToken, lastKnownSyncToken,
+) {
    return this._baseApis.getKeyChanges(
-        oldSyncToken, this.lastKnownSyncToken,
+        oldSyncToken, lastKnownSyncToken,
    ).then((r) => {
        if (!r.changed || !Array.isArray(r.changed)) {
            return;
@@ -853,7 +940,6 @@ Crypto.prototype._invalidateDeviceListsSince = function(oldSyncToken) {
                this._deviceList.invalidateUserDeviceList(u);
            }
        });
-        return this._deviceList.refreshOutdatedDeviceLists();
    });
 };

@@ -631,19 +631,26 @@ module.exports.MatrixHttpApi.prototype = {
        let timedOut = false;
        let req;
        const localTimeoutMs = opts.localTimeoutMs || this.opts.localTimeoutMs;
-        if (localTimeoutMs) {
-            timeoutId = callbacks.setTimeout(function() {
-                timedOut = true;
-                if (req && req.abort) {
-                    req.abort();
+
+        const resetTimeout = () => {
+            if (localTimeoutMs) {
+                if (timeoutId) {
+                    callbacks.clearTimeout(timeoutId);
                }
-                defer.reject(new module.exports.MatrixError({
-                    error: "Locally timed out waiting for a response",
-                    errcode: "ORG.MATRIX.JSSDK_TIMEOUT",
-                    timeout: localTimeoutMs,
-                }));
-            }, localTimeoutMs);
-        }
+                timeoutId = callbacks.setTimeout(function() {
+                    timedOut = true;
+                    if (req && req.abort) {
+                        req.abort();
+                    }
+                    defer.reject(new module.exports.MatrixError({
+                        error: "Locally timed out waiting for a response",
+                        errcode: "ORG.MATRIX.JSSDK_TIMEOUT",
+                        timeout: localTimeoutMs,
+                    }));
+                }, localTimeoutMs);
+            }
+        };
+        resetTimeout();

        const reqPromise = defer.promise;

@@ -679,10 +686,22 @@ module.exports.MatrixHttpApi.prototype = {
                    handlerFn(err, response, body);
                },
            );
-            if (req && req.abort) {
+            if (req) {
+                // This will only work in a browser, where opts.request is the
+                // `browser-request` import. Currently `request` does not support progress
+                // updates - see https://github.com/request/request/pull/2346.
+                // `browser-request` returns an XHRHttpRequest which exposes `onprogress`
+                if ('onprogress' in req) {
+                    req.onprogress = (e) => {
+                        // Prevent the timeout from rejecting the deferred promise if progress is
+                        // seen with the request
+                        resetTimeout();
+                    };
+                }
+
                // FIXME: This is EVIL, but I can't think of a better way to expose
                // abort() operations on underlying HTTP requests :(
-                reqPromise.abort = req.abort.bind(req);
+                if (req.abort) reqPromise.abort = req.abort.bind(req);
            }
        } catch (ex) {
            defer.reject(ex);
@@ -0,0 +1,24 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Separate exports file for the indexeddb web worker, which is designed
+ * to be used separately
+ */
+
+/** The {@link module:indexeddb-store-worker~IndexedDBStoreWorker} class. */
+module.exports.IndexedDBStoreWorker = require("./store/indexeddb-store-worker.js");
+
@@ -1,5 +1,6 @@
 /*
 Copyright 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,9 +18,13 @@ limitations under the License.

 /** @module interactive-auth */
 const q = require("q");
+const url = require("url");

 const utils = require("./utils");

+const EMAIL_STAGE_TYPE = "m.login.email.identity";
+const MSISDN_STAGE_TYPE = "m.login.msisdn";
+
 /**
 * Abstracts the logic used to drive the interactive auth process.
 *
@@ -37,44 +42,123 @@ const utils = require("./utils");
 *
 * @param {object} opts  options object
 *
+ * @param {object} opts.matrixClient A matrix client to use for the auth process
+ *
 * @param {object?} opts.authData error response from the last request. If
 *    null, a request will be made with no auth before starting.
 *
- * @param {function(object?): module:client.Promise} opts.doRequest
- *     called with the new auth dict to submit the request. Should return a
+ * @param {function(object?, bool?): module:client.Promise} opts.doRequest
+ *     called with the new auth dict to submit the request and a flag set
+ *     to true if this request is a background request. Should return a
 *     promise which resolves to the successful response or rejects with a
 *     MatrixError.
 *
- * @param {function(string, object?)} opts.startAuthStage
- *     called to ask the UI to start a particular auth stage. The arguments
- *     are: the login type (eg m.login.password); and (if the last request
- *     returned an error), an error object, with fields 'errcode' and 'error'.
+ * @param {function(string, object?)} opts.stateUpdated
+ *     called when the status of the UI auth changes, ie. when the state of
+ *     an auth stage changes of when the auth flow moves to a new stage.
+ *     The arguments are: the login type (eg m.login.password); and an object
+ *     which is either an error or an informational object specific to the
+ *     login type. If the 'errcode' key is defined, the object is an error,
+ *     and has keys:
+ *         errcode: string, the textual error code, eg. M_UNKNOWN
+ *         error: string, human readable string describing the error
+ *
+ *     The login type specific objects are as follows:
+ *         m.login.email.identity:
+ *          * emailSid: string, the sid of the active email auth session
+ *
+ * @param {object?} opts.inputs Inputs provided by the user and used by different
+ *     stages of the auto process. The inputs provided will affect what flow is chosen.
+ *
+ * @param {string?} opts.inputs.emailAddress An email address. If supplied, a flow
+ *     using email verification will be chosen.
+ *
+ * @param {string?} opts.inputs.phoneCountry An ISO two letter country code. Gives
+ *     the country that opts.phoneNumber should be resolved relative to.
+ *
+ * @param {string?} opts.inputs.phoneNumber A phone number. If supplied, a flow
+ *     using phone number validation will be chosen.
+ *
+ * @param {string?} opts.sessionId If resuming an existing interactive auth session,
+ *     the sessionId of that session.
+ *
+ * @param {string?} opts.clientSecret If resuming an existing interactive auth session,
+ *     the client secret for that session
+ *
+ * @param {string?} opts.emailSid If returning from having completed m.login.email.identity
+ *     auth, the sid for the email verification session.
 *
 */
 function InteractiveAuth(opts) {
-    this._data = opts.authData;
+    this._matrixClient = opts.matrixClient;
+    this._data = opts.authData || {};
    this._requestCallback = opts.doRequest;
-    this._startAuthStageCallback = opts.startAuthStage;
+    // startAuthStage included for backwards compat
+    this._stateUpdatedCallback = opts.stateUpdated || opts.startAuthStage;
    this._completionDeferred = null;
+    this._inputs = opts.inputs || {};
+
+    if (opts.sessionId) this._data.session = opts.sessionId;
+    this._clientSecret = opts.clientSecret || this._matrixClient.generateClientSecret();
+    this._emailSid = opts.emailSid;
+    if (this._emailSid === undefined) this._emailSid = null;
+
+    this._currentStage = null;
 }

 InteractiveAuth.prototype = {
    /**
     * begin the authentication process.
     *
-     * @return {module:client.Promise}  which resolves to the response on success,
-     * or rejects with the error on failure.
+     * @return {module:client.Promise} which resolves to the response on success,
+     * or rejects with the error on failure. Rejects with NoAuthFlowFoundError if
+     *     no suitable authentication flow can be found
     */
    attemptAuth: function() {
        this._completionDeferred = q.defer();

-        if (!this._data) {
-            this._doRequest(null);
-        } else {
-            this._startNextAuthStage();
+        // wrap in a promise so that if _startNextAuthStage
+        // throws, it rejects the promise in a consistent way
+        return q().then(() => {
+            // if we have no flows, try a request (we'll have
+            // just a session ID in _data if resuming)
+            if (!this._data.flows) {
+                this._doRequest(this._data);
+            } else {
+                this._startNextAuthStage();
+            }
+            return this._completionDeferred.promise;
+        });
+    },
+
+    /**
+     * Poll to check if the auth session or current stage has been
+     * completed out-of-band. If so, the attemptAuth promise will
+     * be resolved.
+     */
+    poll: function() {
+        if (!this._data.session) return;
+
+        let authDict = {};
+        if (this._currentStage == EMAIL_STAGE_TYPE) {
+            // The email can be validated out-of-band, but we need to provide the
+            // creds so the HS can go & check it.
+            if (this._emailSid) {
+                const idServerParsedUrl = url.parse(
+                    this._matrixClient.getIdentityServerUrl(),
+                );
+                authDict = {
+                    type: EMAIL_STAGE_TYPE,
+                    threepid_creds: {
+                        sid: this._emailSid,
+                        client_secret: this._clientSecret,
+                        id_server: idServerParsedUrl.host,
+                    },
+                };
+            }
        }

-        return this._completionDeferred.promise;
+        this.submitAuthDict(authDict, true);
    },

    /**
@@ -86,6 +170,16 @@ InteractiveAuth.prototype = {
        return this._data ? this._data.session : undefined;
    },

+    /**
+     * get the client secret used for validation sessions
+     * with the ID server.
+     *
+     * @return {string} client secret
+     */
+    getClientSecret: function() {
+        return this._clientSecret;
+    },
+
    /**
     * get the server params for a given stage
     *
@@ -108,8 +202,11 @@ InteractiveAuth.prototype = {
     * @param {object} authData new auth dict to send to the server. Should
     *    include a `type` propterty denoting the login type, as well as any
     *    other params for that stage.
+     * @param {bool} background If true, this request failing will not result
+     *    in the attemptAuth promise being rejected. This can be set to true
+     *    for requests that just poll to see if auth has been completed elsewhere.
     */
-    submitAuthDict: function(authData) {
+    submitAuthDict: function(authData, background) {
        if (!this._completionDeferred) {
            throw new Error("submitAuthDict() called before attemptAuth()");
        }
@@ -120,7 +217,29 @@ InteractiveAuth.prototype = {
        };
        utils.extend(auth, authData);

-        this._doRequest(auth);
+        this._doRequest(auth, background);
+    },
+
+    /**
+     * Gets the sid for the email validation session
+     * Specific to m.login.email.identity
+     *
+     * @returns {string} The sid of the email auth session
+     */
+    getEmailSid: function() {
+        return this._emailSid;
+    },
+
+    /**
+     * Sets the sid for the email validation session
+     * This must be set in order to successfully poll for completion
+     * of the email validation.
+     * Specific to m.login.email.identity
+     *
+     * @param {string} sid The sid for the email validation session
+     */
+    setEmailSid: function(sid) {
+        this._emailSid = sid;
    },

    /**
@@ -129,8 +248,12 @@ InteractiveAuth.prototype = {
     *
     * @private
     * @param {object?} auth new auth dict, including session id
+     * @param {bool?} background If true, this request is a background poll, so it
+     *    failing will not result in the attemptAuth promise being rejected.
+     *    This can be set to true for requests that just poll to see if auth has
+     *    been completed elsewhere.
     */
-    _doRequest: function(auth) {
+    _doRequest: function(auth, background) {
        const self = this;

        // hackery to make sure that synchronous exceptions end up in the catch
@@ -138,45 +261,76 @@ InteractiveAuth.prototype = {
        // extra q().then)
        let prom;
        try {
-            prom = this._requestCallback(auth);
+            prom = this._requestCallback(auth, background);
        } catch (e) {
            prom = q.reject(e);
        }

-        prom.then(
+        prom = prom.then(
            function(result) {
                console.log("result from request: ", result);
                self._completionDeferred.resolve(result);
            }, function(error) {
-                if (error.httpStatus !== 401 || !error.data || !error.data.flows) {
+                // sometimes UI auth errors don't come with flows
+                const errorFlows = error.data ? error.data.flows : null;
+                const haveFlows = Boolean(self._data.flows) || Boolean(errorFlows);
+                if (error.httpStatus !== 401 || !error.data || !haveFlows) {
                    // doesn't look like an interactive-auth failure. fail the whole lot.
                    throw error;
                }
+                // if the error didn't come with flows, completed flows or session ID,
+                // copy over the ones we have. Synapse sometimes sends responses without
+                // any UI auth data (eg. when polling for email validation, if the email
+                // has not yet been validated). This appears to be a Synapse bug, which
+                // we workaround here.
+                if (!error.data.flows && !error.data.completed && !error.data.session) {
+                    error.data.flows = self._data.flows;
+                    error.data.completed = self._data.completed;
+                    error.data.session = self._data.session;
+                }
                self._data = error.data;
                self._startNextAuthStage();
            },
-        ).catch(this._completionDeferred.reject).done();
+        );
+        if (!background) {
+            prom = prom.catch(this._completionDeferred.reject);
+        } else {
+            // We ignore all failures here (even non-UI auth related ones)
+            // since we don't want to suddenly fail if the internet connection
+            // had a blip whilst we were polling
+            prom = prom.catch((error) => {
+                console.log("Ignoring error from UI auth: " + error);
+            });
+        }
+        prom.done();
    },

    /**
     * Pick the next stage and call the callback
     *
     * @private
+     * @throws {NoAuthFlowFoundError} If no suitable authentication flow can be found
     */
    _startNextAuthStage: function() {
        const nextStage = this._chooseStage();
        if (!nextStage) {
            throw new Error("No incomplete flows from the server");
        }
+        this._currentStage = nextStage;

-        let stageError = null;
        if (this._data.errcode || this._data.error) {
-            stageError = {
+            this._stateUpdatedCallback(nextStage, {
                errcode: this._data.errcode || "",
                error: this._data.error || "",
-            };
+            });
+            return;
        }
-        this._startAuthStageCallback(nextStage, stageError);
+
+        const stageStatus = {};
+        if (nextStage == EMAIL_STAGE_TYPE) {
+            stageStatus.emailSid = this._emailSid;
+        }
+        this._stateUpdatedCallback(nextStage, stageStatus);
    },

    /**
@@ -184,6 +338,7 @@ InteractiveAuth.prototype = {
     *
     * @private
     * @return {string?} login type
+     * @throws {NoAuthFlowFoundError} If no suitable authentication flow can be found
     */
    _chooseStage: function() {
        const flow = this._chooseFlow();
@@ -195,14 +350,53 @@ InteractiveAuth.prototype = {

    /**
     * Pick one of the flows from the returned list
+     * If a flow using all of the inputs is found, it will
+     * be returned, otherwise, null will be returned.
+     *
+     * Only flows using all given inputs are chosen because it
+     * is likley to be surprising if the user provides a
+     * credential and it is not used. For example, for registration,
+     * this could result in the email not being used which would leave
+     * the account with no means to reset a password.
     *
     * @private
     * @return {object} flow
+     * @throws {NoAuthFlowFoundError} If no suitable authentication flow can be found
     */
    _chooseFlow: function() {
        const flows = this._data.flows || [];
-        // always use the first flow for now
-        return flows[0];
+
+        // we've been given an email or we've already done an email part
+        const haveEmail = Boolean(this._inputs.emailAddress) || Boolean(this._emailSid);
+        const haveMsisdn = (
+            Boolean(this._inputs.phoneCountry) &&
+            Boolean(this._inputs.phoneNumber)
+        );
+
+        for (const flow of flows) {
+            let flowHasEmail = false;
+            let flowHasMsisdn = false;
+            for (const stage of flow.stages) {
+                if (stage === EMAIL_STAGE_TYPE) {
+                    flowHasEmail = true;
+                } else if (stage == MSISDN_STAGE_TYPE) {
+                    flowHasMsisdn = true;
+                }
+            }
+
+            if (flowHasEmail == haveEmail && flowHasMsisdn == haveMsisdn) {
+                return flow;
+            }
+        }
+        // Throw an error with a fairly generic description, but with more
+        // information such that the app can give a better one if so desired.
+        const err = new Error("No appropriate authentication flow found");
+        err.name = 'NoAuthFlowFoundError';
+        err.required_stages = [];
+        if (haveEmail) err.required_stages.push(EMAIL_STAGE_TYPE);
+        if (haveMsisdn) err.required_stages.push(MSISDN_STAGE_TYPE);
+        err.available_flows = flows;
+        throw err;
    },

    /**
@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,6 +22,12 @@ module.exports.MatrixEvent = require("./models/event").MatrixEvent;
 module.exports.EventStatus = require("./models/event").EventStatus;
 /** The {@link module:store/memory.MatrixInMemoryStore|MatrixInMemoryStore} class. */
 module.exports.MatrixInMemoryStore = require("./store/memory").MatrixInMemoryStore;
+/** The {@link module:store/indexeddb.IndexedDBStore|IndexedDBStore} class. */
+module.exports.IndexedDBStore = require("./store/indexeddb").IndexedDBStore;
+/** The {@link module:store/indexeddb.IndexedDBStoreBackend|IndexedDBStoreBackend} class. */
+module.exports.IndexedDBStoreBackend = require("./store/indexeddb").IndexedDBStoreBackend;
+/** The {@link module:sync-accumulator.SyncAccumulator|SyncAccumulator} class. */
+module.exports.SyncAccumulator = require("./sync-accumulator");
 /** The {@link module:http-api.MatrixHttpApi|MatrixHttpApi} class. */
 module.exports.MatrixHttpApi = require("./http-api").MatrixHttpApi;
 /** The {@link module:http-api.MatrixError|MatrixError} class. */
@@ -154,10 +154,11 @@ EventTimelineSet.prototype.replaceEventId = function(oldEventId, newEventId) {
 * @fires module:client~MatrixClient#event:"Room.timelineReset"
 */
 EventTimelineSet.prototype.resetLiveTimeline = function(backPaginationToken, flush) {
-    let newTimeline;
+    // if timeline support is disabled, forget about the old timelines
+    const resetAllTimelines = !this._timelineSupport || flush;

-    if (!this._timelineSupport || flush) {
-        // if timeline support is disabled, forget about the old timelines
+    let newTimeline;
+    if (resetAllTimelines) {
        newTimeline = new EventTimeline(this);
        this._timelines = [newTimeline];
        this._eventIdToTimeline = {};
@@ -187,7 +188,7 @@ EventTimelineSet.prototype.resetLiveTimeline = function(backPaginationToken, flu
    newTimeline.setPaginationToken(backPaginationToken, EventTimeline.BACKWARDS);

    this._liveTimeline = newTimeline;
-    this.emit("Room.timelineReset", this.room, this);
+    this.emit("Room.timelineReset", this.room, this, resetAllTimelines);
 };

 /**
@@ -655,4 +656,5 @@ module.exports = EventTimelineSet;
 * @event module:client~MatrixClient#"Room.timelineReset"
 * @param {Room} room The room whose live timeline was reset, if any
 * @param {EventTimelineSet} timelineSet timelineSet room whose live timeline was reset
+ * @param {boolean} resetAllTimelines True if all timelines were reset.
 */
@@ -1,3 +1,18 @@
+/*
+Copyright 2016, 2017 OpenMarket Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
 "use strict";

 /**
@@ -49,6 +49,8 @@ module.exports.EventStatus = {
    CANCELLED: "cancelled",
 };

+const interns = {};
+
 /**
 * Construct a Matrix Event object
 * @constructor
@@ -61,7 +63,9 @@ module.exports.EventStatus = {
 * from changes to event JSON between Matrix versions.
 *
 * @prop {RoomMember} sender The room member who sent this event, or null e.g.
- * this is a presence event.
+ * this is a presence event. This is only guaranteed to be set for events that
+ * appear in a timeline, ie. do not guarantee that it will be set on state
+ * events.
 * @prop {RoomMember} target The room member who is the target of this event, e.g.
 * the invitee, the person being banned, etc.
 * @prop {EventStatus} status The sending status of the event.
@@ -73,7 +77,31 @@ module.exports.EventStatus = {
 module.exports.MatrixEvent = function MatrixEvent(
    event,
 ) {
+    // intern the values of matrix events to force share strings and reduce the
+    // amount of needless string duplication. This can save moderate amounts of
+    // memory (~10% on a 350MB heap).
+    ["state_key", "type", "sender", "room_id"].forEach((prop) => {
+        if (!event[prop]) {
+            return;
+        }
+        if (!interns[event[prop]]) {
+            interns[event[prop]] = event[prop];
+        }
+        event[prop] = interns[event[prop]];
+    });
+
+    ["membership", "avatar_url", "displayname"].forEach((prop) => {
+        if (!event.content || !event.content[prop]) {
+            return;
+        }
+        if (!interns[event.content[prop]]) {
+            interns[event.content[prop]] = event.content[prop];
+        }
+        event.content[prop] = interns[event.content[prop]];
+    });
+
    this.event = event || {};
+
    this.sender = null;
    this.target = null;
    this.status = null;
@@ -404,31 +432,18 @@ utils.extend(module.exports.MatrixEvent.prototype, {
 });


-/* http://matrix.org/docs/spec/r0.0.1/client_server.html#redactions says:
+/* _REDACT_KEEP_KEY_MAP gives the keys we keep when an event is redacted
 *
- * the server should strip off any keys not in the following list:
- *    event_id
- *    type
- *    room_id
- *    user_id
- *    state_key
- *    prev_state
- *    content
- *    [we keep 'unsigned' as well, since that is created by the local server]
+ * This is specified here:
+ *  http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#redactions
 *
- * The content object should also be stripped of all keys, unless it is one of
- * one of the following event types:
- *    m.room.member allows key membership
- *    m.room.create allows key creator
- *    m.room.join_rules allows key join_rule
- *    m.room.power_levels allows keys ban, events, events_default, kick,
- *        redact, state_default, users, users_default.
- *    m.room.aliases allows key aliases
+ * Also:
+ *  - We keep 'unsigned' since that is created by the local server
+ *  - We keep user_id for backwards-compat with v1
 */
-// a map giving the keys we keep when an event is redacted
 const _REDACT_KEEP_KEY_MAP = [
-    'event_id', 'type', 'room_id', 'user_id', 'state_key', 'prev_state',
-    'content', 'unsigned',
+    'event_id', 'type', 'room_id', 'user_id', 'sender', 'state_key', 'prev_state',
+    'content', 'unsigned', 'origin_server_ts',
 ].reduce(function(ret, val) {
    ret[val] = 1; return ret;
 }, {});
@@ -204,10 +204,12 @@ Room.prototype.getLiveTimeline = function() {
 * <p>This is used when /sync returns a 'limited' timeline.
 *
 * @param {string=} backPaginationToken   token for back-paginating the new timeline
+ * @param {boolean=} flush True to remove all events in all timelines. If false, only
+ * the live timeline is reset.
 */
-Room.prototype.resetLiveTimeline = function(backPaginationToken) {
+Room.prototype.resetLiveTimeline = function(backPaginationToken, flush) {
    for (let i = 0; i < this._timelineSets.length; i++) {
-        this._timelineSets[i].resetLiveTimeline(backPaginationToken);
+        this._timelineSets[i].resetLiveTimeline(backPaginationToken, flush);
    }

    this._fixUpLegacyTimelineFields();
@@ -1175,7 +1177,9 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {

    // get members that are NOT ourselves and are actually in the room.
    const otherMembers = utils.filter(room.currentState.getMembers(), function(m) {
-        return (m.userId !== userId && m.membership !== "leave");
+        return (
+            m.userId !== userId && m.membership !== "leave" && m.membership !== "ban"
+        );
    });
    const allMembers = utils.filter(room.currentState.getMembers(), function(m) {
        return (m.membership !== "leave");
@@ -168,7 +168,7 @@ function _runCallbacks() {
    for (let i = 0; i < callbacksToRun.length; i++) {
        cb = callbacksToRun[i];
        try {
-            cb.func.apply(null, cb.params);
+            cb.func.apply(global, cb.params);
        } catch (e) {
            console.error("Uncaught exception in callback function",
                          e.stack || e);
@@ -0,0 +1,318 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import q from "q";
+import SyncAccumulator from "../sync-accumulator";
+import utils from "../utils";
+
+const VERSION = 1;
+
+function createDatabase(db) {
+    // Make user store, clobber based on user ID. (userId property of User objects)
+    db.createObjectStore("users", { keyPath: ["userId"] });
+
+    // Make account data store, clobber based on event type.
+    // (event.type property of MatrixEvent objects)
+    db.createObjectStore("accountData", { keyPath: ["type"] });
+
+    // Make /sync store (sync tokens, room data, etc), always clobber (const key).
+    db.createObjectStore("sync", { keyPath: ["clobber"] });
+}
+
+/**
+ * Helper method to collect results from a Cursor and promiseify it.
+ * @param {ObjectStore|Index} store The store to perform openCursor on.
+ * @param {IDBKeyRange=} keyRange Optional key range to apply on the cursor.
+ * @param {Function} resultMapper A function which is repeatedly called with a
+ * Cursor.
+ * Return the data you want to keep.
+ * @return {Promise<T[]>} Resolves to an array of whatever you returned from
+ * resultMapper.
+ */
+function selectQuery(store, keyRange, resultMapper) {
+    const query = store.openCursor(keyRange);
+    return q.Promise((resolve, reject) => { /*eslint new-cap: 0*/
+        const results = [];
+        query.onerror = (event) => {
+            reject(new Error("Query failed: " + event.target.errorCode));
+        };
+        // collect results
+        query.onsuccess = (event) => {
+            const cursor = event.target.result;
+            if (!cursor) {
+                resolve(results);
+                return; // end of results
+            }
+            results.push(resultMapper(cursor));
+            cursor.continue();
+        };
+    });
+}
+
+function promiseifyTxn(txn) {
+    return new q.Promise((resolve, reject) => {
+        txn.oncomplete = function(event) {
+            resolve(event);
+        };
+        txn.onerror = function(event) {
+            reject(event);
+        };
+    });
+}
+
+function promiseifyRequest(req) {
+    return new q.Promise((resolve, reject) => {
+        req.onsuccess = function(event) {
+            resolve(event);
+        };
+        req.onerror = function(event) {
+            reject(event);
+        };
+    });
+}
+
+/**
+ * Does the actual reading from and writing to the indexeddb
+ *
+ * Construct a new Indexed Database store backend. This requires a call to
+ * <code>connect()</code> before this store can be used.
+ * @constructor
+ * @param {Object} indexedDBInterface The Indexed DB interface e.g
+ * <code>window.indexedDB</code>
+ * @param {string=} dbName Optional database name. The same name must be used
+ * to open the same database.
+ */
+const LocalIndexedDBStoreBackend = function LocalIndexedDBStoreBackend(
+    indexedDBInterface, dbName,
+) {
+    this.indexedDB = indexedDBInterface;
+    this._dbName = "matrix-js-sdk:" + (dbName || "default");
+    this.db = null;
+    this._syncAccumulator = new SyncAccumulator();
+};
+
+
+LocalIndexedDBStoreBackend.prototype = {
+    /**
+     * Attempt to connect to the database. This can fail if the user does not
+     * grant permission.
+     * @return {Promise} Resolves if successfully connected.
+     */
+    connect: function() {
+        if (this.db) {
+            return q();
+        }
+        const req = this.indexedDB.open(this._dbName, VERSION);
+        req.onupgradeneeded = (ev) => {
+            const db = ev.target.result;
+            const oldVersion = ev.oldVersion;
+            if (oldVersion < 1) { // The database did not previously exist.
+                createDatabase(db);
+            }
+            // Expand as needed.
+        };
+
+        return promiseifyRequest(req).then((ev) => {
+            this.db = ev.target.result;
+
+            // add a poorly-named listener for when deleteDatabase is called
+            // so we can close our db connections.
+            this.db.onversionchange = () => {
+                this.db.close();
+            };
+
+            return this._init();
+        });
+    },
+
+    /**
+     * Having connected, load initial data from the database and prepare for use
+     * @return {Promise} Resolves on success
+     */
+    _init: function() {
+        return q.all([
+            this._loadAccountData(),
+            this._loadSyncData(),
+        ]).then(([accountData, syncData]) => {
+            this._syncAccumulator.accumulate({
+                next_batch: syncData.nextBatch,
+                rooms: syncData.roomsData,
+                account_data: {
+                    events: accountData,
+                },
+            });
+        });
+    },
+
+    /**
+     * Clear the entire database. This should be used when logging out of a client
+     * to prevent mixing data between accounts.
+     * @return {Promise} Resolved when the database is cleared.
+     */
+    clearDatabase: function() {
+        console.log("Removing indexeddb instance: ", this._dbName);
+        return promiseifyRequest(this.indexedDB.deleteDatabase(this._dbName));
+    },
+
+    /**
+     * @param {boolean=} copy If false, the data returned is from internal
+     * buffers and must not be muated. Otherwise, a copy is made before
+     * returning such that the data can be safely mutated. Default: true.
+     *
+     * @return {Promise} Resolves with a sync response to restore the
+     * client state to where it was at the last save, or null if there
+     * is no saved sync data.
+     */
+    getSavedSync: function(copy) {
+        if (copy === undefined) copy = true;
+
+        const data = this._syncAccumulator.getJSON();
+        if (!data.nextBatch) return q(null);
+        if (copy) {
+            // We must deep copy the stored data so that the /sync processing code doesn't
+            // corrupt the internal state of the sync accumulator (it adds non-clonable keys)
+            return q(utils.deepCopy(data));
+        } else {
+            return q(data);
+        }
+    },
+
+    setSyncData: function(syncData) {
+        return q().then(() => {
+            this._syncAccumulator.accumulate(syncData);
+        });
+    },
+
+    syncToDatabase: function(userTuples) {
+        const syncData = this._syncAccumulator.getJSON();
+
+        return q.all([
+            this._persistUserPresenceEvents(userTuples),
+            this._persistAccountData(syncData.accountData),
+            this._persistSyncData(syncData.nextBatch, syncData.roomsData),
+        ]);
+    },
+
+    /**
+     * Persist rooms /sync data along with the next batch token.
+     * @param {string} nextBatch The next_batch /sync value.
+     * @param {Object} roomsData The 'rooms' /sync data from a SyncAccumulator
+     * @return {Promise} Resolves if the data was persisted.
+     */
+    _persistSyncData: function(nextBatch, roomsData) {
+        console.log("Persisting sync data up to ", nextBatch);
+        return q.try(() => {
+            const txn = this.db.transaction(["sync"], "readwrite");
+            const store = txn.objectStore("sync");
+            store.put({
+                clobber: "-", // constant key so will always clobber
+                nextBatch: nextBatch,
+                roomsData: roomsData,
+            }); // put == UPSERT
+            return promiseifyTxn(txn);
+        });
+    },
+
+    /**
+     * Persist a list of account data events. Events with the same 'type' will
+     * be replaced.
+     * @param {Object[]} accountData An array of raw user-scoped account data events
+     * @return {Promise} Resolves if the events were persisted.
+     */
+    _persistAccountData: function(accountData) {
+        return q.try(() => {
+            const txn = this.db.transaction(["accountData"], "readwrite");
+            const store = txn.objectStore("accountData");
+            for (let i = 0; i < accountData.length; i++) {
+                store.put(accountData[i]); // put == UPSERT
+            }
+            return promiseifyTxn(txn);
+        });
+    },
+
+    /**
+     * Persist a list of [user id, presence event] they are for.
+     * Users with the same 'userId' will be replaced.
+     * Presence events should be the event in its raw form (not the Event
+     * object)
+     * @param {Object[]} tuples An array of [userid, event] tuples
+     * @return {Promise} Resolves if the users were persisted.
+     */
+    _persistUserPresenceEvents: function(tuples) {
+        return q.try(() => {
+            const txn = this.db.transaction(["users"], "readwrite");
+            const store = txn.objectStore("users");
+            for (const tuple of tuples) {
+                store.put({
+                    userId: tuple[0],
+                    event: tuple[1],
+                }); // put == UPSERT
+            }
+            return promiseifyTxn(txn);
+        });
+    },
+
+    /**
+     * Load all user presence events from the database. This is not cached.
+     * FIXME: It would probably be more sensible to store the events in the
+     * sync.
+     * @return {Promise<Object[]>} A list of presence events in their raw form.
+     */
+    getUserPresenceEvents: function() {
+        return q.try(() => {
+            const txn = this.db.transaction(["users"], "readonly");
+            const store = txn.objectStore("users");
+            return selectQuery(store, undefined, (cursor) => {
+                return [cursor.value.userId, cursor.value.event];
+            });
+        });
+    },
+
+    /**
+     * Load all the account data events from the database. This is not cached.
+     * @return {Promise<Object[]>} A list of raw global account events.
+     */
+    _loadAccountData: function() {
+        return q.try(() => {
+            const txn = this.db.transaction(["accountData"], "readonly");
+            const store = txn.objectStore("accountData");
+            return selectQuery(store, undefined, (cursor) => {
+                return cursor.value;
+            });
+        });
+    },
+
+    /**
+     * Load the sync data from the database.
+     * @return {Promise<Object>} An object with "roomsData" and "nextBatch" keys.
+     */
+    _loadSyncData: function() {
+        return q.try(() => {
+            const txn = this.db.transaction(["sync"], "readonly");
+            const store = txn.objectStore("sync");
+            return selectQuery(store, undefined, (cursor) => {
+                return cursor.value;
+            }).then((results) => {
+                if (results.length > 1) {
+                    console.warn("loadSyncData: More than 1 sync row found.");
+                }
+                return (results.length > 0 ? results[0] : {});
+            });
+        });
+    },
+};
+
+export default LocalIndexedDBStoreBackend;
@@ -0,0 +1,140 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import q from "q";
+
+/**
+ * An IndexedDB store backend where the actual backend sits in a web
+ * worker.
+ *
+ * Construct a new Indexed Database store backend. This requires a call to
+ * <code>connect()</code> before this store can be used.
+ * @constructor
+ * @param {string} workerScript URL to the worker script
+ * @param {string=} dbName Optional database name. The same name must be used
+ * to open the same database.
+ * @param {Object} WorkerApi The web worker compatible interface object
+ */
+const RemoteIndexedDBStoreBackend = function RemoteIndexedDBStoreBackend(
+    workerScript, dbName, WorkerApi,
+) {
+    this._dbName = dbName;
+    this._worker = new WorkerApi(workerScript);
+    this._nextSeq = 0;
+    // The currently in-flight requests to the actual backend
+    this._inFlight = {
+        // seq: promise,
+    };
+
+    this._worker.onmessage = this._onWorkerMessage.bind(this);
+};
+
+
+RemoteIndexedDBStoreBackend.prototype = {
+    /**
+     * Attempt to connect to the database. This can fail if the user does not
+     * grant permission.
+     * @return {Promise} Resolves if successfully connected.
+     */
+    connect: function() {
+        return this._doCmd('_setupWorker', [this._dbName]).then(() => {
+            console.log("IndexedDB worker is ready");
+            return this._doCmd('connect');
+        });
+    },
+
+    /**
+     * Clear the entire database. This should be used when logging out of a client
+     * to prevent mixing data between accounts.
+     * @return {Promise} Resolved when the database is cleared.
+     */
+    clearDatabase: function() {
+        return this._doCmd('clearDatabase');
+    },
+
+    /**
+     * @return {Promise} Resolves with a sync response to restore the
+     * client state to where it was at the last save, or null if there
+     * is no saved sync data.
+     */
+    getSavedSync: function() {
+        return this._doCmd('getSavedSync');
+    },
+
+    setSyncData: function(syncData) {
+        return this._doCmd('setSyncData', [syncData]);
+    },
+
+    syncToDatabase: function(users) {
+        return this._doCmd('syncToDatabase', [users]);
+    },
+
+
+    /**
+     * Load all user presence events from the database. This is not cached.
+     * @return {Promise<Object[]>} A list of presence events in their raw form.
+     */
+    getUserPresenceEvents: function() {
+        return this._doCmd('getUserPresenceEvents');
+    },
+
+    _doCmd: function(cmd, args) {
+        // wrap in a q so if the postMessage throws,
+        // the promise automatically gets rejected
+        return q().then(() => {
+            const seq = this._nextSeq++;
+            const def = q.defer();
+
+            this._inFlight[seq] = def;
+
+            this._worker.postMessage({
+                command: cmd,
+                seq: seq,
+                args: args,
+            });
+
+            return def.promise;
+        });
+    },
+
+    _onWorkerMessage: function(ev) {
+        const msg = ev.data;
+
+        if (msg.command == 'cmd_success' || msg.command == 'cmd_fail') {
+            if (msg.seq === undefined) {
+                console.error("Got reply from worker with no seq");
+                return;
+            }
+
+            const def = this._inFlight[msg.seq];
+            if (def === undefined) {
+                console.error("Got reply for unknown seq " + msg.seq);
+                return;
+            }
+            delete this._inFlight[msg.seq];
+
+            if (msg.command == 'cmd_success') {
+                def.resolve(msg.result);
+            } else {
+                def.reject(msg.error);
+            }
+        } else {
+            console.warn("Unrecognised message from worker: " + msg);
+        }
+    },
+};
+
+export default RemoteIndexedDBStoreBackend;
@@ -0,0 +1,122 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import q from "q";
+import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";
+
+/**
+ * This class lives in the webworker and drives a LocalIndexedDBStoreBackend
+ * controlled by messages from the main process.
+ *
+ * It should be instantiated by a web worker script provided by the application
+ * in a script, for example:
+ *
+ * import {IndexedDBStoreWorker} from 'matrix-js-sdk/lib/indexeddb-worker.js';
+ * const remoteWorker = new IndexedDBStoreWorker(postMessage);
+ * onmessage = remoteWorker.onMessage;
+ *
+ * Note that it is advisable to import this class by referencing the file directly to
+ * avoid a dependency on the whole js-sdk.
+ *
+ */
+class IndexedDBStoreWorker {
+    /**
+     * @param {function} postMessage The web worker postMessage function that
+     * should be used to communicate back to the main script.
+     */
+    constructor(postMessage) {
+        this.backend = null;
+        this.postMessage = postMessage;
+
+        this.onMessage = this.onMessage.bind(this);
+    }
+
+    /**
+     * Passes a message event from the main script into the class. This method
+     * can be directly assigned to the web worker `onmessage` variable.
+     *
+     * @param {Object} ev The message event
+     */
+    onMessage(ev) {
+        const msg = ev.data;
+        let prom;
+
+        switch (msg.command) {
+            case '_setupWorker':
+                this.backend = new LocalIndexedDBStoreBackend(
+                    // this is the 'indexedDB' global (where global != window
+                    // because it's a web worker and there is no window).
+                    indexedDB, msg.args[0],
+                );
+                prom = q();
+                break;
+            case 'connect':
+                prom = this.backend.connect();
+                break;
+            case 'clearDatabase':
+                prom = this.backend.clearDatabase().then((result) => {
+                    // This returns special classes which can't be cloned
+                    // across to the main script, so don't try.
+                    return {};
+                });
+                break;
+            case 'getSavedSync':
+                prom = this.backend.getSavedSync(false);
+                break;
+            case 'setSyncData':
+                prom = this.backend.setSyncData(...msg.args);
+                break;
+            case 'syncToDatabase':
+                prom = this.backend.syncToDatabase(...msg.args).then(() => {
+                    // This also returns IndexedDB events which are not cloneable
+                    return {};
+                });
+                break;
+            case 'getUserPresenceEvents':
+                prom = this.backend.getUserPresenceEvents();
+                break;
+        }
+
+        if (prom === undefined) {
+            postMessage({
+                command: 'cmd_fail',
+                seq: msg.seq,
+                // Can't be an Error because they're not structured cloneable
+                error: "Unrecognised command",
+            });
+            return;
+        }
+
+        prom.done((ret) => {
+            this.postMessage.call(null, {
+                command: 'cmd_success',
+                seq: msg.seq,
+                result: ret,
+            });
+        }, (err) => {
+            console.error("Error running command: "+msg.command);
+            console.error(err);
+            this.postMessage.call(null, {
+                command: 'cmd_fail',
+                seq: msg.seq,
+                // Just send a string because Error objects aren't cloneable
+                error: "Error running command",
+            });
+        });
+    }
+}
+
+module.exports = IndexedDBStoreWorker;
@@ -0,0 +1,195 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import q from "q";
+import {MatrixInMemoryStore} from "./memory";
+import utils from "../utils";
+import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";
+import RemoteIndexedDBStoreBackend from "./indexeddb-remote-backend.js";
+import User from "../models/user";
+import {MatrixEvent} from "../models/event";
+
+/**
+ * This is an internal module. See {@link IndexedDBStore} for the public class.
+ * @module store/indexeddb
+ */
+
+// If this value is too small we'll be writing very often which will cause
+// noticable stop-the-world pauses. If this value is too big we'll be writing
+// so infrequently that the /sync size gets bigger on reload. Writing more
+// often does not affect the length of the pause since the entire /sync
+// response is persisted each time.
+const WRITE_DELAY_MS = 1000 * 60 * 5; // once every 5 minutes
+
+
+/**
+ * Construct a new Indexed Database store, which extends MatrixInMemoryStore.
+ *
+ * This store functions like a MatrixInMemoryStore except it periodically persists
+ * the contents of the store to an IndexedDB backend.
+ *
+ * All data is still kept in-memory but can be loaded from disk by calling
+ * <code>startup()</code>. This can make startup times quicker as a complete
+ * sync from the server is not required. This does not reduce memory usage as all
+ * the data is eagerly fetched when <code>startup()</code> is called.
+ * <pre>
+ * let opts = { localStorage: window.localStorage };
+ * let store = new IndexedDBStore();
+ * await store.startup(); // load from indexed db
+ * let client = sdk.createClient({
+ *     store: store,
+ * });
+ * client.startClient();
+ * client.on("sync", function(state, prevState, data) {
+ *     if (state === "PREPARED") {
+ *         console.log("Started up, now with go faster stripes!");
+ *     }
+ * });
+ * </pre>
+ *
+ * @constructor
+ * @extends MatrixInMemoryStore
+ * @param {Object} opts Options object.
+ * @param {Object} opts.indexedDB The Indexed DB interface e.g.
+ * <code>window.indexedDB</code>
+ * @param {string=} opts.dbName Optional database name. The same name must be used
+ * to open the same database.
+ * @param {string=} opts.workerScript Optional URL to a script to invoke a web
+ * worker with to run IndexedDB queries on the web worker. The IndexedDbStoreWorker
+ * class is provided for this purpose and requires the application to provide a
+ * trivial wrapper script around it.
+ * @param {Object=} opts.workerApi The webWorker API object. If omitted, the global Worker
+ * object will be used if it exists.
+ * @prop {IndexedDBStoreBackend} backend The backend instance. Call through to
+ * this API if you need to perform specific indexeddb actions like deleting the
+ * database.
+ */
+const IndexedDBStore = function IndexedDBStore(opts) {
+    MatrixInMemoryStore.call(this, opts);
+
+    if (!opts.indexedDB) {
+        throw new Error('Missing required option: indexedDB');
+    }
+
+    if (opts.workerScript) {
+        // try & find a webworker-compatible API
+        let workerApi = opts.workerApi;
+        if (!workerApi) {
+            // default to the global Worker object (which is where it in a browser)
+            workerApi = global.Worker;
+        }
+        this.backend = new RemoteIndexedDBStoreBackend(
+            opts.workerScript, opts.dbName, workerApi,
+        );
+    } else {
+        this.backend = new LocalIndexedDBStoreBackend(opts.indexedDB, opts.dbName);
+    }
+
+    this.startedUp = false;
+    this._syncTs = 0;
+
+    // Records the last-modified-time of each user at the last point we saved
+    // the database, such that we can derive the set if users that have been
+    // modified since we last saved.
+    this._userModifiedMap = {
+        // user_id : timestamp
+    };
+};
+utils.inherits(IndexedDBStore, MatrixInMemoryStore);
+
+/**
+ * @return {Promise} Resolved when loaded from indexed db.
+  */
+IndexedDBStore.prototype.startup = function() {
+    if (this.startedUp) {
+        return q();
+    }
+
+    return this.backend.connect().then(() => {
+        return this.backend.getUserPresenceEvents();
+    }).then((userPresenceEvents) => {
+        userPresenceEvents.forEach(([userId, rawEvent]) => {
+            const u = new User(userId);
+            if (rawEvent) {
+                u.setPresenceEvent(new MatrixEvent(rawEvent));
+            }
+            this._userModifiedMap[u.userId] = u.getLastModifiedTime();
+            this.storeUser(u);
+        });
+    });
+};
+
+/**
+ * @return {Promise} Resolves with a sync response to restore the
+ * client state to where it was at the last save, or null if there
+ * is no saved sync data.
+ */
+IndexedDBStore.prototype.getSavedSync = function() {
+    return this.backend.getSavedSync();
+};
+
+/**
+ * Delete all data from this store.
+ * @return {Promise} Resolves if the data was deleted from the database.
+ */
+IndexedDBStore.prototype.deleteAllData = function() {
+    MatrixInMemoryStore.prototype.deleteAllData.call(this);
+    return this.backend.clearDatabase().then(() => {
+        console.log("Deleted indexeddb data.");
+    }, (err) => {
+        console.error("Failed to delete indexeddb data: ", err);
+        throw err;
+    });
+};
+
+/**
+ * Possibly write data to the database.
+ * @return {Promise} Promise resolves after the write completes.
+ */
+IndexedDBStore.prototype.save = function() {
+    const now = Date.now();
+    if (now - this._syncTs > WRITE_DELAY_MS) {
+        return this._reallySave();
+    }
+    return q();
+};
+
+IndexedDBStore.prototype._reallySave = function() {
+    this._syncTs = Date.now(); // set now to guard against multi-writes
+
+    // work out changed users (this doesn't handle deletions but you
+    // can't 'delete' users as they are just presence events).
+    const userTuples = [];
+    for (const u of this.getUsers()) {
+        if (this._userModifiedMap[u.userId] === u.getLastModifiedTime()) continue;
+        if (!u.events.presence) continue;
+
+        userTuples.push([u.userId, u.events.presence.event]);
+
+        // note that we've saved this version of the user
+        this._userModifiedMap[u.userId] = u.getLastModifiedTime();
+    }
+
+    return this.backend.syncToDatabase(userTuples).catch((err) => {
+        console.error("sync fail:", err);
+    });
+};
+
+IndexedDBStore.prototype.setSyncData = function(syncData) {
+    return this.backend.setSyncData(syncData);
+};
+
+module.exports.IndexedDBStore = IndexedDBStore;
@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -20,6 +21,7 @@ limitations under the License.
 */
 const utils = require("../utils");
 const User = require("../models/user");
+ const q = require("q");

 /**
 * Construct a new in-memory data store for the Matrix Client.
@@ -275,9 +277,58 @@ module.exports.MatrixInMemoryStore.prototype = {
        return this.accountData[eventType];
    },

-    // TODO
-    //setMaxHistoryPerRoom: function(maxHistory) {},
+    /**
+     * setSyncData does nothing as there is no backing data store.
+     *
+     * @param {Object} syncData The sync data
+     * @return {Promise} An immediately resolved promise.
+     */
+    setSyncData: function(syncData) {
+        return q();
+    },

-    // TODO
-    //reapOldMessages: function() {},
+    /**
+     * Save does nothing as there is no backing data store.
+     */
+    save: function() {},
+
+    /**
+     * Startup does nothing as this store doesn't require starting up.
+     * @return {Promise} An immediately resolved promise.
+     */
+    startup: function() {
+        return q();
+    },
+
+    /**
+     * @return {Promise} Resolves with a sync response to restore the
+     * client state to where it was at the last save, or null if there
+     * is no saved sync data.
+     */
+    getSavedSync: function() {
+        return q(null);
+    },
+
+    /**
+     * Delete all data from this store.
+     * @return {Promise} An immediately resolved promise.
+     */
+    deleteAllData: function() {
+        this.rooms = {
+            // roomId: Room
+        };
+        this.users = {
+            // userId: User
+        };
+        this.syncToken = null;
+        this.filters = {
+            // userId: {
+            //    filterId: Filter
+            // }
+        };
+        this.accountData = {
+            // type : content
+        };
+        return q();
+    },
 };
@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,6 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 "use strict";
+import q from "q";
 /**
 * This is an internal module.
 * @module store/stub
@@ -180,11 +182,46 @@ StubStore.prototype = {

    },

-    // TODO
-    //setMaxHistoryPerRoom: function(maxHistory) {},
+    /**
+     * setSyncData does nothing as there is no backing data store.
+     *
+     * @param {Object} syncData The sync data
+     * @return {Promise} An immediately resolved promise.
+     */
+    setSyncData: function(syncData) {
+        return q();
+    },

-    // TODO
-    //reapOldMessages: function() {},
+    /**
+     * Save does nothing as there is no backing data store.
+     */
+    save: function() {},
+
+    /**
+     * Startup does nothing.
+     * @return {Promise} An immediately resolved promise.
+     */
+    startup: function() {
+        return q();
+    },
+
+    /**
+     * @return {Promise} Resolves with a sync response to restore the
+     * client state to where it was at the last save, or null if there
+     * is no saved sync data.
+     */
+    getSavedSync: function() {
+        return q(null);
+    },
+
+    /**
+     * Delete all data from this store. Does nothing since this store
+     * doesn't store anything.
+     * @return {Promise} An immediately resolved promise.
+     */
+    deleteAllData: function() {
+        return q();
+    },
 };

 /** Stub Store class. */
@@ -0,0 +1,488 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * This is an internal module. See {@link SyncAccumulator} for the public class.
+ * @module sync-accumulator
+ */
+
+import utils from "./utils";
+
+
+/**
+ * The purpose of this class is to accumulate /sync responses such that a
+ * complete "initial" JSON response can be returned which accurately represents
+ * the sum total of the /sync responses accumulated to date. It only handles
+ * room data: that is, everything under the "rooms" top-level key.
+ *
+ * This class is used when persisting room data so a complete /sync response can
+ * be loaded from disk and incremental syncs can be performed on the server,
+ * rather than asking the server to do an initial sync on startup.
+ */
+class SyncAccumulator {
+
+    /**
+     * @param {Object} opts
+     * @param {Number=} opts.maxTimelineEntries The ideal maximum number of
+     * timeline entries to keep in the sync response. This is best-effort, as
+     * clients do not always have a back-pagination token for each event, so
+     * it's possible there may be slightly *less* than this value. There will
+     * never be more. This cannot be 0 or else it makes it impossible to scroll
+     * back in a room. Default: 50.
+     */
+    constructor(opts) {
+        opts = opts || {};
+        opts.maxTimelineEntries = opts.maxTimelineEntries || 50;
+        this.opts = opts;
+        this.accountData = {
+            //$event_type: Object
+        };
+        this.inviteRooms = {
+            //$roomId: { ... sync 'invite' json data ... }
+        };
+        this.joinRooms = {
+            //$roomId: {
+            //    _currentState: { $event_type: { $state_key: json } },
+            //    _timeline: [
+            //       { event: $event, token: null|token },
+            //       { event: $event, token: null|token },
+            //       { event: $event, token: null|token },
+            //       ...
+            //    ],
+            //    _accountData: { $event_type: json },
+            //    _unreadNotifications: { ... unread_notifications JSON ... },
+            //    _readReceipts: { $user_id: { data: $json, eventId: $event_id }}
+            //}
+        };
+        // the /sync token which corresponds to the last time rooms were
+        // accumulated. We remember this so that any caller can obtain a
+        // coherent /sync response and know at what point they should be
+        // streaming from without losing events.
+        this.nextBatch = null;
+    }
+
+    accumulate(syncResponse) {
+        this._accumulateRooms(syncResponse);
+        this._accumulateAccountData(syncResponse);
+        this.nextBatch = syncResponse.next_batch;
+    }
+
+    _accumulateAccountData(syncResponse) {
+        if (!syncResponse.account_data || !syncResponse.account_data.events) {
+            return;
+        }
+        // Clobbers based on event type.
+        syncResponse.account_data.events.forEach((e) => {
+            this.accountData[e.type] = e;
+        });
+    }
+
+    /**
+     * Accumulate incremental /sync room data.
+     * @param {Object} syncResponse the complete /sync JSON
+     */
+    _accumulateRooms(syncResponse) {
+        if (!syncResponse.rooms) {
+            return;
+        }
+        if (syncResponse.rooms.invite) {
+            Object.keys(syncResponse.rooms.invite).forEach((roomId) => {
+                this._accumulateRoom(
+                    roomId, "invite", syncResponse.rooms.invite[roomId],
+                );
+            });
+        }
+        if (syncResponse.rooms.join) {
+            Object.keys(syncResponse.rooms.join).forEach((roomId) => {
+                this._accumulateRoom(
+                    roomId, "join", syncResponse.rooms.join[roomId],
+                );
+            });
+        }
+        if (syncResponse.rooms.leave) {
+            Object.keys(syncResponse.rooms.leave).forEach((roomId) => {
+                this._accumulateRoom(
+                    roomId, "leave", syncResponse.rooms.leave[roomId],
+                );
+            });
+        }
+    }
+
+    _accumulateRoom(roomId, category, data) {
+        // Valid /sync state transitions
+        //       +--------+ <======+            1: Accept an invite
+        //   +== | INVITE |        | (5)        2: Leave a room
+        //   |   +--------+ =====+ |            3: Join a public room previously
+        //   |(1)            (4) | |               left (handle as if new room)
+        //   V         (2)       V |            4: Reject an invite
+        // +------+ ========> +--------+         5: Invite to a room previously
+        // | JOIN |    (3)    | LEAVE* |            left (handle as if new room)
+        // +------+ <======== +--------+
+        //
+        // * equivalent to "no state"
+        switch (category) {
+            case "invite": // (5)
+                this._accumulateInviteState(roomId, data);
+                break;
+            case "join":
+                if (this.inviteRooms[roomId]) { // (1)
+                    // was previously invite, now join. We expect /sync to give
+                    // the entire state and timeline on 'join', so delete previous
+                    // invite state
+                    delete this.inviteRooms[roomId];
+                }
+                // (3)
+                this._accumulateJoinState(roomId, data);
+                break;
+            case "leave":
+                if (this.inviteRooms[roomId]) { // (4)
+                    delete this.inviteRooms[roomId];
+                } else { // (2)
+                    delete this.joinRooms[roomId];
+                }
+                break;
+            default:
+                console.error("Unknown cateogory: ", category);
+        }
+    }
+
+    _accumulateInviteState(roomId, data) {
+        if (!data.invite_state || !data.invite_state.events) { // no new data
+            return;
+        }
+        if (!this.inviteRooms[roomId]) {
+            this.inviteRooms[roomId] = {
+                invite_state: data.invite_state,
+            };
+            return;
+        }
+        // accumulate extra keys for invite->invite transitions
+        // clobber based on event type / state key
+        // We expect invite_state to be small, so just loop over the events
+        const currentData = this.inviteRooms[roomId];
+        data.invite_state.events.forEach((e) => {
+            let hasAdded = false;
+            for (let i = 0; i < currentData.invite_state.events.length; i++) {
+                const current = currentData.invite_state.events[i];
+                if (current.type === e.type && current.state_key == e.state_key) {
+                    currentData.invite_state.events[i] = e; // update
+                    hasAdded = true;
+                }
+            }
+            if (!hasAdded) {
+                currentData.invite_state.events.push(e);
+            }
+        });
+    }
+
+    // Accumulate timeline and state events in a room.
+    _accumulateJoinState(roomId, data) {
+        // We expect this function to be called a lot (every /sync) so we want
+        // this to be fast. /sync stores events in an array but we often want
+        // to clobber based on type/state_key. Rather than convert arrays to
+        // maps all the time, just keep private maps which contain
+        // the actual current accumulated sync state, and array-ify it when
+        // getJSON() is called.
+
+        // State resolution:
+        // The 'state' key is the delta from the previous sync (or start of time
+        // if no token was supplied), to the START of the timeline. To obtain
+        // the current state, we need to "roll forward" state by reading the
+        // timeline. We want to store the current state so we can drop events
+        // out the end of the timeline based on opts.maxTimelineEntries.
+        //
+        //      'state'     'timeline'     current state
+        // |-------x<======================>x
+        //          T   I   M   E
+        //
+        // When getJSON() is called, we 'roll back' the current state by the
+        // number of entries in the timeline to work out what 'state' should be.
+
+        // Back-pagination:
+        // On an initial /sync, the server provides a back-pagination token for
+        // the start of the timeline. When /sync deltas come down, they also
+        // include back-pagination tokens for the start of the timeline. This
+        // means not all events in the timeline have back-pagination tokens, as
+        // it is only the ones at the START of the timeline which have them.
+        // In order for us to have a valid timeline (and back-pagination token
+        // to match), we need to make sure that when we remove old timeline
+        // events, that we roll forward to an event which has a back-pagination
+        // token. This means we can't keep a strict sliding-window based on
+        // opts.maxTimelineEntries, and we may have a few less. We should never
+        // have more though, provided that the /sync limit is less than or equal
+        // to opts.maxTimelineEntries.
+
+        if (!this.joinRooms[roomId]) {
+            // Create truly empty objects so event types of 'hasOwnProperty' and co
+            // don't cause this code to break.
+            this.joinRooms[roomId] = {
+                _currentState: Object.create(null),
+                _timeline: [],
+                _accountData: Object.create(null),
+                _unreadNotifications: {},
+                _readReceipts: {},
+            };
+        }
+        const currentData = this.joinRooms[roomId];
+
+        if (data.account_data && data.account_data.events) {
+            // clobber based on type
+            data.account_data.events.forEach((e) => {
+                currentData._accountData[e.type] = e;
+            });
+        }
+
+        // these probably clobber, spec is unclear.
+        if (data.unread_notifications) {
+            currentData._unreadNotifications = data.unread_notifications;
+        }
+
+        if (data.ephemeral && data.ephemeral.events) {
+            data.ephemeral.events.forEach((e) => {
+                // We purposefully do not persist m.typing events.
+                // Technically you could refresh a browser before the timer on a
+                // typing event is up, so it'll look like you aren't typing when
+                // you really still are. However, the alternative is worse. If
+                // we do persist typing events, it will look like people are
+                // typing forever until someone really does start typing (which
+                // will prompt Synapse to send down an actual m.typing event to
+                // clobber the one we persisted).
+                if (e.type !== "m.receipt" || !e.content) {
+                    // This means we'll drop unknown ephemeral events but that
+                    // seems okay.
+                    return;
+                }
+                // Handle m.receipt events. They clobber based on:
+                //   (user_id, receipt_type)
+                // but they are keyed in the event as:
+                //   content:{ $event_id: { $receipt_type: { $user_id: {json} }}}
+                // so store them in the former so we can accumulate receipt deltas
+                // quickly and efficiently (we expect a lot of them). Fold the
+                // receipt type into the key name since we only have 1 at the
+                // moment (m.read) and nested JSON objects are slower and more
+                // of a hassle to work with. We'll inflate this back out when
+                // getJSON() is called.
+                Object.keys(e.content).forEach((eventId) => {
+                    if (!e.content[eventId]["m.read"]) {
+                        return;
+                    }
+                    Object.keys(e.content[eventId]["m.read"]).forEach((userId) => {
+                        // clobber on user ID
+                        currentData._readReceipts[userId] = {
+                            data: e.content[eventId]["m.read"][userId],
+                            eventId: eventId,
+                        };
+                    });
+                });
+            });
+        }
+
+        // if we got a limited sync, we need to remove all timeline entries or else
+        // we will have gaps in the timeline.
+        if (data.timeline && data.timeline.limited) {
+            currentData._timeline = [];
+        }
+
+        // Work out the current state. The deltas need to be applied in the order:
+        // - existing state which didn't come down /sync.
+        // - State events under the 'state' key.
+        // - State events in the 'timeline'.
+        if (data.state && data.state.events) {
+            data.state.events.forEach((e) => {
+                setState(currentData._currentState, e);
+            });
+        }
+        if (data.timeline && data.timeline.events) {
+            data.timeline.events.forEach((e, index) => {
+                // this nops if 'e' isn't a state event
+                setState(currentData._currentState, e);
+                // append the event to the timeline. The back-pagination token
+                // corresponds to the first event in the timeline
+                currentData._timeline.push({
+                    event: e,
+                    token: index === 0 ? data.timeline.prev_batch : null,
+                });
+            });
+        }
+
+        // attempt to prune the timeline by jumping between events which have
+        // pagination tokens.
+        if (currentData._timeline.length > this.opts.maxTimelineEntries) {
+            const startIndex = (
+                currentData._timeline.length - this.opts.maxTimelineEntries
+            );
+            for (let i = startIndex; i < currentData._timeline.length; i++) {
+                if (currentData._timeline[i].token) {
+                    // keep all events after this, including this one
+                    currentData._timeline = currentData._timeline.slice(
+                        i, currentData._timeline.length,
+                    );
+                    break;
+                }
+            }
+        }
+    }
+
+    /**
+     * Return everything under the 'rooms' key from a /sync response which
+     * represents all room data that should be stored. This should be paired
+     * with the sync token which represents the most recent /sync response
+     * provided to accumulate().
+     * @return {Object} An object with a "nextBatch", "roomsData" and "accountData"
+     * keys.
+     * The "nextBatch" key is a string which represents at what point in the
+     * /sync stream the accumulator reached. This token should be used when
+     * restarting a /sync stream at startup. Failure to do so can lead to missing
+     * events. The "roomsData" key is an Object which represents the entire
+     * /sync response from the 'rooms' key onwards. The "accountData" key is
+     * a list of raw events which represent global account data.
+     */
+    getJSON() {
+        const data = {
+            join: {},
+            invite: {},
+            // always empty. This is set by /sync when a room was previously
+            // in 'invite' or 'join'. On fresh startup, the client won't know
+            // about any previous room being in 'invite' or 'join' so we can
+            // just omit mentioning it at all, even if it has previously come
+            // down /sync.
+            // The notable exception is when a client is kicked or banned:
+            // we may want to hold onto that room so the client can clearly see
+            // why their room has disappeared. We don't persist it though because
+            // it is unclear *when* we can safely remove the room from the DB.
+            // Instead, we assume that if you're loading from the DB, you've
+            // refreshed the page, which means you've seen the kick/ban already.
+            leave: {},
+        };
+        Object.keys(this.inviteRooms).forEach((roomId) => {
+            data.invite[roomId] = this.inviteRooms[roomId];
+        });
+        Object.keys(this.joinRooms).forEach((roomId) => {
+            const roomData = this.joinRooms[roomId];
+            const roomJson = {
+                ephemeral: { events: [] },
+                account_data: { events: [] },
+                state: { events: [] },
+                timeline: {
+                    events: [],
+                    prev_batch: null,
+                },
+                unread_notifications: roomData._unreadNotifications,
+            };
+            // Add account data
+            Object.keys(roomData._accountData).forEach((evType) => {
+                roomJson.account_data.events.push(roomData._accountData[evType]);
+            });
+
+            // Add receipt data
+            const receiptEvent = {
+                type: "m.receipt",
+                room_id: roomId,
+                content: {
+                    // $event_id: { "m.read": { $user_id: $json } }
+                },
+            };
+            Object.keys(roomData._readReceipts).forEach((userId) => {
+                const receiptData = roomData._readReceipts[userId];
+                if (!receiptEvent.content[receiptData.eventId]) {
+                    receiptEvent.content[receiptData.eventId] = {
+                        "m.read": {},
+                    };
+                }
+                receiptEvent.content[receiptData.eventId]["m.read"][userId] = (
+                    receiptData.data
+                );
+            });
+            // add only if we have some receipt data
+            if (Object.keys(receiptEvent.content).length > 0) {
+                roomJson.ephemeral.events.push(receiptEvent);
+            }
+
+            // Add timeline data
+            roomData._timeline.forEach((msgData) => {
+                if (!roomJson.timeline.prev_batch) {
+                    // the first event we add to the timeline MUST match up to
+                    // the prev_batch token.
+                    if (!msgData.token) {
+                        return; // this shouldn't happen as we prune constantly.
+                    }
+                    roomJson.timeline.prev_batch = msgData.token;
+                }
+                roomJson.timeline.events.push(msgData.event);
+            });
+
+            // Add state data: roll back current state to the start of timeline,
+            // by "reverse clobbering" from the end of the timeline to the start.
+            // Convert maps back into arrays.
+            const rollBackState = Object.create(null);
+            for (let i = roomJson.timeline.events.length - 1; i >=0; i--) {
+                const timelineEvent = roomJson.timeline.events[i];
+                if (timelineEvent.state_key === null ||
+                        timelineEvent.state_key === undefined) {
+                    continue; // not a state event
+                }
+                // since we're going back in time, we need to use the previous
+                // state value else we'll break causality. We don't have the
+                // complete previous state event, so we need to create one.
+                const prevStateEvent = utils.deepCopy(timelineEvent);
+                if (prevStateEvent.unsigned) {
+                    if (prevStateEvent.unsigned.prev_content) {
+                        prevStateEvent.content = prevStateEvent.unsigned.prev_content;
+                    }
+                    if (prevStateEvent.unsigned.prev_sender) {
+                        prevStateEvent.sender = prevStateEvent.unsigned.prev_sender;
+                    }
+                }
+                setState(rollBackState, prevStateEvent);
+            }
+            Object.keys(roomData._currentState).forEach((evType) => {
+                Object.keys(roomData._currentState[evType]).forEach((stateKey) => {
+                    let ev = roomData._currentState[evType][stateKey];
+                    if (rollBackState[evType] && rollBackState[evType][stateKey]) {
+                        // use the reverse clobbered event instead.
+                        ev = rollBackState[evType][stateKey];
+                    }
+                    roomJson.state.events.push(ev);
+                });
+            });
+            data.join[roomId] = roomJson;
+        });
+
+        // Add account data
+        const accData = [];
+        Object.keys(this.accountData).forEach((evType) => {
+            accData.push(this.accountData[evType]);
+        });
+
+        return {
+            nextBatch: this.nextBatch,
+            roomsData: data,
+            accountData: accData,
+        };
+    }
+}
+
+function setState(eventMap, event) {
+    if (event.state_key === null || event.state_key === undefined || !event.type) {
+        return;
+    }
+    if (!eventMap[event.type]) {
+        eventMap[event.type] = Object.create(null);
+    }
+    eventMap[event.type][event.state_key] = event;
+}
+
+module.exports = SyncAccumulator;
@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -38,6 +39,11 @@ const DEBUG = true;
 // to determine the max time we're willing to wait.
 const BUFFER_PERIOD_MS = 80 * 1000;

+// Number of consecutive failed syncs that will lead to a syncState of ERROR as opposed
+// to RECONNECTING. This is needed to inform the client of server issues when the
+// keepAlive is successful but the server /sync fails.
+const FAILED_SYNC_ERROR_THRESHOLD = 3;
+
 function getFilterName(userId, suffix) {
    // scope this on the user ID because people may login on many accounts
    // and they all need to be stored!
@@ -59,6 +65,11 @@ function debuglog() {
 * @param {MatrixClient} client The matrix client instance to use.
 * @param {Object} opts Config options
 * @param {module:crypto=} opts.crypto Crypto manager
+ * @param {Function=} opts.canResetEntireTimeline A function which is called
+ * with a room ID and returns a boolean. It should return 'true' if the SDK can
+ * SAFELY remove events from this room. It may not be safe to remove events if
+ * there are other references to the timelines for this room.
+ * Default: returns false.
 */
 function SyncApi(client, opts) {
    this.client = client;
@@ -69,14 +80,21 @@ function SyncApi(client, opts) {
    opts.resolveInvitesToProfiles = opts.resolveInvitesToProfiles || false;
    opts.pollTimeout = opts.pollTimeout || (30 * 1000);
    opts.pendingEventOrdering = opts.pendingEventOrdering || "chronological";
+    if (!opts.canResetEntireTimeline) {
+        opts.canResetEntireTimeline = function(roomId) {
+            return false;
+        };
+    }
    this.opts = opts;
    this._peekRoomId = null;
    this._currentSyncRequest = null;
    this._syncState = null;
+    this._catchingUp = false;
    this._running = false;
    this._keepAliveTimer = null;
    this._connectionReturnedDefer = null;
    this._notifEvents = []; // accumulator of sync events in the current sync response
+    this._failedSyncCount = 0; // Number of consecutive failed /sync requests

    if (client.getNotifTimelineSet()) {
        reEmit(client, client.getNotifTimelineSet(),
@@ -360,9 +378,6 @@ SyncApi.prototype.getSyncState = function() {
 * Main entry point
 */
 SyncApi.prototype.sync = function() {
-    debuglog("SyncApi.sync: starting with sync token " +
-             this.client.store.getSyncToken());
-
    const client = this.client;
    const self = this;

@@ -486,9 +501,30 @@ SyncApi.prototype._sync = function(syncOptions) {

    const syncToken = client.store.getSyncToken();

+    let pollTimeout = this.opts.pollTimeout;
+
+    if (this.getSyncState() !== 'SYNCING' || this._catchingUp) {
+        // unless we are happily syncing already, we want the server to return
+        // as quickly as possible, even if there are no events queued. This
+        // serves two purposes:
+        //
+        // * When the connection dies, we want to know asap when it comes back,
+        //   so that we can hide the error from the user. (We don't want to
+        //   have to wait for an event or a timeout).
+        //
+        // * We want to know if the server has any to_device messages queued up
+        //   for us. We do that by calling it with a zero timeout until it
+        //   doesn't give us any more to_device messages.
+        this._catchingUp = true;
+        pollTimeout = 0;
+    }
+
+    // normal timeout= plus buffer time
+    const clientSideTimeoutMs = pollTimeout + BUFFER_PERIOD_MS;
+
    const qps = {
        filter: filterId,
-        timeout: this.opts.pollTimeout,
+        timeout: pollTimeout,
    };

    if (syncToken) {
@@ -508,19 +544,59 @@ SyncApi.prototype._sync = function(syncOptions) {
        qps.timeout = 0;
    }

-    // normal timeout= plus buffer time
-    const clientSideTimeoutMs = this.opts.pollTimeout + BUFFER_PERIOD_MS;
+    let isCachedResponse = false;

-    this._currentSyncRequest = client._http.authedRequest(
-        undefined, "GET", "/sync", qps, undefined, clientSideTimeoutMs,
-    );
+    let syncPromise;
+    if (!syncOptions.hasSyncedBefore) {
+        // Don't do an HTTP hit to /sync. Instead, load up the persisted /sync data,
+        // if there is data there.
+        syncPromise = client.store.getSavedSync();
+    } else {
+        syncPromise = q(null);
+    }
+
+    syncPromise.then((savedSync) => {
+        if (savedSync) {
+            debuglog("sync(): not doing HTTP hit, instead returning stored /sync data");
+            isCachedResponse = true;
+            return {
+                next_batch: savedSync.nextBatch,
+                rooms: savedSync.roomsData,
+                account_data: {
+                    events: savedSync.accountData,
+                },
+            };
+        } else {
+            //debuglog('Starting sync since=' + syncToken);
+            this._currentSyncRequest = client._http.authedRequest(
+                undefined, "GET", "/sync", qps, undefined, clientSideTimeoutMs,
+            );
+            return this._currentSyncRequest;
+        }
+    }).then(function(data) {
+        //debuglog('Completed sync, next_batch=' + data.next_batch);

-    this._currentSyncRequest.done(function(data) {
        // set the sync token NOW *before* processing the events. We do this so
        // if something barfs on an event we can skip it rather than constantly
        // polling with the same token.
        client.store.setSyncToken(data.next_batch);

+        // Reset after a successful sync
+        self._failedSyncCount = 0;
+
+        // We need to wait until the sync data has been sent to the backend
+        // because it appears that the sync data gets modified somewhere in
+        // processing it in such a way as to make it no longer cloneable.
+        // XXX: Find out what is modifying it!
+        if (!isCachedResponse) {
+            // Don't give the store back its own cached data
+            return client.store.setSyncData(data).then(() => {
+                return data;
+            });
+        } else {
+            return q(data);
+        }
+    }).done((data) => {
        try {
            self._processSyncResponse(syncToken, data);
        } catch (e) {
@@ -533,6 +609,7 @@ SyncApi.prototype._sync = function(syncOptions) {
        const syncEventData = {
            oldSyncToken: syncToken,
            nextSyncToken: data.next_batch,
+            catchingUp: self._catchingUp,
        };

        if (!syncOptions.hasSyncedBefore) {
@@ -541,8 +618,17 @@ SyncApi.prototype._sync = function(syncOptions) {
        }

        // keep emitting SYNCING -> SYNCING for clients who want to do bulk updates
-        self._updateSyncState("SYNCING", syncEventData);
+        if (!isCachedResponse) {
+            self._updateSyncState("SYNCING", syncEventData);

+            // tell databases that everything is now in a consistent state and can be
+            // saved (no point doing so if we only have the data we just got out of the
+            // store).
+            client.store.save();
+        }
+
+
+        // Begin next sync
        self._sync(syncOptions);
    }, function(err) {
        if (!self._running) {
@@ -557,6 +643,9 @@ SyncApi.prototype._sync = function(syncOptions) {
        console.error("/sync error %s", err);
        console.error(err);

+        self._failedSyncCount++;
+        console.log('Number of consecutive failed sync requests:', self._failedSyncCount);
+
        debuglog("Starting keep-alive");
        // Note that we do *not* mark the sync connection as
        // lost yet: we only do this if a keepalive poke
@@ -569,7 +658,11 @@ SyncApi.prototype._sync = function(syncOptions) {
            self._sync(syncOptions);
        });
        self._currentSyncRequest = null;
-        self._updateSyncState("RECONNECTING");
+        // Transition from RECONNECTING to ERROR after a given number of failed syncs
+        self._updateSyncState(
+            self._failedSyncCount >= FAILED_SYNC_ERROR_THRESHOLD ?
+                "ERROR" : "RECONNECTING",
+        );
    });
 };

@@ -656,7 +749,9 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
    }

    // handle to-device events
-    if (data.to_device && utils.isArray(data.to_device.events)) {
+    if (data.to_device && utils.isArray(data.to_device.events) &&
+        data.to_device.events.length > 0
+       ) {
        data.to_device.events
            .map(client.getEventMapper())
            .forEach(
@@ -666,8 +761,10 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
                        toDeviceEvent.getType() == "m.room.message" &&
                            content.msgtype == "m.bad.encrypted"
                    ) {
-                        console.warn(
-                            "Unable to decrypt to-device event: " + content.body,
+                        // the mapper already logged a warning.
+                        console.log(
+                            'Ignoring undecryptable to-device event from ' +
+                                toDeviceEvent.getSender(),
                        );
                        return;
                    }
@@ -675,6 +772,9 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
                    client.emit("toDeviceEvent", toDeviceEvent);
                },
            );
+    } else {
+        // no more to-device events: we can stop polling with a short timeout.
+        this._catchingUp = false;
    }

    // the returned json structure is a bit crap, so make it into a
@@ -783,7 +883,10 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
                // timeline.
                room.currentState.paginationToken = syncToken;
                self._deregisterStateListeners(room);
-                room.resetLiveTimeline(joinObj.timeline.prev_batch);
+                room.resetLiveTimeline(
+                    joinObj.timeline.prev_batch,
+                    self.opts.canResetEntireTimeline(room.roomId),
+                );

                // We have to assume any gap in any timeline is
                // reason to stop incrementally tracking notifications and
@@ -880,7 +983,7 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
 * @param {number} delay How long to delay until the first poll.
 *        defaults to a short, randomised interval (to prevent
 *        tightlooping if /versions succeeds but /sync etc. fail).
- * @return {promise}
+ * @return {promise} which resolves once the connection returns
 */
 SyncApi.prototype._startKeepAlives = function(delay) {
    if (delay === undefined) {
@@ -906,7 +1009,11 @@ SyncApi.prototype._startKeepAlives = function(delay) {
 };

 /**
+ * Make a dummy call to /_matrix/client/versions, to see if the HS is
+ * reachable.
 *
+ * On failure, schedules a call back to itself. On success, resolves
+ * this._connectionReturnedDefer.
 */
 SyncApi.prototype._pokeKeepAlive = function() {
    const self = this;
@@ -24,6 +24,47 @@ const DEBUG = true;  // set true to enable console logging.

 // events: hangup, error(err), replaced(call), state(state, oldState)

+/**
+ * Fires when the MatrixCall encounters an error when sending a Matrix event.
+ * <p>
+ * This is required to allow errors, which occur during sending of events, to bubble up.
+ * (This is because call.js does a hangup when it encounters a normal `error`, which in
+ * turn could lead to an UnknownDeviceError.)
+ * <p>
+ * To deal with an UnknownDeviceError when trying to send events, the application should let
+ * users know that there are new devices in the encrypted room (into which the event was
+ * sent) and give the user the options to resend unsent events or cancel them. Resending
+ * is done using {@link module:client~MatrixClient#resendEvent} and cancelling can be done by using
+ * {@link module:client~MatrixClient#cancelPendingEvent}.
+ * <p>
+ * MatrixCall will not do anything in response to an error that causes `send_event_error`
+ * to be emitted with the exception of sending `m.call.candidates`, which is retried upon
+ * failure when ICE candidates are being sent. This happens during call setup.
+ *
+ * @event module:webrtc/call~MatrixCall#"send_event_error"
+ * @param {Error} err The error caught from calling client.sendEvent in call.js.
+ * @example
+ * matrixCall.on("send_event_error", function(err){
+ *   console.error(err);
+ * });
+ */
+
+/**
+ * Fires whenever an error occurs when call.js encounters an issue with setting up the call.
+ * <p>
+ * The error given will have a code equal to either `MatrixCall.ERR_LOCAL_OFFER_FAILED` or
+ * `MatrixCall.ERR_NO_USER_MEDIA`. `ERR_LOCAL_OFFER_FAILED` is emitted when the local client
+ * fails to create an offer. `ERR_NO_USER_MEDIA` is emitted when the user has denied access
+ * to their audio/video hardware.
+ *
+ * @event module:webrtc/call~MatrixCall#"error"
+ * @param {Error} err The error raised by MatrixCall.
+ * @example
+ * matrixCall.on("error", function(err){
+ *   console.error(err.code, err);
+ * });
+ */
+
 /**
 * Construct a new Matrix Call.
 * @constructor
@@ -207,22 +248,23 @@ MatrixCall.prototype.pauseElement = function(element, queueId) {
 * serialising the operation into a chain of promises to avoid racing access
 * to the element
 * @param {Element} element HTMLMediaElement element to pause
- * @param {string} src the src attribute value to assign to the element
+ * @param {MediaStream} srcObject the srcObject attribute value to assign to the element
 * @param {string} queueId Arbitrary ID to track the chain of promises to be used
 */
-MatrixCall.prototype.assignElement = function(element, src, queueId) {
-    console.log("queuing assign on " + queueId + " element " + element + " for " + src);
+MatrixCall.prototype.assignElement = function(element, srcObject, queueId) {
+    console.log("queuing assign on " + queueId + " element " + element + " for " +
+        srcObject);
    if (this.mediaPromises[queueId]) {
        this.mediaPromises[queueId] =
            this.mediaPromises[queueId].then(function() {
                console.log("previous promise completed for " + queueId);
-                element.src = src;
+                element.srcObject = srcObject;
            }, function() {
                console.log("previous promise failed for " + queueId);
-                element.src = src;
+                element.srcObject = srcObject;
            });
    } else {
-        element.src = src;
+        element.srcObject = srcObject;
    }
 };

@@ -262,9 +304,7 @@ MatrixCall.prototype.setLocalVideoElement = function(element) {

    if (element && this.localAVStream && this.type === 'video') {
        element.autoplay = true;
-        this.assignElement(element,
-                           this.URL.createObjectURL(this.localAVStream),
-                           "localVideo");
+        this.assignElement(element, this.localAVStream, "localVideo");
        element.muted = true;
        const self = this;
        setTimeout(function() {
@@ -499,13 +539,9 @@ MatrixCall.prototype._gotUserMediaForInvite = function(stream) {
        videoEl.autoplay = true;
        if (this.screenSharingStream) {
            debuglog("Setting screen sharing stream to the local video element");
-            this.assignElement(videoEl,
-                   this.URL.createObjectURL(this.screenSharingStream),
-                   "localVideo");
+            this.assignElement(videoEl, this.screenSharingStream, "localVideo");
        } else {
-            this.assignElement(videoEl,
-                   this.URL.createObjectURL(stream),
-                   "localVideo");
+            this.assignElement(videoEl, stream, "localVideo");
        }
        videoEl.muted = true;
        setTimeout(function() {
@@ -547,9 +583,7 @@ MatrixCall.prototype._gotUserMediaForAnswer = function(stream) {

    if (localVidEl && self.type == 'video') {
        localVidEl.autoplay = true;
-        this.assignElement(localVidEl,
-               this.URL.createObjectURL(stream),
-               "localVideo");
+        this.assignElement(localVidEl, stream, "localVideo");
        localVidEl.muted = true;
        setTimeout(function() {
            const vel = self.getLocalVideoElement();
@@ -903,7 +937,11 @@ const setState = function(self, state) {
 * @return {Promise}
 */
 const sendEvent = function(self, eventType, content) {
-    return self.client.sendEvent(self.roomId, eventType, content);
+    return self.client.sendEvent(self.roomId, eventType, content).catch(
+        (err) => {
+            self.emit('send_event_error', err);
+        },
+    );
 };

 const sendCandidate = function(self, content) {
@@ -922,19 +960,19 @@ const terminate = function(self, hangupParty, hangupReason, shouldEmit) {
        if (self.getRemoteVideoElement().pause) {
            self.pauseElement(self.getRemoteVideoElement(), "remoteVideo");
        }
-        self.assignElement(self.getRemoteVideoElement(), "", "remoteVideo");
+        self.assignElement(self.getRemoteVideoElement(), null, "remoteVideo");
    }
    if (self.getRemoteAudioElement()) {
        if (self.getRemoteAudioElement().pause) {
            self.pauseElement(self.getRemoteAudioElement(), "remoteAudio");
        }
-        self.assignElement(self.getRemoteAudioElement(), "", "remoteAudio");
+        self.assignElement(self.getRemoteAudioElement(), null, "remoteAudio");
    }
    if (self.getLocalVideoElement()) {
        if (self.getLocalVideoElement().pause) {
            self.pauseElement(self.getLocalVideoElement(), "localVideo");
        }
-        self.assignElement(self.getLocalVideoElement(), "", "localVideo");
+        self.assignElement(self.getLocalVideoElement(), null, "localVideo");
    }
    self.hangupParty = hangupParty;
    self.hangupReason = hangupReason;
@@ -992,9 +1030,7 @@ const _tryPlayRemoteStream = function(self) {
    if (self.getRemoteVideoElement() && self.remoteAVStream) {
        const player = self.getRemoteVideoElement();
        player.autoplay = true;
-        self.assignElement(player,
-                           self.URL.createObjectURL(self.remoteAVStream),
-                           "remoteVideo");
+        self.assignElement(player, self.remoteAVStream, "remoteVideo");
        setTimeout(function() {
            const vel = self.getRemoteVideoElement();
            if (vel.play) {
@@ -1012,9 +1048,7 @@ const _tryPlayRemoteAudioStream = function(self) {
    if (self.getRemoteAudioElement() && self.remoteAStream) {
        const player = self.getRemoteAudioElement();
        player.autoplay = true;
-        self.assignElement(player,
-                           self.URL.createObjectURL(self.remoteAStream),
-                           "remoteAudio");
+        self.assignElement(player, self.remoteAStream, "remoteAudio");
        setTimeout(function() {
            const ael = self.getRemoteAudioElement();
            if (ael.play) {